Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "Jan integrates Cortex"

Browse Source 
This reverts commit ad6fbea22df6deaba31e146dddb456e4a5d5dd75

Revert "chore: add engine logo from local instead of metadata logo (#3363)"

This reverts commit ad6fbea22df6deaba31e146dddb456e4a5d5dd75.

Revert "fix: LaTex formula render issue (#3353)"

This reverts commit 3b2c84c4fee61b886c883c68801be3bc5a8584ad.

Revert "chore: minor ui improvement (#3352)"

This reverts commit 6dd387db2b5b9890f19d0c3505cf9cb770fd492f.

Revert "fix: failed to relaunch app to update (#3351)"

This reverts commit fcaf98a2fa4e674799602e8093914bcc04ced153.

Revert "chore: add back GPU information to system monitoring bar (#3350)"

This reverts commit 03455a91807c7af6c6325901997c6d7231d2cd0d.

Revert "fix: empty model page not shown when delete all threads and models (#3343)"

This reverts commit 9e29fcd69eb9085843896686806fd453a1285723.

Revert "feat: allow user configure remote model from my model (#3348)"

This reverts commit fdab8af057f80cf1ccaae0dc42c4e5161925f51e.

Revert "chore: ui fix button outline for configure cloud model (#3347)"

This reverts commit fe8ed1f26dc86ead92ffea4f36e2989caf7dad88.

Revert "feat: move icon create new thread into top panel (#3346)"

This reverts commit 46cb1b45b997181e2188f8dafb2fc0d0cc12ddcd.

Revert "chore(UI): update experience model dropdown (#3342)"

This reverts commit 8b44613015a907dc491113aeb99c963080424892.

Revert "Chore/simple bug template and correct a copy (#3344)"

This reverts commit 23cd5fd3979e7529811045da5c4912369bcc7532.

Revert "chore(ui): fix alignment loader starter screen (#3338)"

This reverts commit e9f5d2f837ce323b0851ea04cded913ab433388c.

Revert "Increase retry upload to R2 to 5 times (#3337)"

This reverts commit dcfb497934edc795955d971b6d391ee1e6309a03.

Revert "fix: broken jan build - add log trace (jan.log) (#3336)"

This reverts commit 77422c3a7ed240909942ac0d8c4b259af8d87a28.

Revert "chore: disable quick ask (#3334)"

This reverts commit 6e4b6b09ae009149f262d86d5b19bb8096267c19.

Revert "fix: update legacy path (#3328)"

This reverts commit 5eb112142c6431cfe0cdf11ce28810ca650a5427.

Revert "chore: add cortex version (#3318)"

This reverts commit 60587649c56a1f24272e763f25aa5b4042f7719a.

Revert "fix: broken app due to incorrect api path (#3316)"

This reverts commit 3de4eab2a0dfbf9f593d73b9dde6bca1d9df2279.

Revert "feat: modal waiting cortex (#3306)"

This reverts commit 1f5168d4af9080b867c19d334c398bf32e4f54b8.

Revert "fix: refresh should not create new thread (#3314)"

This reverts commit 624d07703c50ea332ed4eeac9dc3a26bc8190d08.

Revert "fix: avoid lose title threads (#3307)"

This reverts commit a4f5fda104c2d1e01ea72798f055e5b4e3cfd616.

Revert "feat: change data folder (#3309)"

This reverts commit b43242b9b24352c7f90995eccab753dede679616.

Revert "feat: embed cortex into jan as a js module (#3305)"

This reverts commit b348110fb73bd5f13c69f1b915168687dea776d0.

Revert "fix: migration item in setting detail omit buttons (#3298)"

This reverts commit 709204b2bc9d9ed08e2245cbb084482f5908ab3a.

Revert "fix: merge gpu arch and os tensorrt models (#3299)"

This reverts commit aa7dbdc9fa701debeee28d9c7eb4af6258685321.

Revert "chore: update cortex new version (#3300)"

This reverts commit 602097909d38b4874db8b9f19a729c65a0ac9619.

Revert "fix: engine logo on model dropdown (#3291)"

This reverts commit 8eb8611c28f6c4cdf1ab142a6e18c82bcc4c2073.

Revert "fix: icon setting can close and open right panel (#3295)"

This reverts commit be31e9315e2df5c483de3f46bd37740d277cfccd.

Revert "fix: error while importing local model is not shown (#3294)"

This reverts commit 26be941e8426462e1e3a28e5b9bf1f834f462f82.

Revert "fix: add lower case quantization support (#3293)"

This reverts commit 3135ccc27e894a4056f882cd25f0bf7e10e56f49.

Revert "fix: onnx can't be selected in download model modal (#3283)"

This reverts commit 2521e1db518e9e01493e89dcc98c181ccd2b48a2.

Revert "feat: add chunk count (#3290)"

This reverts commit bad481bf05aa38edcf553e1273f5d692a65c9225.

Revert "fix: RAM always show 0% (#3287)"

This reverts commit 2201e6c5f87538b953503937fe6b135fe1aa2d94.

Revert "fix: remote engine should not allow reinit (#3284)"

This reverts commit 98abff0da3467c090618233db12a25bfa4c1db69.

Revert "chore": update minor UI (#3281)"

This reverts commit 105a9aa1a1830648a32ae285f751b4078c8ac2b2.

Revert "chore: update z-index tooltip (#3280)"

This reverts commit 5a81865508c205ed8c54df209092553a0c40054f.

Revert "feat: add nvidia engine (#3279)"

This reverts commit 8372f30f0ee99606b123351e7bb62636c62c8b23.

Revert "fix: migration wrong directory (#3278)"

This reverts commit 7fb1354287677f577070ccb065ed3a5f9e5b9882.

Revert "fix: clearer app loading prompt (#3275)"

This reverts commit 44a6401000334b79b225ab6fd6afb79f9da4bd51.

Revert "fix: allow user to reinit engine from settings page (#3277)"

This reverts commit 57cf3c7b3d5bface785763d06813906ba6eab7c9.

Revert "feat: enable copy over instructions (#3266)"

This reverts commit 2074511067201f0addb9d274cc90d1e782f2bc1d.

Revert "chore: toast message on model import fail with reason (#3276)"

This reverts commit 3bebdfe67e1571c7414065a36d16eb5941115ee0.

Revert "fix: should not let second instance terminate cortex (#3274)"

This reverts commit d074a5a445b73ca195a49814a935300f9e895aaa.

Revert "chore: remnove focus button (#3272)"

This reverts commit 07fa79e71a401becdbc0f474c27b860654a8bd62.

Revert "chore: update hub search result (#3273)"

This reverts commit 10b4a9087af709d147b34f6c3ee63d2d3b75c77a.

Revert "chore: temporary hidden import model (#3270)"

This reverts commit db5d8aba454fd4cc1e07253ca4805d4b1b3e7fb2.

Revert "fix: set cortex data folder path when starting jan (#3252)"

This reverts commit 91c77eda78ecd251d480e58b853fe7b261f6de50.

Revert "fix: remote model added manually does not shown in model drop down (#3261)"

This reverts commit 224ca3f7cc25b2577ab123829907964b78b78aa8.

Revert "feat: add more options for cortex popup (#3236)"

This reverts commit 5e06ed8a122aaed9d68fbd04ce42b65bf8987e58.

Revert "feat: manage cloud models from threads screen (#3223)"

This reverts commit 37a3c4f844419e66cfe3f2a9ff79ba688538241f.

Revert "chore: check the legacy incompatible message type (#3248)"

This reverts commit c10caf8d7f1f9cf68551e41de5d54cd4450cf44a.

Revert "chore: minor copy for grammar (#3235)"

This reverts commit f0f23078f31f58e01ba27787d6926f5c1eb2ff0b.

Revert "fix: add back normalize message function (#3234)"

This reverts commit 83579df3a40ff61eac25975da8295fceaec679dc.

Revert "chore: update conditional starter screen after cortex load (#3227)"

This reverts commit 4d3a97f1dca9e6c3ea746586e8607541f2d1c0b3.

Revert "fix: broken status parse due to empty category (#3233)"

This reverts commit 68714eeaf9212a6fdacd5c6a48d8691db9cc99eb.

Revert "feat: make scroll area type auto for make default visible scrollbar (#3220)"

This reverts commit 13428d60e7d3ea6a24c0df8871ea13e2dec0d5fd.

Revert "fix: update new api from cortex to support 0.5.0 (#3221)"

This reverts commit ec9b5bf682a8676e132a08075b6ae03cf9e23132.

Revert "feat: new starter screen (#3217)"

This reverts commit e8ee694abd33b34112d2c7d09f8c03370c2d22cc.

Revert "bump-cortex-0.5.0-1 (#3218)"

This reverts commit 5369da78f5b83b1c8761cb48820ccf3111728a90.

Revert "Deprecate Docker and K8s (#3219)"

This reverts commit 7611a05c44982d07465bec57658d5bf965f30ad5.

Revert "chore: set container max width for chat message and new hub screen (#3213)"

This reverts commit 007daa71616268b0e741e7a890b319401e49a81e.

Revert "feat: integrating cortex (#3001)"

This reverts commit 101268f6f36df96b62982a9eeb8581ebe103a909.
This commit is contained in:
Louis 2024-08-15 10:36:52 +07:00
parent ad6fbea22d
commit a699f8f32f
No known key found for this signature in database
GPG Key ID: 44FA9F4D33C37DE2
595 changed files with 30343 additions and 23196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
.github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@ -1,37 +0,0 @@
---
name: Report
about: Create a report to help us improve Jan
title: 'bug: '
labels: 'type: bug'
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**Steps to reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your issue.
**Environment details**
- Operating System: [Specify your OS. e.g., MacOS Sonoma 14.2.1, Windows 11, Ubuntu 22, etc]
- Jan Version: [e.g., 0.4.xxx nightly or manual]
- Processor: [e.g., Apple M1, Intel Core i7, AMD Ryzen 5, etc]
- RAM: [e.g., 8GB, 16GB]
- Any additional relevant hardware specifics: [e.g., Graphics card, SSD/HDD]
**Logs**
If the cause of the error is not clear, kindly provide your usage logs: https://jan.ai/docs/troubleshooting#how-to-get-error-logs
**Additional context**
Add any other context or information that could be helpful in diagnosing the problem.

33
.github/workflows/jan-electron-linter-and-test.yml vendored
View File

@ -67,9 +67,9 @@ jobs:
run: |
echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}})" >> $GITHUB_ENV
# - name: 'Config report portal'
# run: |
# make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App macos" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
- name: 'Config report portal'
run: |
make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App macos" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
- name: Linter and test
run: |
@ -147,10 +147,10 @@ jobs:
run: |
echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}}" >> $GITHUB_ENV
# - name: 'Config report portal'
# shell: bash
# run: |
# make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows ${{ matrix.antivirus-tools }}" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
- name: 'Config report portal'
shell: bash
run: |
make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows ${{ matrix.antivirus-tools }}" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
- name: Linter and test
shell: powershell
@ -195,14 +195,11 @@ jobs:
run: |
echo "REPORT_PORTAL_DESCRIPTION=${{github.event.after}}" >> $GITHUB_ENV
# - name: 'Config report portal'
# shell: bash
# run: |
# make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
- name: 'Config report portal'
shell: bash
run: |
make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Windows" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
- name: Setup node-gyp - distutils
run: pip3 install --upgrade setuptools
- name: Linter and test
shell: powershell
run: |
@ -278,10 +275,10 @@ jobs:
run: |
echo "REPORT_PORTAL_DESCRIPTION=${{github.sha}}" >> $GITHUB_ENV
# - name: 'Config report portal'
# shell: bash
# run: |
# make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Linux" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
- name: 'Config report portal'
shell: bash
run: |
make update-playwright-config REPORT_PORTAL_URL=${{ secrets.REPORT_PORTAL_URL }} REPORT_PORTAL_API_KEY=${{ secrets.REPORT_PORTAL_API_KEY }} REPORT_PORTAL_PROJECT_NAME=${{ secrets.REPORT_PORTAL_PROJECT_NAME }} REPORT_PORTAL_LAUNCH_NAME="Jan App Linux" REPORT_PORTAL_DESCRIPTION="${{env.REPORT_PORTAL_DESCRIPTION}}"
- name: Linter and test
run: |

40
.github/workflows/jan-server-build-nightly.yml vendored Normal file
View File

@ -0,0 +1,40 @@
name: Docker Builder - Nightly / Manual
on:
push:
branches:
- main
- feature/helmchart-and-ci-jan-server
paths-ignore:
- 'README.md'
- 'docs/**'
schedule:
- cron: '0 21 * * 1,2,3' # At 8 PM UTC on Monday, Tuesday, and Wednesday which is 4 AM UTC+7 Tuesday, Wednesday, and Thursday
workflow_dispatch:
jobs:
# Job create Update app version based on latest release tag with build number and save to output
get-update-version:
uses: ./.github/workflows/template-get-update-version.yml
build-cpu:
uses: ./.github/workflows/template-build-jan-server.yml
permissions:
packages: write
secrets: inherit
needs: [get-update-version]
with:
dockerfile_path: ./Dockerfile
docker_image_tag: "ghcr.io/janhq/jan-server:dev-cpu-latest,ghcr.io/janhq/jan-server:dev-cpu-${{ needs.get-update-version.outputs.new_version }}"
build-gpu:
uses: ./.github/workflows/template-build-jan-server.yml
permissions:
packages: write
secrets: inherit
needs: [get-update-version]
with:
dockerfile_path: ./Dockerfile.gpu
docker_image_tag: "ghcr.io/janhq/jan-server:dev-cuda-12.2-latest,ghcr.io/janhq/jan-server:dev-cuda-12.2-${{ needs.get-update-version.outputs.new_version }}"

30
.github/workflows/jan-server-build.yml vendored Normal file
View File

@ -0,0 +1,30 @@
name: Docker Builder - Tag
on:
push:
tags: ["v[0-9]+.[0-9]+.[0-9]+"]
jobs:
# Job create Update app version based on latest release tag with build number and save to output
get-update-version:
uses: ./.github/workflows/template-get-update-version.yml
build-cpu:
permissions:
packages: write
uses: ./.github/workflows/template-build-jan-server.yml
secrets: inherit
needs: [get-update-version]
with:
dockerfile_path: ./Dockerfile
docker_image_tag: "ghcr.io/janhq/jan-server:cpu-latest,ghcr.io/janhq/jan-server:cpu-${{ needs.get-update-version.outputs.new_version }}"
build-gpu:
permissions:
packages: write
uses: ./.github/workflows/template-build-jan-server.yml
secrets: inherit
needs: [get-update-version]
with:
dockerfile_path: ./Dockerfile.gpu
docker_image_tag: "ghcr.io/janhq/jan-server:cuda-12.2-latest,ghcr.io/janhq/jan-server:cuda-12.2-${{ needs.get-update-version.outputs.new_version }}"

1
.github/workflows/template-build-linux-x64.yml vendored
View File

@ -91,7 +91,6 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
AWS_EC2_METADATA_DISABLED: "true"
AWS_MAX_ATTEMPTS: "5"
- name: Build and publish app to github
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && inputs.public_provider == 'github'

6
.github/workflows/template-build-macos-arm64.yml vendored
View File

@ -56,11 +56,6 @@ jobs:
with:
node-version: 20
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Install jq
uses: dcarbone/install-jq-action@v2.0.1
@ -135,7 +130,6 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_EC2_METADATA_DISABLED: "true"
AWS_MAX_ATTEMPTS: "5"
- name: Build and publish app to github
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && inputs.public_provider == 'github'

6
.github/workflows/template-build-macos-x64.yml vendored
View File

@ -56,11 +56,6 @@ jobs:
with:
node-version: 20
- name: Install python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Install jq
uses: dcarbone/install-jq-action@v2.0.1
@ -135,7 +130,6 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_EC2_METADATA_DISABLED: "true"
AWS_MAX_ATTEMPTS: "5"
- name: Build and publish app to github
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && inputs.public_provider == 'github'

1
.github/workflows/template-build-windows-x64.yml vendored
View File

@ -120,7 +120,6 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_EC2_METADATA_DISABLED: "true"
AWS_MAX_ATTEMPTS: "5"
- name: Build app and publish app to github
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && inputs.public_provider == 'github'

3
.gitignore vendored
View File

@ -12,9 +12,6 @@ yarn.lock
dist
build
.DS_Store
electron/resources/win/*
electron/resources/linux/*
electron/resources/mac/*
electron/renderer
electron/models
electron/docs

60
Dockerfile Normal file
View File

@ -0,0 +1,60 @@
FROM node:20-bookworm AS base
# 1. Install dependencies only when needed
FROM base AS builder
# Install g++ 11
RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel && rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Install dependencies based on the preferred package manager
COPY . ./
RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \
jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json
RUN make install-and-build
# # 2. Rebuild the source code only when needed
FROM base AS runner
# Install g++ 11
RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel && rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy the package.json and yarn.lock of root yarn space to leverage Docker cache
COPY --from=builder /app/package.json ./package.json
COPY --from=builder /app/node_modules ./node_modules/
COPY --from=builder /app/yarn.lock ./yarn.lock
# Copy the package.json, yarn.lock, and build output of server yarn space to leverage Docker cache
COPY --from=builder /app/core ./core/
COPY --from=builder /app/server ./server/
RUN cd core && yarn install && yarn run build
RUN yarn workspace @janhq/server install && yarn workspace @janhq/server build
COPY --from=builder /app/docs/openapi ./docs/openapi/
# Copy pre-install dependencies
COPY --from=builder /app/pre-install ./pre-install/
# Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache
COPY --from=builder /app/joi ./joi/
COPY --from=builder /app/web ./web/
RUN yarn workspace @janhq/joi install && yarn workspace @janhq/joi build
RUN yarn workspace @janhq/web install
RUN npm install -g serve@latest
EXPOSE 1337 3000 3928
ENV JAN_API_HOST 0.0.0.0
ENV JAN_API_PORT 1337
ENV API_BASE_URL http://localhost:1337
CMD ["sh", "-c", "export NODE_ENV=production && yarn workspace @janhq/web build && cd web && npx serve out & cd server && node build/main.js"]
# docker build -t jan .
# docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 jan

87
Dockerfile.gpu Normal file
View File

@ -0,0 +1,87 @@
# Please change the base image to the appropriate CUDA version base on NVIDIA Driver Compatibility
# Run nvidia-smi to check the CUDA version and the corresponding driver version
# Then update the base image to the appropriate CUDA version refer https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags
FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base
# 1. Install dependencies only when needed
FROM base AS builder
# Install g++ 11
RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/*
# Update alternatives for GCC and related tools
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \
--slave /usr/bin/g++ g++ /usr/bin/g++-11 \
--slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
--slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
--slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \
update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110
RUN npm install -g yarn
WORKDIR /app
# Install dependencies based on the preferred package manager
COPY . ./
RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \
jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json
RUN make install-and-build
# # 2. Rebuild the source code only when needed
FROM base AS runner
# Install g++ 11
RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt-get install nodejs -y && rm -rf /var/lib/apt/lists/*
# Update alternatives for GCC and related tools
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \
--slave /usr/bin/g++ g++ /usr/bin/g++-11 \
--slave /usr/bin/gcov gcov /usr/bin/gcov-11 \
--slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \
--slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \
update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110
RUN npm install -g yarn
WORKDIR /app
# Copy the package.json and yarn.lock of root yarn space to leverage Docker cache
COPY --from=builder /app/package.json ./package.json
COPY --from=builder /app/node_modules ./node_modules/
COPY --from=builder /app/yarn.lock ./yarn.lock
# Copy the package.json, yarn.lock, and build output of server yarn space to leverage Docker cache
COPY --from=builder /app/core ./core/
COPY --from=builder /app/server ./server/
RUN cd core && yarn install && yarn run build
RUN yarn workspace @janhq/server install && yarn workspace @janhq/server build
COPY --from=builder /app/docs/openapi ./docs/openapi/
# Copy pre-install dependencies
COPY --from=builder /app/pre-install ./pre-install/
# Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache
COPY --from=builder /app/joi ./joi/
COPY --from=builder /app/web ./web/
RUN yarn workspace @janhq/joi install && yarn workspace @janhq/joi build
RUN yarn workspace @janhq/web install
RUN npm install -g serve@latest
EXPOSE 1337 3000 3928
ENV LD_LIBRARY_PATH=/usr/local/cuda/targets/x86_64-linux/lib:/usr/local/cuda-12.0/compat${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
ENV JAN_API_HOST 0.0.0.0
ENV JAN_API_PORT 1337
ENV API_BASE_URL http://localhost:1337
CMD ["sh", "-c", "export NODE_ENV=production && yarn workspace @janhq/web build && cd web && npx serve out & cd server && node build/main.js"]
# pre-requisites: nvidia-docker
# docker build -t jan-gpu . -f Dockerfile.gpu
# docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 --gpus all jan-gpu

12
Makefile
View File

@ -18,14 +18,16 @@ else
cd joi && yarn install && yarn build
endif
# Installs yarn dependencies and builds core
# Installs yarn dependencies and builds core and extensions
install-and-build: build-joi
ifeq ($(OS),Windows_NT)
yarn config set network-timeout 300000
endif
yarn global add turbo@1.13.2
yarn build:core
yarn build:server
yarn install
yarn build:extensions
check-file-counts: install-and-build
ifeq ($(OS),Windows_NT)
@ -34,11 +36,11 @@ else
@tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d -exec test -e '{}/package.json' \; -print | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi
endif
dev: install-and-build
dev: check-file-counts
yarn dev
# Linting
lint: install-and-build
lint: check-file-counts
yarn lint
update-playwright-config:
@ -106,11 +108,11 @@ test: lint
yarn test
# Builds and publishes the app
build-and-publish: install-and-build
build-and-publish: check-file-counts
yarn build:publish
# Build
build: install-and-build
build: check-file-counts
yarn build
clean:

6
README.md
View File

@ -210,6 +210,12 @@ Contributions are welcome! Please read the [CONTRIBUTING.md](CONTRIBUTING.md) fi
This will start the development server and open the desktop app.
3. (Optional) **Run the API server without frontend**
```bash
yarn dev:server
```
### For production build
```bash

6
charts/server/Chart.lock Normal file
View File

@ -0,0 +1,6 @@
dependencies:
- name: common
repository: oci://ghcr.io/janhq/charts
version: 0.1.2
digest: sha256:35e98bde174130787755b0f8ea2359b7b6790d965a7157c2f7cabf1bc8c04471
generated: "2024-02-20T16:20:37.6530108+07:00"

10
charts/server/Chart.yaml Normal file
View File

@ -0,0 +1,10 @@
apiVersion: v2
name: jan-server
description: A Helm chart for Kubernetes
type: application
version: 0.1.0
appVersion: '1.0.0'
dependencies:
- name: common
version: 0.1.2 # common-chart-version
repository: oci://ghcr.io/janhq/charts

BIN
charts/server/charts/common-0.1.2.tgz Normal file
View File

Binary file not shown.

4
charts/server/config.json Normal file
View File

@ -0,0 +1,4 @@
{
"image-list": "server=ghcr.io/janhq/jan-server",
"platforms": "linux/amd64"
}

256
charts/server/values.yaml Normal file
View File

@ -0,0 +1,256 @@
common:
imageTag: v0.4.6-cpu
# DO NOT CHANGE THE LINE ABOVE. MAKE ALL CHANGES BELOW
# Global pvc for all workload
pvc:
enabled: false
name: 'janroot'
accessModes: 'ReadWriteOnce'
storageClassName: ''
capacity: '50Gi'
# Global image pull secret
imagePullSecrets: []
externalSecret:
create: false
name: ''
annotations: {}
nameOverride: 'jan-server'
fullnameOverride: 'jan-server'
serviceAccount:
create: true
annotations: {}
name: 'jan-server-service-account'
podDisruptionBudget:
create: false
minAvailable: 1
workloads:
- name: server
image:
repository: ghcr.io/janhq/jan-server
pullPolicy: Always
command: ['/bin/sh', '-c']
args: ['cd server && node build/main.js']
replicaCount: 1
ports:
containerPort: 1337
strategy:
canary:
steps:
- setWeight: 50
- pause: { duration: 1m }
ingress:
enabled: true
className: 'nginx'
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: '100m'
nginx.ingress.kubernetes.io/proxy-read-timeout: '1800'
nginx.ingress.kubernetes.io/proxy-send-timeout: '1800'
# cert-manager.io/cluster-issuer: 'jan-ai-dns01-cluster-issuer'
# nginx.ingress.kubernetes.io/force-ssl-redirect: 'true'
nginx.ingress.kubernetes.io/backend-protocol: HTTP
hosts:
- host: server.local
paths:
- path: /
pathType: Prefix
tls:
[]
# - hosts:
# - server-dev.jan.ai
# secretName: jan-server-prod-tls-v2
instrumentation:
enabled: false
podAnnotations: {}
podSecurityContext: {}
securityContext: {}
service:
externalLabel: {}
type: ClusterIP
port: 1337
targetPort: 1337
# If you want to use GPU, please uncomment the following lines and change imageTag to the one with GPU support
resources:
# limits:
# nvidia.com/gpu: 1
requests:
cpu: 2000m
memory: 8192M
# If you want to use pv, please uncomment the following lines and enable pvc.enabled
volumes:
[]
# - name: janroot
# persistentVolumeClaim:
# claimName: janroot
volumeMounts:
[]
# - name: janroot
# mountPath: /app/server/build/jan
# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME, AWS_ENDPOINT, AWS_REGION should mount as a secret env instead of plain text here
# Change API_BASE_URL to your server's public domain
env:
- name: API_BASE_URL
value: 'http://server.local'
lifecycle: {}
autoscaling:
enabled: false
minReplicas: 2
maxReplicas: 3
targetCPUUtilizationPercentage: 95
targetMemoryUtilizationPercentage: 95
kedaScaling:
enabled: false # ignore if autoscaling.enable = true
cooldownPeriod: 30
pollingInterval: 2
minReplicas: 1
maxReplicas: 5
metricName: celery_queue_length
query: celery_queue_length{queue_name="myqueue"} # change queue_name here
serverAddress: http://prometheus-prod-kube-prome-prometheus.monitoring.svc:9090
threshold: '3'
nodeSelector: {}
tolerations: []
podSecurityGroup:
enabled: false
securitygroupid: []
# Reloader Option
reloader: 'false'
vpa:
enabled: false
- name: web
image:
repository: ghcr.io/janhq/jan-server
pullPolicy: Always
command: ['/bin/sh', '-c']
args:
[
'export NODE_ENV=production && yarn workspace @janhq/web build && cd web && npx serve out',
]
replicaCount: 1
ports:
containerPort: 3000
strategy:
canary:
steps:
- setWeight: 50
- pause: { duration: 1m }
ingress:
enabled: true
className: 'nginx'
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: '100m'
nginx.ingress.kubernetes.io/proxy-read-timeout: '1800'
nginx.ingress.kubernetes.io/proxy-send-timeout: '1800'
# cert-manager.io/cluster-issuer: 'jan-ai-dns01-cluster-issuer'
# nginx.ingress.kubernetes.io/force-ssl-redirect: 'true'
nginx.ingress.kubernetes.io/backend-protocol: HTTP
hosts:
- host: web.local
paths:
- path: /
pathType: Prefix
tls:
[]
# - hosts:
# - server-dev.jan.ai
# secretName: jan-server-prod-tls-v2
instrumentation:
enabled: false
podAnnotations: {}
podSecurityContext: {}
securityContext: {}
service:
externalLabel: {}
type: ClusterIP
port: 3000
targetPort: 3000
resources:
limits:
cpu: 1000m
memory: 2048M
requests:
cpu: 50m
memory: 500M
volumes:
[]
# - name: janroot
# persistentVolumeClaim:
# claimName: janroot
volumeMounts:
[]
# - name: janroot
# mountPath: /app/server/build/jan
# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME, AWS_ENDPOINT, AWS_REGION should mount as a secret env instead of plain text here
# Change API_BASE_URL to your server's public domain
env:
- name: API_BASE_URL
value: 'http://server.local'
lifecycle: {}
autoscaling:
enabled: true
minReplicas: 1
maxReplicas: 3
targetCPUUtilizationPercentage: 95
targetMemoryUtilizationPercentage: 95
kedaScaling:
enabled: false # ignore if autoscaling.enable = true
cooldownPeriod: 30
pollingInterval: 2
minReplicas: 1
maxReplicas: 5
metricName: celery_queue_length
query: celery_queue_length{queue_name="myqueue"} # change queue_name here
serverAddress: http://prometheus-prod-kube-prome-prometheus.monitoring.svc:9090
threshold: '3'
nodeSelector: {}
tolerations: []
podSecurityGroup:
enabled: false
securitygroupid: []
# Reloader Option
reloader: 'false'
vpa:
enabled: false

4
core/README.md
View File

@ -40,7 +40,7 @@ import * as node from "@janhq/core/node";
private static inference(incomingMessage: MessageRequestData) {
// Prepare customized message content
const content: MessageContent = {
const content: ThreadContent = {
type: ContentType.Text,
text: {
value: "I'm Jan Assistant!",
@ -49,7 +49,7 @@ import * as node from "@janhq/core/node";
};
// Modify message and send out
const outGoingMessage: Message = {
const outGoingMessage: ThreadMessage = {
...incomingMessage,
content
};

12
core/package.json
View File

@ -8,7 +8,7 @@
],
"homepage": "https://jan.ai",
"license": "AGPL-3.0",
"main": "dist/lib/index.js",
"main": "dist/core.es5.js",
"module": "dist/core.cjs.js",
"typings": "dist/types/index.d.ts",
"files": [
@ -17,18 +17,18 @@
],
"author": "Jan <service@jan.ai>",
"exports": {
".": "./dist/lib/index.js",
".": "./dist/core.es5.js",
"./node": "./dist/node/index.cjs.js"
},
"typesVersions": {
"*": {
".": [
"./dist/lib/index.js",
"./dist/core.es5.js.map",
"./dist/types/index.d.ts"
],
"node": [
"./dist/node/index.cjs.js.map",
"./dist/types/index.d.ts"
"./dist/types/node/index.d.ts"
]
}
},
@ -40,7 +40,6 @@
"start": "rollup -c rollup.config.ts -w"
},
"devDependencies": {
"openai": "4.51.0",
"@rollup/plugin-replace": "^5.0.5",
"@types/jest": "^29.5.12",
"@types/node": "^20.11.4",
@ -59,6 +58,7 @@
"typescript": "^5.3.3"
},
"dependencies": {
"rxjs": "^7.8.1"
"rxjs": "^7.8.1",
"ulidx": "^2.3.0"
}
}

3
core/rollup.config.ts
View File

@ -43,7 +43,7 @@ export default [
],
},
{
input: `src/index.ts`,
input: `src/node/index.ts`,
output: [{ file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }],
// Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
external: [
@ -52,6 +52,7 @@ export default [
'pacote',
'@types/pacote',
'@npmcli/arborist',
'ulidx',
'node-fetch',
'fs',
'request',

165
core/src/browser/core.ts Normal file
View File

@ -0,0 +1,165 @@
import { DownloadRequest, FileStat, NetworkConfig, SystemInformation } from '../types'
/**
* Execute a extension module function in main process
*
* @param extension extension name to import
* @param method function name to execute
* @param args arguments to pass to the function
* @returns Promise<any>
*
*/
const executeOnMain: (extension: string, method: string, ...args: any[]) => Promise<any> = (
extension,
method,
...args
) => globalThis.core?.api?.invokeExtensionFunc(extension, method, ...args)
/**
* Downloads a file from a URL and saves it to the local file system.
*
* @param {DownloadRequest} downloadRequest - The request to download the file.
* @param {NetworkConfig} network - Optional object to specify proxy/whether to ignore SSL certificates.
*
* @returns {Promise<any>} A promise that resolves when the file is downloaded.
*/
const downloadFile: (downloadRequest: DownloadRequest, network?: NetworkConfig) => Promise<any> = (
downloadRequest,
network
) => globalThis.core?.api?.downloadFile(downloadRequest, network)
/**
* Get unit in bytes for a remote file.
*
* @param url - The url of the file.
* @returns {Promise<number>} - A promise that resolves with the file size.
*/
const getFileSize: (url: string) => Promise<number> = (url: string) =>
globalThis.core.api?.getFileSize(url)
/**
* Aborts the download of a specific file.
* @param {string} fileName - The name of the file whose download is to be aborted.
* @returns {Promise<any>} A promise that resolves when the download has been aborted.
*/
const abortDownload: (fileName: string) => Promise<any> = (fileName) =>
globalThis.core.api?.abortDownload(fileName)
/**
* Gets Jan's data folder path.
*
* @returns {Promise<string>} A Promise that resolves with Jan's data folder path.
*/
const getJanDataFolderPath = (): Promise<string> => globalThis.core.api?.getJanDataFolderPath()
/**
* Opens the file explorer at a specific path.
* @param {string} path - The path to open in the file explorer.
* @returns {Promise<any>} A promise that resolves when the file explorer is opened.
*/
const openFileExplorer: (path: string) => Promise<any> = (path) =>
globalThis.core.api?.openFileExplorer(path)
/**
* Joins multiple paths together.
* @param paths - The paths to join.
* @returns {Promise<string>} A promise that resolves with the joined path.
*/
const joinPath: (paths: string[]) => Promise<string> = (paths) =>
globalThis.core.api?.joinPath(paths)
/**
* Retrieve the basename from an url.
* @param path - The path to retrieve.
* @returns {Promise<string>} A promise that resolves with the basename.
*/
const baseName: (paths: string) => Promise<string> = (path) => globalThis.core.api?.baseName(path)
/**
* Opens an external URL in the default web browser.
*
* @param {string} url - The URL to open.
* @returns {Promise<any>} - A promise that resolves when the URL has been successfully opened.
*/
const openExternalUrl: (url: string) => Promise<any> = (url) =>
globalThis.core.api?.openExternalUrl(url)
/**
* Gets the resource path of the application.
*
* @returns {Promise<string>} - A promise that resolves with the resource path.
*/
const getResourcePath: () => Promise<string> = () => globalThis.core.api?.getResourcePath()
/**
* Gets the user's home path.
* @returns return user's home path
*/
const getUserHomePath = (): Promise<string> => globalThis.core.api?.getUserHomePath()
/**
* Log to file from browser processes.
*
* @param message - Message to log.
*/
const log: (message: string, fileName?: string) => void = (message, fileName) =>
globalThis.core.api?.log(message, fileName)
/**
* Check whether the path is a subdirectory of another path.
*
* @param from - The path to check.
* @param to - The path to check against.
*
* @returns {Promise<boolean>} - A promise that resolves with a boolean indicating whether the path is a subdirectory.
*/
const isSubdirectory: (from: string, to: string) => Promise<boolean> = (from: string, to: string) =>
globalThis.core.api?.isSubdirectory(from, to)
/**
* Get system information
* @returns {Promise<any>} - A promise that resolves with the system information.
*/
const systemInformation: () => Promise<SystemInformation> = () =>
globalThis.core.api?.systemInformation()
/**
* Show toast message from browser processes.
* @param title
* @param message
* @returns
*/
const showToast: (title: string, message: string) => void = (title, message) =>
globalThis.core.api?.showToast(title, message)
/**
* Register extension point function type definition
*/
export type RegisterExtensionPoint = (
extensionName: string,
extensionId: string,
method: Function,
priority?: number
) => void
/**
* Functions exports
*/
export {
executeOnMain,
downloadFile,
abortDownload,
getJanDataFolderPath,
openFileExplorer,
getResourcePath,
joinPath,
openExternalUrl,
baseName,
log,
isSubdirectory,
getUserHomePath,
systemInformation,
showToast,
getFileSize,
FileStat,
}

35
core/src/browser/events.ts Normal file
View File

@ -0,0 +1,35 @@
/**
* Adds an observer for an event.
*
* @param eventName The name of the event to observe.
* @param handler The handler function to call when the event is observed.
*/
const on: (eventName: string, handler: Function) => void = (eventName, handler) => {
globalThis.core?.events?.on(eventName, handler)
}
/**
* Removes an observer for an event.
*
* @param eventName The name of the event to stop observing.
* @param handler The handler function to call when the event is observed.
*/
const off: (eventName: string, handler: Function) => void = (eventName, handler) => {
globalThis.core?.events?.off(eventName, handler)
}
/**
* Emits an event.
*
* @param eventName The name of the event to emit.
* @param object The object to pass to the event callback.
*/
const emit: (eventName: string, object: any) => void = (eventName, object) => {
globalThis.core?.events?.emit(eventName, object)
}
export const events = {
on,
off,
emit,
}

211
core/src/browser/extension.ts Normal file
View File

@ -0,0 +1,211 @@
import { SettingComponentProps } from '../types'
import { getJanDataFolderPath, joinPath } from './core'
import { fs } from './fs'
export enum ExtensionTypeEnum {
Assistant = 'assistant',
Conversational = 'conversational',
Inference = 'inference',
Model = 'model',
SystemMonitoring = 'systemMonitoring',
HuggingFace = 'huggingFace',
}
export interface ExtensionType {
type(): ExtensionTypeEnum | undefined
}
export interface Compatibility {
platform: string[]
version: string
}
const ALL_INSTALLATION_STATE = [
'NotRequired', // not required.
'Installed', // require and installed. Good to go.
'NotInstalled', // require to be installed.
'Corrupted', // require but corrupted. Need to redownload.
'NotCompatible', // require but not compatible.
] as const
export type InstallationStateTuple = typeof ALL_INSTALLATION_STATE
export type InstallationState = InstallationStateTuple[number]
/**
* Represents a base extension.
* This class should be extended by any class that represents an extension.
*/
export abstract class BaseExtension implements ExtensionType {
protected settingFolderName = 'settings'
protected settingFileName = 'settings.json'
/** @type {string} Name of the extension. */
name: string
/** @type {string} Product Name of the extension. */
productName?: string
/** @type {string} The URL of the extension to load. */
url: string
/** @type {boolean} Whether the extension is activated or not. */
active
/** @type {string} Extension's description. */
description
/** @type {string} Extension's version. */
version
constructor(
url: string,
name: string,
productName?: string,
active?: boolean,
description?: string,
version?: string
) {
this.name = name
this.productName = productName
this.url = url
this.active = active
this.description = description
this.version = version
}
/**
* Returns the type of the extension.
* @returns {ExtensionType} The type of the extension
* Undefined means its not extending any known extension by the application.
*/
type(): ExtensionTypeEnum | undefined {
return undefined
}
/**
* Called when the extension is loaded.
* Any initialization logic for the extension should be put here.
*/
abstract onLoad(): void
/**
* Called when the extension is unloaded.
* Any cleanup logic for the extension should be put here.
*/
abstract onUnload(): void
/**
* The compatibility of the extension.
* This is used to check if the extension is compatible with the current environment.
* @property {Array} platform
*/
compatibility(): Compatibility | undefined {
return undefined
}
async registerSettings(settings: SettingComponentProps[]): Promise<void> {
if (!this.name) {
console.error('Extension name is not defined')
return
}
const extensionSettingFolderPath = await joinPath([
await getJanDataFolderPath(),
'settings',
this.name,
])
settings.forEach((setting) => {
setting.extensionName = this.name
})
try {
await fs.mkdir(extensionSettingFolderPath)
const settingFilePath = await joinPath([extensionSettingFolderPath, this.settingFileName])
if (await fs.existsSync(settingFilePath)) return
await fs.writeFileSync(settingFilePath, JSON.stringify(settings, null, 2))
} catch (err) {
console.error(err)
}
}
async getSetting<T>(key: string, defaultValue: T) {
const keySetting = (await this.getSettings()).find((setting) => setting.key === key)
const value = keySetting?.controllerProps.value
return (value as T) ?? defaultValue
}
onSettingUpdate<T>(key: string, value: T) {
return
}
/**
* Determine if the prerequisites for the extension are installed.
*
* @returns {boolean} true if the prerequisites are installed, false otherwise.
*/
async installationState(): Promise<InstallationState> {
return 'NotRequired'
}
/**
* Install the prerequisites for the extension.
*
* @returns {Promise<void>}
*/
async install(): Promise<void> {
return
}
async getSettings(): Promise<SettingComponentProps[]> {
if (!this.name) return []
const settingPath = await joinPath([
await getJanDataFolderPath(),
this.settingFolderName,
this.name,
this.settingFileName,
])
try {
const content = await fs.readFileSync(settingPath, 'utf-8')
const settings: SettingComponentProps[] = JSON.parse(content)
return settings
} catch (err) {
console.warn(err)
return []
}
}
async updateSettings(componentProps: Partial<SettingComponentProps>[]): Promise<void> {
if (!this.name) return
const settings = await this.getSettings()
const updatedSettings = settings.map((setting) => {
const updatedSetting = componentProps.find(
(componentProp) => componentProp.key === setting.key
)
if (updatedSetting && updatedSetting.controllerProps) {
setting.controllerProps.value = updatedSetting.controllerProps.value
}
return setting
})
const settingPath = await joinPath([
await getJanDataFolderPath(),
this.settingFolderName,
this.name,
this.settingFileName,
])
await fs.writeFileSync(settingPath, JSON.stringify(updatedSettings, null, 2))
updatedSettings.forEach((setting) => {
this.onSettingUpdate<typeof setting.controllerProps.value>(
setting.key,
setting.controllerProps.value
)
})
}
}

19
core/src/browser/extensions/assistant.ts Normal file
View File

@ -0,0 +1,19 @@
import { Assistant, AssistantInterface } from '../../types'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
/**
* Assistant extension for managing assistants.
* @extends BaseExtension
*/
export abstract class AssistantExtension extends BaseExtension implements AssistantInterface {
/**
* Assistant extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Assistant
}
abstract createAssistant(assistant: Assistant): Promise<void>
abstract deleteAssistant(assistant: Assistant): Promise<void>
abstract getAssistants(): Promise<Assistant[]>
}

26
core/src/browser/extensions/conversational.ts Normal file
View File

@ -0,0 +1,26 @@
import { Thread, ThreadInterface, ThreadMessage, MessageInterface } from '../../types'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
/**
* Conversational extension. Persists and retrieves conversations.
* @abstract
* @extends BaseExtension
*/
export abstract class ConversationalExtension
extends BaseExtension
implements ThreadInterface, MessageInterface
{
/**
* Conversation extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Conversational
}
abstract getThreads(): Promise<Thread[]>
abstract saveThread(thread: Thread): Promise<void>
abstract deleteThread(threadId: string): Promise<void>
abstract addNewMessage(message: ThreadMessage): Promise<void>
abstract writeMessages(threadId: string, messages: ThreadMessage[]): Promise<void>
abstract getAllMessages(threadId: string): Promise<ThreadMessage[]>
}

104
core/src/browser/extensions/engines/AIEngine.ts Normal file
View File

@ -0,0 +1,104 @@
import { getJanDataFolderPath, joinPath } from '../../core'
import { events } from '../../events'
import { BaseExtension } from '../../extension'
import { fs } from '../../fs'
import { MessageRequest, Model, ModelEvent } from '../../../types'
import { EngineManager } from './EngineManager'
/**
* Base AIEngine
* Applicable to all AI Engines
*/
export abstract class AIEngine extends BaseExtension {
private static modelsFolder = 'models'
// The inference engine
abstract provider: string
/**
* On extension load, subscribe to events.
*/
override onLoad() {
this.registerEngine()
events.on(ModelEvent.OnModelInit, (model: Model) => this.loadModel(model))
events.on(ModelEvent.OnModelStop, (model: Model) => this.unloadModel(model))
}
/**
* Registers AI Engines
*/
registerEngine() {
EngineManager.instance().register(this)
}
async registerModels(models: Model[]): Promise<void> {
const modelFolderPath = await joinPath([await getJanDataFolderPath(), AIEngine.modelsFolder])
let shouldNotifyModelUpdate = false
for (const model of models) {
const modelPath = await joinPath([modelFolderPath, model.id])
const isExist = await fs.existsSync(modelPath)
if (isExist) {
await this.migrateModelIfNeeded(model, modelPath)
continue
}
await fs.mkdir(modelPath)
await fs.writeFileSync(
await joinPath([modelPath, 'model.json']),
JSON.stringify(model, null, 2)
)
shouldNotifyModelUpdate = true
}
if (shouldNotifyModelUpdate) {
events.emit(ModelEvent.OnModelsUpdate, {})
}
}
async migrateModelIfNeeded(model: Model, modelPath: string): Promise<void> {
try {
const modelJson = await fs.readFileSync(await joinPath([modelPath, 'model.json']), 'utf-8')
const currentModel: Model = JSON.parse(modelJson)
if (currentModel.version !== model.version) {
await fs.writeFileSync(
await joinPath([modelPath, 'model.json']),
JSON.stringify(model, null, 2)
)
events.emit(ModelEvent.OnModelsUpdate, {})
}
} catch (error) {
console.warn('Error while try to migrating model', error)
}
}
/**
* Loads the model.
*/
async loadModel(model: Model): Promise<any> {
if (model.engine.toString() !== this.provider) return Promise.resolve()
events.emit(ModelEvent.OnModelReady, model)
return Promise.resolve()
}
/**
* Stops the model.
*/
async unloadModel(model?: Model): Promise<any> {
if (model?.engine && model.engine.toString() !== this.provider) return Promise.resolve()
events.emit(ModelEvent.OnModelStopped, model ?? {})
return Promise.resolve()
}
/*
* Inference request
*/
inference(data: MessageRequest) {}
/**
* Stop inference
*/
stopInference() {}
}

32
core/src/browser/extensions/engines/EngineManager.ts Normal file
View File

@ -0,0 +1,32 @@
import { AIEngine } from './AIEngine'
/**
* Manages the registration and retrieval of inference engines.
*/
export class EngineManager {
public engines = new Map<string, AIEngine>()
/**
* Registers an engine.
* @param engine - The engine to register.
*/
register<T extends AIEngine>(engine: T) {
this.engines.set(engine.provider, engine)
}
/**
* Retrieves a engine by provider.
* @param provider - The name of the engine to retrieve.
* @returns The engine, if found.
*/
get<T extends AIEngine>(provider: string): T | undefined {
return this.engines.get(provider) as T | undefined
}
/**
* The instance of the engine manager.
*/
static instance(): EngineManager {
return window.core?.engineManager as EngineManager ?? new EngineManager()
}
}

64
core/src/browser/extensions/engines/LocalOAIEngine.ts Normal file
View File

@ -0,0 +1,64 @@
import { executeOnMain, getJanDataFolderPath, joinPath, systemInformation } from '../../core'
import { events } from '../../events'
import { Model, ModelEvent } from '../../../types'
import { OAIEngine } from './OAIEngine'
/**
* Base OAI Local Inference Provider
* Added the implementation of loading and unloading model (applicable to local inference providers)
*/
export abstract class LocalOAIEngine extends OAIEngine {
// The inference engine
abstract nodeModule: string
loadModelFunctionName: string = 'loadModel'
unloadModelFunctionName: string = 'unloadModel'
/**
* On extension load, subscribe to events.
*/
override onLoad() {
super.onLoad()
// These events are applicable to local inference providers
events.on(ModelEvent.OnModelInit, (model: Model) => this.loadModel(model))
events.on(ModelEvent.OnModelStop, (model: Model) => this.unloadModel(model))
}
/**
* Load the model.
*/
override async loadModel(model: Model): Promise<void> {
if (model.engine.toString() !== this.provider) return
const modelFolderName = 'models'
const modelFolder = await joinPath([await getJanDataFolderPath(), modelFolderName, model.id])
const systemInfo = await systemInformation()
const res = await executeOnMain(
this.nodeModule,
this.loadModelFunctionName,
{
modelFolder,
model,
},
systemInfo
)
if (res?.error) {
events.emit(ModelEvent.OnModelFail, { error: res.error })
return Promise.reject(res.error)
} else {
this.loadedModel = model
events.emit(ModelEvent.OnModelReady, model)
return Promise.resolve()
}
}
/**
* Stops the model.
*/
override async unloadModel(model?: Model) {
if (model?.engine && model.engine?.toString() !== this.provider) return Promise.resolve()
this.loadedModel = undefined
await executeOnMain(this.nodeModule, this.unloadModelFunctionName).then(() => {
events.emit(ModelEvent.OnModelStopped, {})
})
}
}

157
core/src/browser/extensions/engines/OAIEngine.ts Normal file
View File

@ -0,0 +1,157 @@
import { requestInference } from './helpers/sse'
import { ulid } from 'ulidx'
import { AIEngine } from './AIEngine'
import {
ChatCompletionRole,
ContentType,
InferenceEvent,
MessageEvent,
MessageRequest,
MessageRequestType,
MessageStatus,
Model,
ModelInfo,
ThreadContent,
ThreadMessage,
} from '../../../types'
import { events } from '../../events'
/**
* Base OAI Inference Provider
* Applicable to all OAI compatible inference providers
*/
export abstract class OAIEngine extends AIEngine {
// The inference engine
abstract inferenceUrl: string
// Controller to handle stop requests
controller = new AbortController()
isCancelled = false
// The loaded model instance
loadedModel: Model | undefined
// Transform the payload
transformPayload?: Function
// Transform the response
transformResponse?: Function
/**
* On extension load, subscribe to events.
*/
override onLoad() {
super.onLoad()
events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => this.inference(data))
events.on(InferenceEvent.OnInferenceStopped, () => this.stopInference())
}
/**
* On extension unload
*/
override onUnload(): void {}
/*
* Inference request
*/
override async inference(data: MessageRequest) {
if (data.model?.engine?.toString() !== this.provider) return
const timestamp = Date.now()
const message: ThreadMessage = {
id: ulid(),
thread_id: data.threadId,
type: data.type,
assistant_id: data.assistantId,
role: ChatCompletionRole.Assistant,
content: [],
status: MessageStatus.Pending,
created: timestamp,
updated: timestamp,
object: 'thread.message',
}
if (data.type !== MessageRequestType.Summary) {
events.emit(MessageEvent.OnMessageResponse, message)
}
this.isCancelled = false
this.controller = new AbortController()
const model: ModelInfo = {
...(this.loadedModel ? this.loadedModel : {}),
...data.model,
}
const header = await this.headers()
let requestBody = {
messages: data.messages ?? [],
model: model.id,
stream: true,
...model.parameters,
}
if (this.transformPayload) {
requestBody = this.transformPayload(requestBody)
}
requestInference(
this.inferenceUrl,
requestBody,
model,
this.controller,
header,
this.transformResponse
).subscribe({
next: (content: any) => {
const messageContent: ThreadContent = {
type: ContentType.Text,
text: {
value: content.trim(),
annotations: [],
},
}
message.content = [messageContent]
events.emit(MessageEvent.OnMessageUpdate, message)
},
complete: async () => {
message.status = message.content.length ? MessageStatus.Ready : MessageStatus.Error
events.emit(MessageEvent.OnMessageUpdate, message)
},
error: async (err: any) => {
console.debug('inference url: ', this.inferenceUrl)
console.debug('header: ', header)
console.error(`Inference error:`, JSON.stringify(err))
if (this.isCancelled || message.content.length) {
message.status = MessageStatus.Stopped
events.emit(MessageEvent.OnMessageUpdate, message)
return
}
message.status = MessageStatus.Error
message.content[0] = {
type: ContentType.Text,
text: {
value: err.message,
annotations: [],
},
}
message.error_code = err.code
events.emit(MessageEvent.OnMessageUpdate, message)
},
})
}
/**
* Stops the inference.
*/
override stopInference() {
this.isCancelled = true
this.controller?.abort()
}
/**
* Headers for the inference request
*/
async headers(): Promise<HeadersInit> {
return {}
}
}

27
core/src/browser/extensions/engines/RemoteOAIEngine.ts Normal file
View File

@ -0,0 +1,27 @@
import { OAIEngine } from './OAIEngine'
/**
* Base OAI Remote Inference Provider
* Added the implementation of loading and unloading model (applicable to local inference providers)
*/
export abstract class RemoteOAIEngine extends OAIEngine {
apiKey?: string
/**
* On extension load, subscribe to events.
*/
override onLoad() {
super.onLoad()
}
/**
* Headers for the inference request
*/
override async headers(): Promise<HeadersInit> {
return {
...(this.apiKey && {
'Authorization': `Bearer ${this.apiKey}`,
'api-key': `${this.apiKey}`,
}),
}
}
}

95
core/src/browser/extensions/engines/helpers/sse.ts Normal file
View File

@ -0,0 +1,95 @@
import { Observable } from 'rxjs'
import { ErrorCode, ModelRuntimeParams } from '../../../../types'
/**
* Sends a request to the inference server to generate a response based on the recent messages.
* @param recentMessages - An array of recent messages to use as context for the inference.
* @returns An Observable that emits the generated response as a string.
*/
export function requestInference(
inferenceUrl: string,
requestBody: any,
model: {
id: string
parameters: ModelRuntimeParams
},
controller?: AbortController,
headers?: HeadersInit,
transformResponse?: Function
): Observable<string> {
return new Observable((subscriber) => {
fetch(inferenceUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Accept': model.parameters.stream ? 'text/event-stream' : 'application/json',
...headers,
},
body: JSON.stringify(requestBody),
signal: controller?.signal,
})
.then(async (response) => {
if (!response.ok) {
const data = await response.json()
let errorCode = ErrorCode.Unknown
if (data.error) {
errorCode = data.error.code ?? data.error.type ?? ErrorCode.Unknown
} else if (response.status === 401) {
errorCode = ErrorCode.InvalidApiKey
}
const error = {
message: data.error?.message ?? 'Error occurred.',
code: errorCode,
}
subscriber.error(error)
subscriber.complete()
return
}
if (model.parameters.stream === false) {
const data = await response.json()
if (transformResponse) {
subscriber.next(transformResponse(data))
} else {
subscriber.next(data.choices[0]?.message?.content ?? '')
}
} else {
const stream = response.body
const decoder = new TextDecoder('utf-8')
const reader = stream?.getReader()
let content = ''
while (true && reader) {
const { done, value } = await reader.read()
if (done) {
break
}
const text = decoder.decode(value)
const lines = text.trim().split('\n')
let cachedLines = ''
for (const line of lines) {
try {
if (transformResponse) {
content += transformResponse(line)
subscriber.next(content ?? '')
} else {
const toParse = cachedLines + line
if (!line.includes('data: [DONE]')) {
const data = JSON.parse(toParse.replace('data: ', ''))
content += data.choices[0]?.delta?.content ?? ''
if (content.startsWith('assistant: ')) {
content = content.replace('assistant: ', '')
}
if (content !== '') subscriber.next(content)
}
}
} catch {
cachedLines = line
}
}
}
}
subscriber.complete()
})
.catch((err) => subscriber.error(err))
})
}

5
core/src/browser/extensions/engines/index.ts Normal file
View File

@ -0,0 +1,5 @@
export * from './AIEngine'
export * from './OAIEngine'
export * from './LocalOAIEngine'
export * from './RemoteOAIEngine'
export * from './EngineManager'

30
core/src/browser/extensions/index.ts Normal file
View File

@ -0,0 +1,30 @@
/**
* Conversational extension. Persists and retrieves conversations.
* @module
*/
export { ConversationalExtension } from './conversational'
/**
* Inference extension. Start, stop and inference models.
*/
export { InferenceExtension } from './inference'
/**
* Monitoring extension for system monitoring.
*/
export { MonitoringExtension } from './monitoring'
/**
* Assistant extension for managing assistants.
*/
export { AssistantExtension } from './assistant'
/**
* Model extension for managing models.
*/
export { ModelExtension } from './model'
/**
* Base AI Engines.
*/
export * from './engines'

16
core/src/browser/extensions/inference.ts Normal file
View File

@ -0,0 +1,16 @@
import { InferenceInterface, MessageRequest, ThreadMessage } from '../../types'
import { BaseExtension, ExtensionTypeEnum } from '../extension'
/**
* Inference extension. Start, stop and inference models.
*/
export abstract class InferenceExtension extends BaseExtension implements InferenceInterface {
/**
* Inference extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Inference
}
abstract inference(data: MessageRequest): Promise<ThreadMessage>
}

36
core/src/browser/extensions/model.ts Normal file
View File

@ -0,0 +1,36 @@
import { BaseExtension, ExtensionTypeEnum } from '../extension'
import {
GpuSetting,
HuggingFaceRepoData,
ImportingModel,
Model,
ModelInterface,
OptionType,
} from '../../types'
/**
* Model extension for managing models.
*/
export abstract class ModelExtension extends BaseExtension implements ModelInterface {
/**
* Model extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.Model
}
abstract downloadModel(
model: Model,
gpuSettings?: GpuSetting,
network?: { proxy: string; ignoreSSL?: boolean }
): Promise<void>
abstract cancelModelDownload(modelId: string): Promise<void>
abstract deleteModel(modelId: string): Promise<void>
abstract saveModel(model: Model): Promise<void>
abstract getDownloadedModels(): Promise<Model[]>
abstract getConfiguredModels(): Promise<Model[]>
abstract importModels(models: ImportingModel[], optionType: OptionType): Promise<void>
abstract updateModelInfo(modelInfo: Partial<Model>): Promise<Model>
abstract fetchHuggingFaceRepoData(repoId: string): Promise<HuggingFaceRepoData>
abstract getDefaultModel(): Promise<Model>
}

20
core/src/browser/extensions/monitoring.ts Normal file
View File

@ -0,0 +1,20 @@
import { BaseExtension, ExtensionTypeEnum } from '../extension'
import { GpuSetting, MonitoringInterface, OperatingSystemInfo } from '../../types'
/**
* Monitoring extension for system monitoring.
* @extends BaseExtension
*/
export abstract class MonitoringExtension extends BaseExtension implements MonitoringInterface {
/**
* Monitoring extension type.
*/
type(): ExtensionTypeEnum | undefined {
return ExtensionTypeEnum.SystemMonitoring
}
abstract getGpuSetting(): Promise<GpuSetting | undefined>
abstract getResourcesInfo(): Promise<any>
abstract getCurrentLoad(): Promise<any>
abstract getOsInfo(): Promise<OperatingSystemInfo>
}

87
core/src/browser/fs.ts Normal file
View File

@ -0,0 +1,87 @@
import { FileStat } from '../types'
/**
* Writes data to a file at the specified path.
* @returns {Promise<any>} A Promise that resolves when the file is written successfully.
*/
const writeFileSync = (...args: any[]) => globalThis.core.api?.writeFileSync(...args)
/**
* Writes blob data to a file at the specified path.
* @param path - The path to file.
* @param data - The blob data.
* @returns
*/
const writeBlob: (path: string, data: string) => Promise<any> = (path, data) =>
globalThis.core.api?.writeBlob(path, data)
/**
* Reads the contents of a file at the specified path.
* @returns {Promise<any>} A Promise that resolves with the contents of the file.
*/
const readFileSync = (...args: any[]) => globalThis.core.api?.readFileSync(...args)
/**
* Check whether the file exists
* @param {string} path
* @returns {boolean} A boolean indicating whether the path is a file.
*/
const existsSync = (...args: any[]) => globalThis.core.api?.existsSync(...args)
/**
* List the directory files
* @returns {Promise<any>} A Promise that resolves with the contents of the directory.
*/
const readdirSync = (...args: any[]) => globalThis.core.api?.readdirSync(...args)
/**
* Creates a directory at the specified path.
* @returns {Promise<any>} A Promise that resolves when the directory is created successfully.
*/
const mkdir = (...args: any[]) => globalThis.core.api?.mkdir(...args)
/**
* Removes a directory at the specified path.
* @returns {Promise<any>} A Promise that resolves when the directory is removed successfully.
*/
const rm = (...args: any[]) => globalThis.core.api?.rm(...args, { recursive: true, force: true })
/**
* Deletes a file from the local file system.
* @param {string} path - The path of the file to delete.
* @returns {Promise<any>} A Promise that resolves when the file is deleted.
*/
const unlinkSync = (...args: any[]) => globalThis.core.api?.unlinkSync(...args)
/**
* Appends data to a file at the specified path.
*/
const appendFileSync = (...args: any[]) => globalThis.core.api?.appendFileSync(...args)
const copyFile: (src: string, dest: string) => Promise<void> = (src, dest) =>
globalThis.core.api?.copyFile(src, dest)
/**
* Gets the file's stats.
*
* @param path - The path to the file.
* @param outsideJanDataFolder - Whether the file is outside the Jan data folder.
* @returns {Promise<FileStat>} - A promise that resolves with the file's stats.
*/
const fileStat: (path: string, outsideJanDataFolder?: boolean) => Promise<FileStat | undefined> = (
path,
outsideJanDataFolder
) => globalThis.core.api?.fileStat(path, outsideJanDataFolder)
// TODO: Export `dummy` fs functions automatically
// Currently adding these manually
export const fs = {
writeFileSync,
readFileSync,
existsSync,
readdirSync,
mkdir,
rm,
unlinkSync,
appendFileSync,
copyFile,
fileStat,
writeBlob,
}

35
core/src/browser/index.ts Normal file
View File

@ -0,0 +1,35 @@
/**
* Export Core module
* @module
*/
export * from './core'
/**
* Export Event module.
* @module
*/
export * from './events'
/**
* Export Filesystem module.
* @module
*/
export * from './fs'
/**
* Export Extension module.
* @module
*/
export * from './extension'
/**
* Export all base extensions.
* @module
*/
export * from './extensions'
/**
* Export all base tools.
* @module
*/
export * from './tools'

2
core/src/browser/tools/index.ts Normal file
View File

@ -0,0 +1,2 @@
export * from './manager'
export * from './tool'

47
core/src/browser/tools/manager.ts Normal file
View File

@ -0,0 +1,47 @@
import { AssistantTool, MessageRequest } from '../../types'
import { InferenceTool } from './tool'
/**
* Manages the registration and retrieval of inference tools.
*/
export class ToolManager {
public tools = new Map<string, InferenceTool>()
/**
* Registers a tool.
* @param tool - The tool to register.
*/
register<T extends InferenceTool>(tool: T) {
this.tools.set(tool.name, tool)
}
/**
* Retrieves a tool by it's name.
* @param name - The name of the tool to retrieve.
* @returns The tool, if found.
*/
get<T extends InferenceTool>(name: string): T | undefined {
return this.tools.get(name) as T | undefined
}
/*
** Process the message request with the tools.
*/
process(request: MessageRequest, tools: AssistantTool[]): Promise<MessageRequest> {
return tools.reduce((prevPromise, currentTool) => {
return prevPromise.then((prevResult) => {
return currentTool.enabled
? this.get(currentTool.type)?.process(prevResult, currentTool) ??
Promise.resolve(prevResult)
: Promise.resolve(prevResult)
})
}, Promise.resolve(request))
}
/**
* The instance of the tool manager.
*/
static instance(): ToolManager {
return (window.core?.toolManager as ToolManager) ?? new ToolManager()
}
}

12
core/src/browser/tools/tool.ts Normal file
View File

@ -0,0 +1,12 @@
import { AssistantTool, MessageRequest } from '../../types'
/**
* Represents a base inference tool.
*/
export abstract class InferenceTool {
abstract name: string
/*
** Process a message request and return the processed message request.
*/
abstract process(request: MessageRequest, tool?: AssistantTool): Promise<MessageRequest>
}

6
core/src/index.ts
View File

@ -4,6 +4,12 @@
*/
export * from './types'
/**
* Export browser module
* @module
*/
export * from './browser'
/**
* Declare global object
*/

8
core/src/node/api/HttpServer.ts Normal file
View File

@ -0,0 +1,8 @@
export interface HttpServer {
post: (route: string, handler: (req: any, res: any) => Promise<any>) => void
get: (route: string, handler: (req: any, res: any) => Promise<any>) => void
patch: (route: string, handler: (req: any, res: any) => Promise<any>) => void
put: (route: string, handler: (req: any, res: any) => Promise<any>) => void
delete: (route: string, handler: (req: any, res: any) => Promise<any>) => void
register: (router: any, opts?: any) => void
}

43
core/src/node/api/common/adapter.ts Normal file
View File

@ -0,0 +1,43 @@
import {
AppRoute,
DownloadRoute,
ExtensionRoute,
FileManagerRoute,
FileSystemRoute,
} from '../../../types/api'
import { Downloader } from '../processors/download'
import { FileSystem } from '../processors/fs'
import { Extension } from '../processors/extension'
import { FSExt } from '../processors/fsExt'
import { App } from '../processors/app'
export class RequestAdapter {
downloader: Downloader
fileSystem: FileSystem
extension: Extension
fsExt: FSExt
app: App
constructor(observer?: Function) {
this.downloader = new Downloader(observer)
this.fileSystem = new FileSystem()
this.extension = new Extension()
this.fsExt = new FSExt()
this.app = new App()
}
// TODO: Clearer Factory pattern here
process(route: string, ...args: any) {
if (route in DownloadRoute) {
return this.downloader.process(route, ...args)
} else if (route in FileSystemRoute) {
return this.fileSystem.process(route, ...args)
} else if (route in ExtensionRoute) {
return this.extension.process(route, ...args)
} else if (route in FileManagerRoute) {
return this.fsExt.process(route, ...args)
} else if (route in AppRoute) {
return this.app.process(route, ...args)
}
}
}

20
core/src/node/api/common/handler.ts Normal file
View File

@ -0,0 +1,20 @@
import { CoreRoutes } from '../../../types/api'
import { RequestAdapter } from './adapter'
export type Handler = (route: string, args: any) => any
export class RequestHandler {
handler: Handler
adapter: RequestAdapter
constructor(handler: Handler, observer?: Function) {
this.handler = handler
this.adapter = new RequestAdapter(observer)
}
handle() {
CoreRoutes.map((route) => {
this.handler(route, async (...args: any[]) => this.adapter.process(route, ...args))
})
}
}

3
core/src/node/api/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * from './HttpServer'
export * from './restful/v1'
export * from './common/handler'

3
core/src/node/api/processors/Processor.ts Normal file
View File

@ -0,0 +1,3 @@
export abstract class Processor {
abstract process(key: string, ...args: any[]): any
}

93
core/src/node/api/processors/app.ts Normal file
View File

@ -0,0 +1,93 @@
import { basename, isAbsolute, join, relative } from 'path'
import { Processor } from './Processor'
import {
log as writeLog,
appResourcePath,
getAppConfigurations as appConfiguration,
updateAppConfiguration,
} from '../../helper'
export class App implements Processor {
observer?: Function
constructor(observer?: Function) {
this.observer = observer
}
process(key: string, ...args: any[]): any {
const instance = this as any
const func = instance[key]
return func(...args)
}
/**
* Joins multiple paths together, respect to the current OS.
*/
joinPath(args: any[]) {
return join(...args)
}
/**
* Checks if the given path is a subdirectory of the given directory.
*
* @param _event - The IPC event object.
* @param from - The path to check.
* @param to - The directory to check against.
*
* @returns {Promise<boolean>} - A promise that resolves with the result.
*/
isSubdirectory(from: any, to: any) {
const rel = relative(from, to)
const isSubdir = rel && !rel.startsWith('..') && !isAbsolute(rel)
if (isSubdir === '') return false
else return isSubdir
}
/**
* Retrieve basename from given path, respect to the current OS.
*/
baseName(args: any) {
return basename(args)
}
/**
* Log message to log file.
*/
log(args: any) {
writeLog(args)
}
getAppConfigurations() {
return appConfiguration()
}
async updateAppConfiguration(args: any) {
await updateAppConfiguration(args)
}
/**
* Start Jan API Server.
*/
async startServer(args?: any) {
const { startServer } = require('@janhq/server')
return startServer({
host: args?.host,
port: args?.port,
isCorsEnabled: args?.isCorsEnabled,
isVerboseEnabled: args?.isVerboseEnabled,
schemaPath: join(await appResourcePath(), 'docs', 'openapi', 'jan.yaml'),
baseDir: join(await appResourcePath(), 'docs', 'openapi'),
prefix: args?.prefix,
})
}
/**
* Stop Jan API Server.
*/
stopServer() {
const { stopServer } = require('@janhq/server')
return stopServer()
}
}

161
core/src/node/api/processors/download.ts Normal file
View File

@ -0,0 +1,161 @@
import { resolve, sep } from 'path'
import { DownloadEvent } from '../../../types/api'
import { normalizeFilePath, validatePath } from '../../helper/path'
import { getJanDataFolderPath } from '../../helper'
import { DownloadManager } from '../../helper/download'
import { createWriteStream, renameSync } from 'fs'
import { Processor } from './Processor'
import { DownloadRequest, DownloadState, NetworkConfig } from '../../../types'
export class Downloader implements Processor {
observer?: Function
constructor(observer?: Function) {
this.observer = observer
}
process(key: string, ...args: any[]): any {
const instance = this as any
const func = instance[key]
return func(this.observer, ...args)
}
downloadFile(observer: any, downloadRequest: DownloadRequest, network?: NetworkConfig) {
const request = require('request')
const progress = require('request-progress')
const strictSSL = !network?.ignoreSSL
const proxy = network?.proxy?.startsWith('http') ? network.proxy : undefined
const { localPath, url } = downloadRequest
let normalizedPath = localPath
if (typeof localPath === 'string') {
normalizedPath = normalizeFilePath(localPath)
}
const array = normalizedPath.split(sep)
const fileName = array.pop() ?? ''
const modelId = array.pop() ?? ''
const destination = resolve(getJanDataFolderPath(), normalizedPath)
validatePath(destination)
const rq = request({ url, strictSSL, proxy })
// Put request to download manager instance
DownloadManager.instance.setRequest(normalizedPath, rq)
// Downloading file to a temp file first
const downloadingTempFile = `${destination}.download`
// adding initial download state
const initialDownloadState: DownloadState = {
modelId,
fileName,
time: {
elapsed: 0,
remaining: 0,
},
speed: 0,
percent: 0,
size: {
total: 0,
transferred: 0,
},
children: [],
downloadState: 'downloading',
extensionId: downloadRequest.extensionId,
downloadType: downloadRequest.downloadType,
localPath: normalizedPath,
}
DownloadManager.instance.downloadProgressMap[modelId] = initialDownloadState
DownloadManager.instance.downloadInfo[normalizedPath] = initialDownloadState
if (downloadRequest.downloadType === 'extension') {
observer?.(DownloadEvent.onFileDownloadUpdate, initialDownloadState)
}
progress(rq, {})
.on('progress', (state: any) => {
const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId]
const downloadState: DownloadState = {
...currentDownloadState,
...state,
fileName: fileName,
downloadState: 'downloading',
}
console.debug('progress: ', downloadState)
observer?.(DownloadEvent.onFileDownloadUpdate, downloadState)
DownloadManager.instance.downloadProgressMap[modelId] = downloadState
})
.on('error', (error: Error) => {
const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId]
const downloadState: DownloadState = {
...currentDownloadState,
fileName: fileName,
error: error.message,
downloadState: 'error',
}
observer?.(DownloadEvent.onFileDownloadError, downloadState)
DownloadManager.instance.downloadProgressMap[modelId] = downloadState
})
.on('end', () => {
const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId]
if (currentDownloadState && DownloadManager.instance.networkRequests[normalizedPath]) {
// Finished downloading, rename temp file to actual file
renameSync(downloadingTempFile, destination)
const downloadState: DownloadState = {
...currentDownloadState,
fileName: fileName,
downloadState: 'end',
}
observer?.(DownloadEvent.onFileDownloadSuccess, downloadState)
DownloadManager.instance.downloadProgressMap[modelId] = downloadState
}
})
.pipe(createWriteStream(downloadingTempFile))
}
abortDownload(observer: any, fileName: string) {
const rq = DownloadManager.instance.networkRequests[fileName]
if (rq) {
DownloadManager.instance.networkRequests[fileName] = undefined
rq?.abort()
}
const downloadInfo = DownloadManager.instance.downloadInfo[fileName]
observer?.(DownloadEvent.onFileDownloadError, {
...downloadInfo,
fileName,
error: 'aborted',
})
}
resumeDownload(_observer: any, fileName: any) {
DownloadManager.instance.networkRequests[fileName]?.resume()
}
pauseDownload(_observer: any, fileName: any) {
DownloadManager.instance.networkRequests[fileName]?.pause()
}
async getFileSize(_observer: any, url: string): Promise<number> {
return new Promise((resolve, reject) => {
const request = require('request')
request(
{
url,
method: 'HEAD',
},
function (err: any, response: any) {
if (err) {
console.error('Getting file size failed:', err)
reject(err)
} else {
const size: number = response.headers['content-length'] ?? -1
resolve(size)
}
}
)
})
}
}

88
core/src/node/api/processors/extension.ts Normal file
View File

@ -0,0 +1,88 @@
import { readdirSync } from 'fs'
import { join, extname } from 'path'
import { Processor } from './Processor'
import { ModuleManager } from '../../helper/module'
import { getJanExtensionsPath as getPath } from '../../helper'
import {
getActiveExtensions as getExtensions,
getExtension,
removeExtension,
installExtensions,
} from '../../extension/store'
import { appResourcePath } from '../../helper/path'
export class Extension implements Processor {
observer?: Function
constructor(observer?: Function) {
this.observer = observer
}
process(key: string, ...args: any[]): any {
const instance = this as any
const func = instance[key]
return func(...args)
}
invokeExtensionFunc(modulePath: string, method: string, ...params: any[]) {
const module = require(join(getPath(), modulePath))
ModuleManager.instance.setModule(modulePath, module)
if (typeof module[method] === 'function') {
return module[method](...params)
} else {
console.debug(module[method])
console.error(`Function "${method}" does not exist in the module.`)
}
}
/**
* Returns the paths of the base extensions.
* @returns An array of paths to the base extensions.
*/
async baseExtensions() {
const baseExtensionPath = join(await appResourcePath(), 'pre-install')
return readdirSync(baseExtensionPath)
.filter((file) => extname(file) === '.tgz')
.map((file) => join(baseExtensionPath, file))
}
/**MARK: Extension Manager handlers */
async installExtension(extensions: any) {
// Install and activate all provided extensions
const installed = await installExtensions(extensions)
return JSON.parse(JSON.stringify(installed))
}
// Register IPC route to uninstall a extension
async uninstallExtension(extensions: any) {
// Uninstall all provided extensions
for (const ext of extensions) {
const extension = getExtension(ext)
await extension.uninstall()
if (extension.name) removeExtension(extension.name)
}
// Reload all renderer pages if needed
return true
}
// Register IPC route to update a extension
async updateExtension(extensions: any) {
// Update all provided extensions
const updated: any[] = []
for (const ext of extensions) {
const extension = getExtension(ext)
const res = await extension.update()
if (res) updated.push(extension)
}
// Reload all renderer pages if needed
return JSON.parse(JSON.stringify(updated))
}
getActiveExtensions() {
return JSON.parse(JSON.stringify(getExtensions()))
}
}

95
core/src/node/api/processors/fs.ts Normal file
View File

@ -0,0 +1,95 @@
import { join, resolve } from 'path'
import { normalizeFilePath, validatePath } from '../../helper/path'
import { getJanDataFolderPath } from '../../helper'
import { Processor } from './Processor'
import fs from 'fs'
export class FileSystem implements Processor {
observer?: Function
private static moduleName = 'fs'
constructor(observer?: Function) {
this.observer = observer
}
process(route: string, ...args: any): any {
const instance = this as any
const func = instance[route]
if (func) {
return func(...args)
} else {
return import(FileSystem.moduleName).then((mdl) =>
mdl[route](
...args.map((arg: any, index: number) => {
if(index !== 0) {
return arg
}
if (index === 0 && typeof arg !== 'string') {
throw new Error(`Invalid argument ${JSON.stringify(args)}`)
}
const path =
(arg.startsWith(`file:/`) || arg.startsWith(`file:\\`))
? join(getJanDataFolderPath(), normalizeFilePath(arg))
: arg
if(path.startsWith(`http://`) || path.startsWith(`https://`)) {
return path
}
const absolutePath = resolve(path)
validatePath(absolutePath)
return absolutePath
})
)
)
}
}
rm(...args: any): Promise<void> {
if (typeof args[0] !== 'string') {
throw new Error(`rm error: Invalid argument ${JSON.stringify(args)}`)
}
let path = args[0]
if (path.startsWith(`file:/`) || path.startsWith(`file:\\`)) {
path = join(getJanDataFolderPath(), normalizeFilePath(path))
}
const absolutePath = resolve(path)
validatePath(absolutePath)
return new Promise((resolve, reject) => {
fs.rm(absolutePath, { recursive: true, force: true }, (err) => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
}
mkdir(...args: any): Promise<void> {
if (typeof args[0] !== 'string') {
throw new Error(`mkdir error: Invalid argument ${JSON.stringify(args)}`)
}
let path = args[0]
if (path.startsWith(`file:/`) || path.startsWith(`file:\\`)) {
path = join(getJanDataFolderPath(), normalizeFilePath(path))
}
const absolutePath = resolve(path)
validatePath(absolutePath)
return new Promise((resolve, reject) => {
fs.mkdir(absolutePath, { recursive: true }, (err) => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
}
}

82
core/src/node/api/processors/fsExt.ts Normal file
View File

@ -0,0 +1,82 @@
import { join } from 'path'
import fs from 'fs'
import { appResourcePath, normalizeFilePath, validatePath } from '../../helper/path'
import { getJanDataFolderPath, getJanDataFolderPath as getPath } from '../../helper'
import { Processor } from './Processor'
import { FileStat } from '../../../types'
export class FSExt implements Processor {
observer?: Function
constructor(observer?: Function) {
this.observer = observer
}
process(key: string, ...args: any): any {
const instance = this as any
const func = instance[key]
return func(...args)
}
// Handles the 'getJanDataFolderPath' IPC event. This event is triggered to get the user space path.
getJanDataFolderPath() {
return Promise.resolve(getPath())
}
// Handles the 'getResourcePath' IPC event. This event is triggered to get the resource path.
getResourcePath() {
return appResourcePath()
}
// Handles the 'getUserHomePath' IPC event. This event is triggered to get the user home path.
getUserHomePath() {
return process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME']
}
// handle fs is directory here
fileStat(path: string, outsideJanDataFolder?: boolean) {
const normalizedPath = normalizeFilePath(path)
const fullPath = outsideJanDataFolder
? normalizedPath
: join(getJanDataFolderPath(), normalizedPath)
const isExist = fs.existsSync(fullPath)
if (!isExist) return undefined
const isDirectory = fs.lstatSync(fullPath).isDirectory()
const size = fs.statSync(fullPath).size
const fileStat: FileStat = {
isDirectory,
size,
}
return fileStat
}
writeBlob(path: string, data: any) {
try {
const normalizedPath = normalizeFilePath(path)
const dataBuffer = Buffer.from(data, 'base64')
const writePath = join(getJanDataFolderPath(), normalizedPath)
validatePath(writePath)
fs.writeFileSync(writePath, dataBuffer)
} catch (err) {
console.error(`writeFile ${path} result: ${err}`)
}
}
copyFile(src: string, dest: string): Promise<void> {
validatePath(dest)
return new Promise((resolve, reject) => {
fs.copyFile(src, dest, (err) => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
}
}

23
core/src/node/api/restful/app/download.ts Normal file
View File

@ -0,0 +1,23 @@
import { DownloadRoute } from '../../../../types/api'
import { DownloadManager } from '../../../helper/download'
import { HttpServer } from '../../HttpServer'
export const downloadRouter = async (app: HttpServer) => {
app.get(`/download/${DownloadRoute.getDownloadProgress}/:modelId`, async (req, res) => {
const modelId = req.params.modelId
console.debug(`Getting download progress for model ${modelId}`)
console.debug(
`All Download progress: ${JSON.stringify(DownloadManager.instance.downloadProgressMap)}`
)
// check if null DownloadManager.instance.downloadProgressMap
if (!DownloadManager.instance.downloadProgressMap[modelId]) {
return res.status(404).send({
message: 'Download progress not found',
})
} else {
return res.status(200).send(DownloadManager.instance.downloadProgressMap[modelId])
}
})
}

13
core/src/node/api/restful/app/handlers.ts Normal file
View File

@ -0,0 +1,13 @@
import { HttpServer } from '../../HttpServer'
import { Handler, RequestHandler } from '../../common/handler'
export function handleRequests(app: HttpServer) {
const restWrapper: Handler = (route: string, listener: (...args: any[]) => any) => {
app.post(`/app/${route}`, async (request: any, reply: any) => {
const args = JSON.parse(request.body) as any[]
reply.send(JSON.stringify(await listener(...args)))
})
}
const handler = new RequestHandler(restWrapper)
handler.handle()
}

82
core/src/node/api/restful/common.ts Normal file
View File

@ -0,0 +1,82 @@
import { HttpServer } from '../HttpServer'
import {
chatCompletions,
deleteBuilder,
downloadModel,
getBuilder,
retrieveBuilder,
createMessage,
createThread,
getMessages,
retrieveMessage,
updateThread,
} from './helper/builder'
import { JanApiRouteConfiguration } from './helper/configuration'
import { startModel, stopModel } from './helper/startStopModel'
import { ModelSettingParams } from '../../../types'
export const commonRouter = async (app: HttpServer) => {
const normalizeData = (data: any) => {
return {
object: 'list',
data,
}
}
// Common Routes
// Read & Delete :: Threads | Models | Assistants
Object.keys(JanApiRouteConfiguration).forEach((key) => {
app.get(`/${key}`, async (_request) =>
getBuilder(JanApiRouteConfiguration[key]).then(normalizeData)
)
app.get(`/${key}/:id`, async (request: any) =>
retrieveBuilder(JanApiRouteConfiguration[key], request.params.id)
)
app.delete(`/${key}/:id`, async (request: any) =>
deleteBuilder(JanApiRouteConfiguration[key], request.params.id)
)
})
// Threads
app.post(`/threads`, async (req, res) => createThread(req.body))
app.get(`/threads/:threadId/messages`, async (req, res) =>
getMessages(req.params.threadId).then(normalizeData)
)
app.get(`/threads/:threadId/messages/:messageId`, async (req, res) =>
retrieveMessage(req.params.threadId, req.params.messageId)
)
app.post(`/threads/:threadId/messages`, async (req, res) =>
createMessage(req.params.threadId as any, req.body as any)
)
app.patch(`/threads/:threadId`, async (request: any) =>
updateThread(request.params.threadId, request.body)
)
// Models
app.get(`/models/download/:modelId`, async (request: any) =>
downloadModel(request.params.modelId, {
ignoreSSL: request.query.ignoreSSL === 'true',
proxy: request.query.proxy,
})
)
app.put(`/models/:modelId/start`, async (request: any) => {
let settingParams: ModelSettingParams | undefined = undefined
if (Object.keys(request.body).length !== 0) {
settingParams = JSON.parse(request.body) as ModelSettingParams
}
return startModel(request.params.modelId, settingParams)
})
app.put(`/models/:modelId/stop`, async (request: any) => stopModel(request.params.modelId))
// Chat Completion
app.post(`/chat/completions`, async (request: any, reply: any) => chatCompletions(request, reply))
}

362
core/src/node/api/restful/helper/builder.ts Normal file
View File

@ -0,0 +1,362 @@
import {
existsSync,
readdirSync,
readFileSync,
writeFileSync,
mkdirSync,
appendFileSync,
createWriteStream,
rmdirSync,
} from 'fs'
import { JanApiRouteConfiguration, RouteConfiguration } from './configuration'
import { join } from 'path'
import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../types'
import { getEngineConfiguration, getJanDataFolderPath } from '../../../helper'
import { DEFAULT_CHAT_COMPLETION_URL } from './consts'
// TODO: Refactor these
export const getBuilder = async (configuration: RouteConfiguration) => {
const directoryPath = join(getJanDataFolderPath(), configuration.dirName)
try {
if (!existsSync(directoryPath)) {
console.debug('model folder not found')
return []
}
const files: string[] = readdirSync(directoryPath)
const allDirectories: string[] = []
for (const file of files) {
if (file === '.DS_Store') continue
allDirectories.push(file)
}
const results = allDirectories
.map((dirName) => {
const jsonPath = join(directoryPath, dirName, configuration.metadataFileName)
return readModelMetadata(jsonPath)
})
.filter((data) => !!data)
const modelData = results
.map((result: any) => {
try {
return JSON.parse(result)
} catch (err) {
console.error(err)
}
})
.filter((e: any) => !!e)
return modelData
} catch (err) {
console.error(err)
return []
}
}
const readModelMetadata = (path: string): string | undefined => {
if (existsSync(path)) {
return readFileSync(path, 'utf-8')
} else {
return undefined
}
}
export const retrieveBuilder = async (configuration: RouteConfiguration, id: string) => {
const data = await getBuilder(configuration)
const filteredData = data.filter((d: any) => d.id === id)[0]
if (!filteredData) {
return undefined
}
return filteredData
}
export const deleteBuilder = async (configuration: RouteConfiguration, id: string) => {
if (configuration.dirName === 'assistants' && id === 'jan') {
return {
message: 'Cannot delete Jan assistant',
}
}
const directoryPath = join(getJanDataFolderPath(), configuration.dirName)
try {
const data = await retrieveBuilder(configuration, id)
if (!data) {
return {
message: 'Not found',
}
}
const objectPath = join(directoryPath, id)
rmdirSync(objectPath, { recursive: true })
return {
id: id,
object: configuration.delete.object,
deleted: true,
}
} catch (ex) {
console.error(ex)
}
}
export const getMessages = async (threadId: string): Promise<ThreadMessage[]> => {
const threadDirPath = join(getJanDataFolderPath(), 'threads', threadId)
const messageFile = 'messages.jsonl'
try {
const files: string[] = readdirSync(threadDirPath)
if (!files.includes(messageFile)) {
console.error(`${threadDirPath} not contains message file`)
return []
}
const messageFilePath = join(threadDirPath, messageFile)
if (!existsSync(messageFilePath)) {
console.debug('message file not found')
return []
}
const lines = readFileSync(messageFilePath, 'utf-8')
.toString()
.split('\n')
.filter((line: any) => line !== '')
const messages: ThreadMessage[] = []
lines.forEach((line: string) => {
messages.push(JSON.parse(line) as ThreadMessage)
})
return messages
} catch (err) {
console.error(err)
return []
}
}
export const retrieveMessage = async (threadId: string, messageId: string) => {
const messages = await getMessages(threadId)
const filteredMessages = messages.filter((m) => m.id === messageId)
if (!filteredMessages || filteredMessages.length === 0) {
return {
message: 'Not found',
}
}
return filteredMessages[0]
}
export const createThread = async (thread: any) => {
const threadMetadataFileName = 'thread.json'
// TODO: add validation
if (!thread.assistants || thread.assistants.length === 0) {
return {
message: 'Thread must have at least one assistant',
}
}
const threadId = generateThreadId(thread.assistants[0].assistant_id)
try {
const updatedThread = {
...thread,
id: threadId,
created: Date.now(),
updated: Date.now(),
}
const threadDirPath = join(getJanDataFolderPath(), 'threads', updatedThread.id)
const threadJsonPath = join(threadDirPath, threadMetadataFileName)
if (!existsSync(threadDirPath)) {
mkdirSync(threadDirPath)
}
await writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2))
return updatedThread
} catch (err) {
return {
error: err,
}
}
}
export const updateThread = async (threadId: string, thread: any) => {
const threadMetadataFileName = 'thread.json'
const currentThreadData = await retrieveBuilder(JanApiRouteConfiguration.threads, threadId)
if (!currentThreadData) {
return {
message: 'Thread not found',
}
}
// we don't want to update the id and object
delete thread.id
delete thread.object
const updatedThread = {
...currentThreadData,
...thread,
updated: Date.now(),
}
try {
const threadDirPath = join(getJanDataFolderPath(), 'threads', updatedThread.id)
const threadJsonPath = join(threadDirPath, threadMetadataFileName)
await writeFileSync(threadJsonPath, JSON.stringify(updatedThread, null, 2))
return updatedThread
} catch (err) {
return {
message: err,
}
}
}
const generateThreadId = (assistantId: string) => {
return `${assistantId}_${(Date.now() / 1000).toFixed(0)}`
}
export const createMessage = async (threadId: string, message: any) => {
const threadMessagesFileName = 'messages.jsonl'
try {
const { ulid } = require('ulidx')
const msgId = ulid()
const createdAt = Date.now()
const threadMessage: ThreadMessage = {
id: msgId,
thread_id: threadId,
status: MessageStatus.Ready,
created: createdAt,
updated: createdAt,
object: 'thread.message',
role: message.role,
content: [
{
type: ContentType.Text,
text: {
value: message.content,
annotations: [],
},
},
],
}
const threadDirPath = join(getJanDataFolderPath(), 'threads', threadId)
const threadMessagePath = join(threadDirPath, threadMessagesFileName)
if (!existsSync(threadDirPath)) {
mkdirSync(threadDirPath)
}
appendFileSync(threadMessagePath, JSON.stringify(threadMessage) + '\n')
return threadMessage
} catch (err) {
return {
message: err,
}
}
}
export const downloadModel = async (
modelId: string,
network?: { proxy?: string; ignoreSSL?: boolean }
) => {
const strictSSL = !network?.ignoreSSL
const proxy = network?.proxy?.startsWith('http') ? network.proxy : undefined
const model = await retrieveBuilder(JanApiRouteConfiguration.models, modelId)
if (!model || model.object !== 'model') {
return {
message: 'Model not found',
}
}
const directoryPath = join(getJanDataFolderPath(), 'models', modelId)
if (!existsSync(directoryPath)) {
mkdirSync(directoryPath)
}
// path to model binary
const modelBinaryPath = join(directoryPath, modelId)
const request = require('request')
const progress = require('request-progress')
for (const source of model.sources) {
const rq = request({ url: source, strictSSL, proxy })
progress(rq, {})
.on('progress', function (state: any) {
console.debug('progress', JSON.stringify(state, null, 2))
})
.on('error', function (err: Error) {
console.error('error', err)
})
.on('end', function () {
console.debug('end')
})
.pipe(createWriteStream(modelBinaryPath))
}
return {
message: `Starting download ${modelId}`,
}
}
export const chatCompletions = async (request: any, reply: any) => {
const modelList = await getBuilder(JanApiRouteConfiguration.models)
const modelId = request.body.model
const matchedModels = modelList.filter((model: Model) => model.id === modelId)
if (matchedModels.length === 0) {
const error = {
error: {
message: `The model ${request.body.model} does not exist`,
type: 'invalid_request_error',
param: null,
code: 'model_not_found',
},
}
reply.code(404).send(error)
return
}
const requestedModel = matchedModels[0]
const engineConfiguration = await getEngineConfiguration(requestedModel.engine)
let apiKey: string | undefined = undefined
let apiUrl: string = DEFAULT_CHAT_COMPLETION_URL
if (engineConfiguration) {
apiKey = engineConfiguration.api_key
apiUrl = engineConfiguration.full_url ?? DEFAULT_CHAT_COMPLETION_URL
}
const headers: Record<string, any> = {
'Content-Type': 'application/json',
}
if (apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`
headers['api-key'] = apiKey
}
if (requestedModel.engine === 'openai' && request.body.stop) {
// openai only allows max 4 stop words
request.body.stop = request.body.stop.slice(0, 4)
}
const fetch = require('node-fetch')
const response = await fetch(apiUrl, {
method: 'POST',
headers: headers,
body: JSON.stringify(request.body),
})
if (response.status !== 200) {
console.error(response)
reply.code(400).send(response)
} else {
reply.raw.writeHead(200, {
'Content-Type': request.body.stream === true ? 'text/event-stream' : 'application/json',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': '*',
})
response.body.pipe(reply.raw)
}
}

31
core/src/node/api/restful/helper/configuration.ts Normal file
View File

@ -0,0 +1,31 @@
export const JanApiRouteConfiguration: Record<string, RouteConfiguration> = {
models: {
dirName: 'models',
metadataFileName: 'model.json',
delete: {
object: 'model',
},
},
assistants: {
dirName: 'assistants',
metadataFileName: 'assistant.json',
delete: {
object: 'assistant',
},
},
threads: {
dirName: 'threads',
metadataFileName: 'thread.json',
delete: {
object: 'thread',
},
},
}
export type RouteConfiguration = {
dirName: string
metadataFileName: string
delete: {
object: string
}
}

19
core/src/node/api/restful/helper/consts.ts Normal file
View File

@ -0,0 +1,19 @@
// The PORT to use for the Nitro subprocess
export const NITRO_DEFAULT_PORT = 3928
// The HOST address to use for the Nitro subprocess
export const LOCAL_HOST = '127.0.0.1'
export const SUPPORTED_MODEL_FORMAT = '.gguf'
// The URL for the Nitro subprocess
const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${NITRO_DEFAULT_PORT}`
// The URL for the Nitro subprocess to load a model
export const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/server/loadmodel`
// The URL for the Nitro subprocess to validate a model
export const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/server/modelstatus`
// The URL for the Nitro subprocess to kill itself
export const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`
export const DEFAULT_CHAT_COMPLETION_URL = `http://${LOCAL_HOST}:${NITRO_DEFAULT_PORT}/inferences/server/chat_completion` // default nitro url

355
core/src/node/api/restful/helper/startStopModel.ts Normal file
View File

@ -0,0 +1,355 @@
import fs from 'fs'
import { join } from 'path'
import {
getJanDataFolderPath,
getJanExtensionsPath,
getSystemResourceInfo,
log,
} from '../../../helper'
import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
import { Model, ModelSettingParams, PromptTemplate } from '../../../../types'
import {
LOCAL_HOST,
NITRO_DEFAULT_PORT,
NITRO_HTTP_KILL_URL,
NITRO_HTTP_LOAD_MODEL_URL,
NITRO_HTTP_VALIDATE_MODEL_URL,
SUPPORTED_MODEL_FORMAT,
} from './consts'
// The subprocess instance for Nitro
let subprocess: ChildProcessWithoutNullStreams | undefined = undefined
// TODO: move this to core type
interface NitroModelSettings extends ModelSettingParams {
llama_model_path: string
cpu_threads: number
}
export const startModel = async (modelId: string, settingParams?: ModelSettingParams) => {
try {
await runModel(modelId, settingParams)
return {
message: `Model ${modelId} started`,
}
} catch (e) {
return {
error: e,
}
}
}
const runModel = async (modelId: string, settingParams?: ModelSettingParams): Promise<void> => {
const janDataFolderPath = getJanDataFolderPath()
const modelFolderFullPath = join(janDataFolderPath, 'models', modelId)
if (!fs.existsSync(modelFolderFullPath)) {
throw new Error(`Model not found: ${modelId}`)
}
const files: string[] = fs.readdirSync(modelFolderFullPath)
// Look for GGUF model file
const ggufBinFile = files.find((file) => file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT))
const modelMetadataPath = join(modelFolderFullPath, 'model.json')
const modelMetadata: Model = JSON.parse(fs.readFileSync(modelMetadataPath, 'utf-8'))
if (!ggufBinFile) {
throw new Error('No GGUF model file found')
}
const modelBinaryPath = join(modelFolderFullPath, ggufBinFile)
const nitroResourceProbe = await getSystemResourceInfo()
const nitroModelSettings: NitroModelSettings = {
// This is critical and requires real CPU physical core count (or performance core)
cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore),
...modelMetadata.settings,
...settingParams,
llama_model_path: modelBinaryPath,
...(modelMetadata.settings.mmproj && {
mmproj: join(modelFolderFullPath, modelMetadata.settings.mmproj),
}),
}
log(`[SERVER]::Debug: Nitro model settings: ${JSON.stringify(nitroModelSettings)}`)
// Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt
if (modelMetadata.settings.prompt_template) {
const promptTemplate = modelMetadata.settings.prompt_template
const prompt = promptTemplateConverter(promptTemplate)
if (prompt?.error) {
throw new Error(prompt.error)
}
nitroModelSettings.system_prompt = prompt.system_prompt
nitroModelSettings.user_prompt = prompt.user_prompt
nitroModelSettings.ai_prompt = prompt.ai_prompt
}
await runNitroAndLoadModel(modelId, nitroModelSettings)
}
// TODO: move to util
const promptTemplateConverter = (promptTemplate: string): PromptTemplate => {
// Split the string using the markers
const systemMarker = '{system_message}'
const promptMarker = '{prompt}'
if (promptTemplate.includes(systemMarker) && promptTemplate.includes(promptMarker)) {
// Find the indices of the markers
const systemIndex = promptTemplate.indexOf(systemMarker)
const promptIndex = promptTemplate.indexOf(promptMarker)
// Extract the parts of the string
const system_prompt = promptTemplate.substring(0, systemIndex)
const user_prompt = promptTemplate.substring(systemIndex + systemMarker.length, promptIndex)
const ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length)
// Return the split parts
return { system_prompt, user_prompt, ai_prompt }
} else if (promptTemplate.includes(promptMarker)) {
// Extract the parts of the string for the case where only promptMarker is present
const promptIndex = promptTemplate.indexOf(promptMarker)
const user_prompt = promptTemplate.substring(0, promptIndex)
const ai_prompt = promptTemplate.substring(promptIndex + promptMarker.length)
// Return the split parts
return { user_prompt, ai_prompt }
}
// Return an error if none of the conditions are met
return { error: 'Cannot split prompt template' }
}
const runNitroAndLoadModel = async (modelId: string, modelSettings: NitroModelSettings) => {
// Gather system information for CPU physical cores and memory
const tcpPortUsed = require('tcp-port-used')
await stopModel(modelId)
await tcpPortUsed.waitUntilFree(NITRO_DEFAULT_PORT, 300, 5000)
/**
* There is a problem with Windows process manager
* Should wait for awhile to make sure the port is free and subprocess is killed
* The tested threshold is 500ms
**/
if (process.platform === 'win32') {
await new Promise((resolve) => setTimeout(resolve, 500))
}
await spawnNitroProcess()
await loadLLMModel(modelSettings)
await validateModelStatus()
}
const spawnNitroProcess = async (): Promise<void> => {
log(`[SERVER]::Debug: Spawning cortex subprocess...`)
let binaryFolder = join(
getJanExtensionsPath(),
'@janhq',
'inference-cortex-extension',
'dist',
'bin'
)
let executableOptions = executableNitroFile()
const tcpPortUsed = require('tcp-port-used')
const args: string[] = ['1', LOCAL_HOST, NITRO_DEFAULT_PORT.toString()]
// Execute the binary
log(
`[SERVER]::Debug: Spawn cortex at path: ${executableOptions.executablePath}, and args: ${args}`
)
subprocess = spawn(
executableOptions.executablePath,
['1', LOCAL_HOST, NITRO_DEFAULT_PORT.toString()],
{
cwd: binaryFolder,
env: {
...process.env,
CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices,
},
}
)
// Handle subprocess output
subprocess.stdout.on('data', (data: any) => {
log(`[SERVER]::Debug: ${data}`)
})
subprocess.stderr.on('data', (data: any) => {
log(`[SERVER]::Error: ${data}`)
})
subprocess.on('close', (code: any) => {
log(`[SERVER]::Debug: cortex exited with code: ${code}`)
subprocess = undefined
})
tcpPortUsed.waitUntilUsed(NITRO_DEFAULT_PORT, 300, 30000).then(() => {
log(`[SERVER]::Debug: cortex is ready`)
})
}
type NitroExecutableOptions = {
executablePath: string
cudaVisibleDevices: string
}
const executableNitroFile = (): NitroExecutableOptions => {
const nvidiaInfoFilePath = join(getJanDataFolderPath(), 'settings', 'settings.json')
let binaryFolder = join(
getJanExtensionsPath(),
'@janhq',
'inference-cortex-extension',
'dist',
'bin'
)
let cudaVisibleDevices = ''
let binaryName = 'cortex-cpp'
/**
* The binary folder is different for each platform.
*/
if (process.platform === 'win32') {
/**
* For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0
*/
let nvidiaInfo = JSON.parse(fs.readFileSync(nvidiaInfoFilePath, 'utf-8'))
if (nvidiaInfo['run_mode'] === 'cpu') {
binaryFolder = join(binaryFolder, 'win-cpu')
} else {
if (nvidiaInfo['cuda'].version === '12') {
binaryFolder = join(binaryFolder, 'win-cuda-12-0')
} else {
binaryFolder = join(binaryFolder, 'win-cuda-11-7')
}
cudaVisibleDevices = nvidiaInfo['gpu_highest_vram']
}
binaryName = 'cortex-cpp.exe'
} else if (process.platform === 'darwin') {
/**
* For MacOS: mac-universal both Silicon and InteL
*/
if(process.arch === 'arm64') {
binaryFolder = join(binaryFolder, 'mac-arm64')
} else {
binaryFolder = join(binaryFolder, 'mac-amd64')
}
} else {
/**
* For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0
*/
let nvidiaInfo = JSON.parse(fs.readFileSync(nvidiaInfoFilePath, 'utf-8'))
if (nvidiaInfo['run_mode'] === 'cpu') {
binaryFolder = join(binaryFolder, 'linux-cpu')
} else {
if (nvidiaInfo['cuda'].version === '12') {
binaryFolder = join(binaryFolder, 'linux-cuda-12-0')
} else {
binaryFolder = join(binaryFolder, 'linux-cuda-11-7')
}
cudaVisibleDevices = nvidiaInfo['gpu_highest_vram']
}
}
return {
executablePath: join(binaryFolder, binaryName),
cudaVisibleDevices,
}
}
const validateModelStatus = async (): Promise<void> => {
// Send a GET request to the validation URL.
// Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries.
const fetchRT = require('fetch-retry')
const fetchRetry = fetchRT(fetch)
return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
},
retries: 5,
retryDelay: 500,
}).then(async (res: Response) => {
log(`[SERVER]::Debug: Validate model state success with response ${JSON.stringify(res)}`)
// If the response is OK, check model_loaded status.
if (res.ok) {
const body = await res.json()
// If the model is loaded, return an empty object.
// Otherwise, return an object with an error message.
if (body.model_loaded) {
return Promise.resolve()
}
}
return Promise.reject('Validate model status failed')
})
}
const loadLLMModel = async (settings: NitroModelSettings): Promise<Response> => {
log(`[SERVER]::Debug: Loading model with params ${JSON.stringify(settings)}`)
const fetchRT = require('fetch-retry')
const fetchRetry = fetchRT(fetch)
return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(settings),
retries: 3,
retryDelay: 500,
})
.then((res: any) => {
log(`[SERVER]::Debug: Load model request with response ${JSON.stringify(res)}`)
return Promise.resolve(res)
})
.catch((err: any) => {
log(`[SERVER]::Error: Load model failed with error ${err}`)
return Promise.reject(err)
})
}
/**
* Stop model and kill nitro process.
*/
export const stopModel = async (_modelId: string) => {
if (!subprocess) {
return {
error: "Model isn't running",
}
}
return new Promise((resolve, reject) => {
const controller = new AbortController()
setTimeout(() => {
controller.abort()
reject({
error: 'Failed to stop model: Timedout',
})
}, 5000)
const tcpPortUsed = require('tcp-port-used')
log(`[SERVER]::Debug: Request to kill cortex`)
fetch(NITRO_HTTP_KILL_URL, {
method: 'DELETE',
signal: controller.signal,
})
.then(() => {
subprocess?.kill()
subprocess = undefined
})
.catch(() => {
// don't need to do anything, we still kill the subprocess
})
.then(() => tcpPortUsed.waitUntilFree(NITRO_DEFAULT_PORT, 300, 5000))
.then(() => log(`[SERVER]::Debug: Nitro process is terminated`))
.then(() =>
resolve({
message: 'Model stopped',
})
)
})
}

16
core/src/node/api/restful/v1.ts Normal file
View File

@ -0,0 +1,16 @@
import { HttpServer } from '../HttpServer'
import { commonRouter } from './common'
import { downloadRouter } from './app/download'
import { handleRequests } from './app/handlers'
export const v1Router = async (app: HttpServer) => {
// MARK: Public API Routes
app.register(commonRouter)
// MARK: Internal Application Routes
handleRequests(app)
// Expanded route for tracking download progress
// TODO: Replace by Observer Wrapper (ZeroMQ / Vanilla Websocket)
app.register(downloadRouter)
}

203
core/src/node/extension/extension.ts Normal file
View File

@ -0,0 +1,203 @@
import { rmdirSync } from 'fs'
import { resolve, join } from 'path'
import { ExtensionManager } from './manager'
/**
* An NPM package that can be used as an extension.
* Used to hold all the information and functions necessary to handle the extension lifecycle.
*/
export default class Extension {
/**
* @property {string} origin Original specification provided to fetch the package.
* @property {Object} installOptions Options provided to pacote when fetching the manifest.
* @property {name} name The name of the extension as defined in the manifest.
* @property {name} productName The display name of the extension as defined in the manifest.
* @property {string} url Electron URL where the package can be accessed.
* @property {string} version Version of the package as defined in the manifest.
* @property {string} main The entry point as defined in the main entry of the manifest.
* @property {string} description The description of extension as defined in the manifest.
*/
origin?: string
installOptions: any
name?: string
productName?: string
url?: string
version?: string
main?: string
description?: string
/** @private */
_active = false
/**
* @private
* @property {Object.<string, Function>} #listeners A list of callbacks to be executed when the Extension is updated.
*/
listeners: Record<string, (obj: any) => void> = {}
/**
* Set installOptions with defaults for options that have not been provided.
* @param {string} [origin] Original specification provided to fetch the package.
* @param {Object} [options] Options provided to pacote when fetching the manifest.
*/
constructor(origin?: string, options = {}) {
const Arborist = require('@npmcli/arborist')
const defaultOpts = {
version: false,
fullMetadata: true,
Arborist,
}
this.origin = origin
this.installOptions = { ...defaultOpts, ...options }
}
/**
* Package name with version number.
* @type {string}
*/
get specifier() {
return this.origin + (this.installOptions.version ? '@' + this.installOptions.version : '')
}
/**
* Whether the extension should be registered with its activation points.
* @type {boolean}
*/
get active() {
return this._active
}
/**
* Set Package details based on it's manifest
* @returns {Promise.<Boolean>} Resolves to true when the action completed
*/
async getManifest() {
// Get the package's manifest (package.json object)
try {
await import('pacote').then((pacote) => {
return pacote.manifest(this.specifier, this.installOptions).then((mnf) => {
// set the Package properties based on the it's manifest
this.name = mnf.name
this.productName = mnf.productName as string | undefined
this.version = mnf.version
this.main = mnf.main
this.description = mnf.description
})
})
} catch (error) {
throw new Error(`Package ${this.origin} does not contain a valid manifest: ${error}`)
}
return true
}
/**
* Extract extension to extensions folder.
* @returns {Promise.<Extension>} This extension
* @private
*/
async _install() {
try {
// import the manifest details
await this.getManifest()
// Install the package in a child folder of the given folder
const pacote = await import('pacote')
await pacote.extract(
this.specifier,
join(ExtensionManager.instance.getExtensionsPath() ?? '', this.name ?? ''),
this.installOptions
)
// Set the url using the custom extensions protocol
this.url = `extension://${this.name}/${this.main}`
this.emitUpdate()
} catch (err) {
// Ensure the extension is not stored and the folder is removed if the installation fails
this.setActive(false)
throw err
}
return [this]
}
/**
* Subscribe to updates of this extension
* @param {string} name name of the callback to register
* @param {callback} cb The function to execute on update
*/
subscribe(name: string, cb: () => void) {
this.listeners[name] = cb
}
/**
* Remove subscription
* @param {string} name name of the callback to remove
*/
unsubscribe(name: string) {
delete this.listeners[name]
}
/**
* Execute listeners
*/
emitUpdate() {
for (const cb in this.listeners) {
this.listeners[cb].call(null, this)
}
}
/**
* Check for updates and install if available.
* @param {string} version The version to update to.
* @returns {boolean} Whether an update was performed.
*/
async update(version = false) {
if (await this.isUpdateAvailable()) {
this.installOptions.version = version
await this._install()
return true
}
return false
}
/**
* Check if a new version of the extension is available at the origin.
* @returns the latest available version if a new version is available or false if not.
*/
async isUpdateAvailable() {
return import('pacote').then((pacote) => {
if (this.origin) {
return pacote.manifest(this.origin).then((mnf) => {
return mnf.version !== this.version ? mnf.version : false
})
}
})
}
/**
* Remove extension and refresh renderers.
* @returns {Promise}
*/
async uninstall(): Promise<void> {
const path = ExtensionManager.instance.getExtensionsPath()
const extPath = resolve(path ?? '', this.name ?? '')
rmdirSync(extPath, { recursive: true })
this.emitUpdate()
}
/**
* Set a extension's active state. This determines if a extension should be loaded on initialisation.
* @param {boolean} active State to set _active to
* @returns {Extension} This extension
*/
setActive(active: boolean) {
this._active = active
this.emitUpdate()
return this
}
}

136
core/src/node/extension/index.ts Normal file
View File

@ -0,0 +1,136 @@
import { readFileSync } from 'fs'
import { normalize } from 'path'
import Extension from './extension'
import {
getAllExtensions,
removeExtension,
persistExtensions,
installExtensions,
getExtension,
getActiveExtensions,
addExtension,
} from './store'
import { ExtensionManager } from './manager'
export function init(options: any) {
// Create extensions protocol to serve extensions to renderer
registerExtensionProtocol()
// perform full setup if extensionsPath is provided
if (options.extensionsPath) {
return useExtensions(options.extensionsPath)
}
return {}
}
/**
* Create extensions protocol to provide extensions to renderer
* @private
* @returns {boolean} Whether the protocol registration was successful
*/
async function registerExtensionProtocol() {
let electron: any = undefined
try {
const moduleName = 'electron'
electron = await import(moduleName)
} catch (err) {
console.error('Electron is not available')
}
const extensionPath = ExtensionManager.instance.getExtensionsPath()
if (electron && electron.protocol) {
return electron.protocol?.registerFileProtocol('extension', (request: any, callback: any) => {
const entry = request.url.substr('extension://'.length - 1)
const url = normalize(extensionPath + entry)
callback({ path: url })
})
}
}
/**
* Set extensions up to run from the extensionPath folder if it is provided and
* load extensions persisted in that folder.
* @param {string} extensionsPath Path to the extensions folder. Required if not yet set up.
* @returns {extensionManager} A set of functions used to manage the extension lifecycle.
*/
export function useExtensions(extensionsPath: string) {
if (!extensionsPath) throw Error('A path to the extensions folder is required to use extensions')
// Store the path to the extensions folder
ExtensionManager.instance.setExtensionsPath(extensionsPath)
// Remove any registered extensions
for (const extension of getAllExtensions()) {
if (extension.name) removeExtension(extension.name, false)
}
// Read extension list from extensions folder
const extensions = JSON.parse(
readFileSync(ExtensionManager.instance.getExtensionsFile(), 'utf-8')
)
try {
// Create and store a Extension instance for each extension in list
for (const p in extensions) {
loadExtension(extensions[p])
}
persistExtensions()
} catch (error) {
// Throw meaningful error if extension loading fails
throw new Error(
'Could not successfully rebuild list of installed extensions.\n' +
error +
'\nPlease check the extensions.json file in the extensions folder.'
)
}
// Return the extension lifecycle functions
return getStore()
}
/**
* Check the given extension object. If it is marked for uninstalling, the extension files are removed.
* Otherwise a Extension instance for the provided object is created and added to the store.
* @private
* @param {Object} ext Extension info
*/
function loadExtension(ext: any) {
// Create new extension, populate it with ext details and save it to the store
const extension = new Extension()
for (const key in ext) {
if (Object.prototype.hasOwnProperty.call(ext, key)) {
// Use Object.defineProperty to set the properties as writable
Object.defineProperty(extension, key, {
value: ext[key],
writable: true,
enumerable: true,
configurable: true,
})
}
}
addExtension(extension, false)
extension.subscribe('pe-persist', persistExtensions)
}
/**
* Returns the publicly available store functions.
* @returns {extensionManager} A set of functions used to manage the extension lifecycle.
*/
export function getStore() {
if (!ExtensionManager.instance.getExtensionsFile()) {
throw new Error(
'The extension path has not yet been set up. Please run useExtensions before accessing the store'
)
}
return {
installExtensions,
getExtension,
getAllExtensions,
getActiveExtensions,
removeExtension,
}
}

45
core/src/node/extension/manager.ts Normal file
View File

@ -0,0 +1,45 @@
import { join, resolve } from 'path'
import { existsSync, mkdirSync, writeFileSync } from 'fs'
/**
* Manages extension installation and migration.
*/
export class ExtensionManager {
public static instance: ExtensionManager = new ExtensionManager()
private extensionsPath: string | undefined
constructor() {
if (ExtensionManager.instance) {
return ExtensionManager.instance
}
}
getExtensionsPath(): string | undefined {
return this.extensionsPath
}
setExtensionsPath(extPath: string) {
// Create folder if it does not exist
let extDir
try {
extDir = resolve(extPath)
if (extDir.length < 2) throw new Error()
if (!existsSync(extDir)) mkdirSync(extDir)
const extensionsJson = join(extDir, 'extensions.json')
if (!existsSync(extensionsJson)) writeFileSync(extensionsJson, '{}')
this.extensionsPath = extDir
} catch (error) {
throw new Error('Invalid path provided to the extensions folder')
}
}
getExtensionsFile() {
return join(this.extensionsPath ?? '', 'extensions.json')
}
}

125
core/src/node/extension/store.ts Normal file
View File

@ -0,0 +1,125 @@
import { writeFileSync } from 'fs'
import Extension from './extension'
import { ExtensionManager } from './manager'
/**
* @module store
* @private
*/
/**
* Register of installed extensions
* @type {Object.<string, Extension>} extension - List of installed extensions
*/
const extensions: Record<string, Extension> = {}
/**
* Get a extension from the stored extensions.
* @param {string} name Name of the extension to retrieve
* @returns {Extension} Retrieved extension
* @alias extensionManager.getExtension
*/
export function getExtension(name: string) {
if (!Object.prototype.hasOwnProperty.call(extensions, name)) {
throw new Error(`Extension ${name} does not exist`)
}
return extensions[name]
}
/**
* Get list of all extension objects.
* @returns {Array.<Extension>} All extension objects
* @alias extensionManager.getAllExtensions
*/
export function getAllExtensions() {
return Object.values(extensions)
}
/**
* Get list of active extension objects.
* @returns {Array.<Extension>} Active extension objects
* @alias extensionManager.getActiveExtensions
*/
export function getActiveExtensions() {
return Object.values(extensions).filter((extension) => extension.active)
}
/**
* Remove extension from store and maybe save stored extensions to file
* @param {string} name Name of the extension to remove
* @param {boolean} persist Whether to save the changes to extensions to file
* @returns {boolean} Whether the delete was successful
* @alias extensionManager.removeExtension
*/
export function removeExtension(name: string, persist = true) {
const del = delete extensions[name]
if (persist) persistExtensions()
return del
}
/**
* Add extension to store and maybe save stored extensions to file
* @param {Extension} extension Extension to add to store
* @param {boolean} persist Whether to save the changes to extensions to file
* @returns {void}
*/
export function addExtension(extension: Extension, persist = true) {
if (extension.name) extensions[extension.name] = extension
if (persist) {
persistExtensions()
extension.subscribe('pe-persist', persistExtensions)
}
}
/**
* Save stored extensions to file
* @returns {void}
*/
export function persistExtensions() {
const persistData: Record<string, Extension> = {}
for (const name in extensions) {
persistData[name] = extensions[name]
}
writeFileSync(ExtensionManager.instance.getExtensionsFile(), JSON.stringify(persistData))
}
/**
* Create and install a new extension for the given specifier.
* @param {Array.<installOptions | string>} extensions A list of NPM specifiers, or installation configuration objects.
* @param {boolean} [store=true] Whether to store the installed extensions in the store
* @returns {Promise.<Array.<Extension>>} New extension
* @alias extensionManager.installExtensions
*/
export async function installExtensions(extensions: any) {
const installed: Extension[] = []
const installations = extensions.map((ext: any): Promise<void> => {
const isObject = typeof ext === 'object'
const spec = isObject ? [ext.specifier, ext] : [ext]
const activate = isObject ? ext.activate !== false : true
// Install and possibly activate extension
const extension = new Extension(...spec)
if (!extension.origin) {
return Promise.resolve()
}
return extension._install().then(() => {
if (activate) extension.setActive(true)
// Add extension to store if needed
addExtension(extension)
installed.push(extension)
})
})
await Promise.all(installations)
// Return list of all installed extensions
return installed
}
/**
* @typedef {Object.<string, any>} installOptions The {@link https://www.npmjs.com/package/pacote|pacote}
* options used to install the extension with some extra options.
* @param {string} specifier the NPM specifier that identifies the package.
* @param {boolean} [activate] Whether this extension should be activated after installation. Defaults to true.
*/

157
core/src/node/helper/config.ts Normal file
View File

@ -0,0 +1,157 @@
import { AppConfiguration, SettingComponentProps } from '../../types'
import { join } from 'path'
import fs from 'fs'
import os from 'os'
import childProcess from 'child_process'
const configurationFileName = 'settings.json'
// TODO: do no specify app name in framework module
// TODO: do not default the os.homedir
const defaultJanDataFolder = join(os?.homedir() || '', 'jan')
const defaultAppConfig: AppConfiguration = {
data_folder: defaultJanDataFolder,
quick_ask: false,
}
/**
* Getting App Configurations.
*
* @returns {AppConfiguration} The app configurations.
*/
export const getAppConfigurations = (): AppConfiguration => {
// Retrieve Application Support folder path
// Fallback to user home directory if not found
const configurationFile = getConfigurationFilePath()
if (!fs.existsSync(configurationFile)) {
// create default app config if we don't have one
console.debug(`App config not found, creating default config at ${configurationFile}`)
fs.writeFileSync(configurationFile, JSON.stringify(defaultAppConfig))
return defaultAppConfig
}
try {
const appConfigurations: AppConfiguration = JSON.parse(
fs.readFileSync(configurationFile, 'utf-8')
)
return appConfigurations
} catch (err) {
console.error(`Failed to read app config, return default config instead! Err: ${err}`)
return defaultAppConfig
}
}
const getConfigurationFilePath = () =>
join(
global.core?.appPath() || process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'],
configurationFileName
)
export const updateAppConfiguration = (configuration: AppConfiguration): Promise<void> => {
const configurationFile = getConfigurationFilePath()
console.debug('updateAppConfiguration, configurationFile: ', configurationFile)
fs.writeFileSync(configurationFile, JSON.stringify(configuration))
return Promise.resolve()
}
/**
* Utility function to get data folder path
*
* @returns {string} The data folder path.
*/
export const getJanDataFolderPath = (): string => {
const appConfigurations = getAppConfigurations()
return appConfigurations.data_folder
}
/**
* Utility function to get extension path
*
* @returns {string} The extensions path.
*/
export const getJanExtensionsPath = (): string => {
const appConfigurations = getAppConfigurations()
return join(appConfigurations.data_folder, 'extensions')
}
/**
* Utility function to physical cpu count
*
* @returns {number} The physical cpu count.
*/
export const physicalCpuCount = async (): Promise<number> => {
const platform = os.platform()
try {
if (platform === 'linux') {
const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l')
return parseInt(output.trim(), 10)
} else if (platform === 'darwin') {
const output = await exec('sysctl -n hw.physicalcpu_max')
return parseInt(output.trim(), 10)
} else if (platform === 'win32') {
const output = await exec('WMIC CPU Get NumberOfCores')
return output
.split(os.EOL)
.map((line: string) => parseInt(line))
.filter((value: number) => !isNaN(value))
.reduce((sum: number, number: number) => sum + number, 1)
} else {
const cores = os.cpus().filter((cpu: any, index: number) => {
const hasHyperthreading = cpu.model.includes('Intel')
const isOdd = index % 2 === 1
return !hasHyperthreading || isOdd
})
return cores.length
}
} catch (err) {
console.warn('Failed to get physical CPU count', err)
// Divide by 2 to get rid of hyper threading
const coreCount = Math.ceil(os.cpus().length / 2)
console.debug('Using node API to get physical CPU count:', coreCount)
return coreCount
}
}
const exec = async (command: string): Promise<string> => {
return new Promise((resolve, reject) => {
childProcess.exec(command, { encoding: 'utf8' }, (error, stdout) => {
if (error) {
reject(error)
} else {
resolve(stdout)
}
})
})
}
// a hacky way to get the api key. we should comes up with a better
// way to handle this
export const getEngineConfiguration = async (engineId: string) => {
if (engineId !== 'openai' && engineId !== 'groq') return undefined
const settingDirectoryPath = join(
getJanDataFolderPath(),
'settings',
'@janhq',
engineId === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension',
'settings.json'
)
const content = fs.readFileSync(settingDirectoryPath, 'utf-8')
const settings: SettingComponentProps[] = JSON.parse(content)
const apiKeyId = engineId === 'openai' ? 'openai-api-key' : 'groq-api-key'
const keySetting = settings.find((setting) => setting.key === apiKeyId)
let fullUrl = settings.find((setting) => setting.key === 'chat-completions-endpoint')
?.controllerProps.value
let apiKey = keySetting?.controllerProps.value
if (typeof apiKey !== 'string') apiKey = ''
if (typeof fullUrl !== 'string') fullUrl = ''
return {
api_key: apiKey,
full_url: fullUrl,
}
}

30
core/src/node/helper/download.ts Normal file
View File

@ -0,0 +1,30 @@
import { DownloadState } from '../../types'
/**
* Manages file downloads and network requests.
*/
export class DownloadManager {
public networkRequests: Record<string, any> = {}
public static instance: DownloadManager = new DownloadManager()
// store the download information with key is model id
public downloadProgressMap: Record<string, DownloadState> = {}
// store the download information with key is normalized file path
public downloadInfo: Record<string, DownloadState> = {}
constructor() {
if (DownloadManager.instance) {
return DownloadManager.instance
}
}
/**
* Sets a network request for a specific file.
* @param {string} fileName - The name of the file.
* @param {Request | undefined} request - The network request to set, or undefined to clear the request.
*/
setRequest(fileName: string, request: any | undefined) {
this.networkRequests[fileName] = request
}
}

6
core/src/node/helper/index.ts Normal file
View File

@ -0,0 +1,6 @@
export * from './config'
export * from './download'
export * from './logger'
export * from './module'
export * from './path'
export * from './resource'

81
core/src/node/helper/logger.ts Normal file
View File

@ -0,0 +1,81 @@
// Abstract Logger class that all loggers should extend.
export abstract class Logger {
// Each logger must have a unique name.
abstract name: string
/**
* Log message to log file.
* This method should be overridden by subclasses to provide specific logging behavior.
*/
abstract log(args: any): void
}
// LoggerManager is a singleton class that manages all registered loggers.
export class LoggerManager {
// Map of registered loggers, keyed by their names.
public loggers = new Map<string, Logger>()
// Array to store logs that are queued before the loggers are registered.
queuedLogs: any[] = []
// Flag to indicate whether flushLogs is currently running.
private isFlushing = false
// Register a new logger. If a logger with the same name already exists, it will be replaced.
register(logger: Logger) {
this.loggers.set(logger.name, logger)
}
// Unregister a logger by its name.
unregister(name: string) {
this.loggers.delete(name)
}
get(name: string) {
return this.loggers.get(name)
}
// Flush queued logs to all registered loggers.
flushLogs() {
// If flushLogs is already running, do nothing.
if (this.isFlushing) {
return
}
this.isFlushing = true
while (this.queuedLogs.length > 0 && this.loggers.size > 0) {
const log = this.queuedLogs.shift()
this.loggers.forEach((logger) => {
logger.log(log)
})
}
this.isFlushing = false
}
// Log message using all registered loggers.
log(args: any) {
this.queuedLogs.push(args)
this.flushLogs()
}
/**
* The instance of the logger.
* If an instance doesn't exist, it creates a new one.
* This ensures that there is only one LoggerManager instance at any time.
*/
static instance(): LoggerManager {
let instance: LoggerManager | undefined = global.core?.logger
if (!instance) {
instance = new LoggerManager()
if (!global.core) global.core = {}
global.core.logger = instance
}
return instance
}
}
export const log = (...args: any) => {
LoggerManager.instance().log(args)
}

31
core/src/node/helper/module.ts Normal file
View File

@ -0,0 +1,31 @@
/**
* Manages imported modules.
*/
export class ModuleManager {
public requiredModules: Record<string, any> = {}
public cleaningResource = false
public static instance: ModuleManager = new ModuleManager()
constructor() {
if (ModuleManager.instance) {
return ModuleManager.instance
}
}
/**
* Sets a module.
* @param {string} moduleName - The name of the module.
* @param {any | undefined} nodule - The module to set, or undefined to clear the module.
*/
setModule(moduleName: string, nodule: any | undefined) {
this.requiredModules[moduleName] = nodule
}
/**
* Clears all imported modules.
*/
clearImportedModules() {
this.requiredModules = {}
}
}

44
core/src/node/helper/path.ts Normal file
View File

@ -0,0 +1,44 @@
import { join, resolve } from 'path'
import { getJanDataFolderPath } from './config'
/**
* Normalize file path
* Remove all file protocol prefix
* @param path
* @returns
*/
export function normalizeFilePath(path: string): string {
return path.replace(/^(file:[\\/]+)([^:\s]+)$/, '$2')
}
export async function appResourcePath(): Promise<string> {
let electron: any = undefined
try {
const moduleName = 'electron'
electron = await import(moduleName)
} catch (err) {
console.error('Electron is not available')
}
// electron
if (electron && electron.protocol) {
let appPath = join(electron.app.getAppPath(), '..', 'app.asar.unpacked')
if (!electron.app.isPackaged) {
// for development mode
appPath = join(electron.app.getAppPath())
}
return appPath
}
// server
return join(global.core.appPath(), '../../..')
}
export function validatePath(path: string) {
const janDataFolderPath = getJanDataFolderPath()
const absolutePath = resolve(__dirname, path)
if (!absolutePath.startsWith(janDataFolderPath)) {
throw new Error(`Invalid path: ${absolutePath}`)
}
}

13
core/src/node/helper/resource.ts Normal file
View File

@ -0,0 +1,13 @@
import { SystemResourceInfo } from '../../types'
import { physicalCpuCount } from './config'
import { log } from './logger'
export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => {
const cpu = await physicalCpuCount()
log(`[CORTEX]::CPU information - ${cpu}`)
return {
numCpuPhysicalCore: cpu,
memAvailable: 0, // TODO: this should not be 0
}
}

8
core/src/node/index.ts Normal file
View File

@ -0,0 +1,8 @@
export * from './extension/index'
export * from './extension/extension'
export * from './extension/manager'
export * from './extension/store'
export * from './api'
export * from './helper'
export * from './../types'
export * from '../types/api'

127
core/src/types/api/index.ts
View File

@ -1,3 +1,5 @@
import { ChatCompletionMessage } from '../inference'
/**
* Native Route APIs
* @description Enum of all the routes exposed by the app
@ -25,19 +27,23 @@ export enum NativeRoute {
quickAskSizeUpdated = 'quickAskSizeUpdated',
ackDeepLink = 'ackDeepLink',
homePath = 'homePath',
getThemes = 'getThemes',
readTheme = 'readTheme',
}
// used for migration. Please remove this later on.
getAllMessagesAndThreads = 'getAllMessagesAndThreads',
getAllLocalModels = 'getAllLocalModels',
syncModelFileToCortex = 'syncModelFileToCortex',
openAppLog = 'openAppLog',
appDataFolder = 'appDataFolder',
changeDataFolder = 'changeDataFolder',
isDirectoryEmpty = 'isDirectoryEmpty',
/**
* App Route APIs
* @description Enum of all the routes exposed by the app
*/
export enum AppRoute {
getAppConfigurations = 'getAppConfigurations',
updateAppConfiguration = 'updateAppConfiguration',
joinPath = 'joinPath',
isSubdirectory = 'isSubdirectory',
baseName = 'baseName',
startServer = 'startServer',
stopServer = 'stopServer',
log = 'log',
systemInformation = 'systemInformation',
showToast = 'showToast',
}
export enum AppEvent {
@ -51,6 +57,22 @@ export enum AppEvent {
onDeepLink = 'onDeepLink',
}
export enum DownloadRoute {
abortDownload = 'abortDownload',
downloadFile = 'downloadFile',
pauseDownload = 'pauseDownload',
resumeDownload = 'resumeDownload',
getDownloadProgress = 'getDownloadProgress',
getFileSize = 'getFileSize',
}
export enum DownloadEvent {
onFileDownloadUpdate = 'onFileDownloadUpdate',
onFileDownloadError = 'onFileDownloadError',
onFileDownloadSuccess = 'onFileDownloadSuccess',
onFileUnzipSuccess = 'onFileUnzipSuccess',
}
export enum LocalImportModelEvent {
onLocalImportModelUpdate = 'onLocalImportModelUpdate',
onLocalImportModelFailed = 'onLocalImportModelFailed',
@ -58,17 +80,92 @@ export enum LocalImportModelEvent {
onLocalImportModelFinished = 'onLocalImportModelFinished',
}
export enum ExtensionRoute {
baseExtensions = 'baseExtensions',
getActiveExtensions = 'getActiveExtensions',
installExtension = 'installExtension',
invokeExtensionFunc = 'invokeExtensionFunc',
updateExtension = 'updateExtension',
uninstallExtension = 'uninstallExtension',
}
export enum FileSystemRoute {
appendFileSync = 'appendFileSync',
unlinkSync = 'unlinkSync',
existsSync = 'existsSync',
readdirSync = 'readdirSync',
rm = 'rm',
mkdir = 'mkdir',
readFileSync = 'readFileSync',
writeFileSync = 'writeFileSync',
}
export enum FileManagerRoute {
copyFile = 'copyFile',
getJanDataFolderPath = 'getJanDataFolderPath',
getResourcePath = 'getResourcePath',
getUserHomePath = 'getUserHomePath',
fileStat = 'fileStat',
writeBlob = 'writeBlob',
}
export type ApiFunction = (...args: any[]) => any
export type NativeRouteFunctions = {
[K in NativeRoute]: ApiFunction
}
export type AppRouteFunctions = {
[K in AppRoute]: ApiFunction
}
export type AppEventFunctions = {
[K in AppEvent]: ApiFunction
}
export type APIFunctions = NativeRouteFunctions & AppEventFunctions
export type DownloadRouteFunctions = {
[K in DownloadRoute]: ApiFunction
}
export const APIRoutes = [...Object.values(NativeRoute)]
export const APIEvents = [...Object.values(AppEvent), ...Object.values(LocalImportModelEvent)]
export type DownloadEventFunctions = {
[K in DownloadEvent]: ApiFunction
}
export type ExtensionRouteFunctions = {
[K in ExtensionRoute]: ApiFunction
}
export type FileSystemRouteFunctions = {
[K in FileSystemRoute]: ApiFunction
}
export type FileManagerRouteFunctions = {
[K in FileManagerRoute]: ApiFunction
}
export type APIFunctions = NativeRouteFunctions &
AppRouteFunctions &
AppEventFunctions &
DownloadRouteFunctions &
DownloadEventFunctions &
ExtensionRouteFunctions &
FileSystemRouteFunctions &
FileManagerRoute
export const CoreRoutes = [
...Object.values(AppRoute),
...Object.values(DownloadRoute),
...Object.values(ExtensionRoute),
...Object.values(FileSystemRoute),
...Object.values(FileManagerRoute),
]
export const APIRoutes = [...CoreRoutes, ...Object.values(NativeRoute)]
export const APIEvents = [
...Object.values(AppEvent),
...Object.values(DownloadEvent),
...Object.values(LocalImportModelEvent),
]
export type PayloadType = {
messages: ChatCompletionMessage[]
model: string
stream: boolean
}

57
core/src/types/assistant/assistantEntity.ts
View File

@ -1,27 +1,38 @@
import {
AssistantTool as OpenAiAssistantTool,
Assistant as OpenAiAssistant,
AssistantCreateParams as OpenAiAssistantCreateParams,
AssistantUpdateParams as OpenAiAssistantUpdateParams,
} from 'openai/resources/beta/assistants'
import { AssistantResponseFormatOption as OpenAIAssistantResponseFormatOption } from 'openai/resources/beta/threads/threads'
export interface Assistant extends OpenAiAssistant {
avatar?: string
tools: AssistantTool[]
}
export type AssistantResponseFormatOption = OpenAIAssistantResponseFormatOption
export interface AssistantToolResources extends OpenAiAssistant.ToolResources {}
export type AssistantTool = OpenAiAssistantTool & {
enabled?: boolean
/**
* Assistant type defines the shape of an assistant object.
* @stored
*/
export type AssistantTool = {
type: string
enabled: boolean
useTimeWeightedRetriever?: boolean
settings: any
}
export interface AssistantCreateParams extends OpenAiAssistantCreateParams {}
export interface AssistantUpdateParams extends OpenAiAssistantUpdateParams {}
export type Assistant = {
/** Represents the avatar of the user. */
avatar: string
/** Represents the location of the thread. */
thread_location: string | undefined
/** Represents the unique identifier of the object. */
id: string
/** Represents the object. */
object: string
/** Represents the creation timestamp of the object. */
created_at: number
/** Represents the name of the object. */
name: string
/** Represents the description of the object. */
description?: string
/** Represents the model of the object. */
model: string
/** Represents the instructions for the object. */
instructions?: string
/** Represents the tools associated with the object. */
tools?: AssistantTool[]
/** Represents the file identifiers associated with the object. */
file_ids: string[]
/** Represents the metadata of the object. */
metadata?: Record<string, unknown>
}

7
core/src/types/assistant/assistantEvent.ts Normal file
View File

@ -0,0 +1,7 @@
/**
* The `EventName` enumeration contains the names of all the available events in the Jan platform.
*/
export enum AssistantEvent {
/** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */
OnAssistantsUpdate = 'OnAssistantsUpdate',
}

26
core/src/types/assistant/assistantInterface.ts Normal file
View File

@ -0,0 +1,26 @@
import { Assistant } from './assistantEntity'
/**
* Assistant extension for managing assistants.
* @extends BaseExtension
*/
export interface AssistantInterface {
/**
* Creates a new assistant.
* @param {Assistant} assistant - The assistant object to be created.
* @returns {Promise<void>} A promise that resolves when the assistant has been created.
*/
createAssistant(assistant: Assistant): Promise<void>
/**
* Deletes an existing assistant.
* @param {Assistant} assistant - The assistant object to be deleted.
* @returns {Promise<void>} A promise that resolves when the assistant has been deleted.
*/
deleteAssistant(assistant: Assistant): Promise<void>
/**
* Retrieves all existing assistants.
* @returns {Promise<Assistant[]>} A promise that resolves to an array of all assistants.
*/
getAssistants(): Promise<Assistant[]>
}

2
core/src/types/assistant/index.ts
View File

@ -1 +1,3 @@
export * from './assistantEntity'
export * from './assistantEvent'
export * from './assistantInterface'

8
core/src/types/config/appConfigEntity.ts
View File

@ -1,8 +1,4 @@
export type AppConfiguration = {
dataFolderPath: string,
quickAsk: boolean,
cortexCppHost: string,
cortexCppPort: number,
apiServerHost: string,
apiServerPort: number,
data_folder: string
quick_ask: boolean
}

2
core/src/types/events/index.ts
View File

@ -1,2 +0,0 @@
export * from './model.event'
export * from './resource.event'

40
core/src/types/events/model.event.ts
View File

@ -1,40 +0,0 @@
export type ModelId = string
const ModelLoadingEvents = [
'starting',
'stopping',
'started',
'stopped',
'starting-failed',
'stopping-failed',
'model-downloaded',
'model-deleted',
] as const
export type ModelLoadingEvent = (typeof ModelLoadingEvents)[number]
const AllModelStates = ['starting', 'stopping', 'started'] as const
export type ModelState = (typeof AllModelStates)[number]
// TODO: should make this model -> id
export interface ModelStatus {
model: ModelId
status: ModelState
metadata: Record<string, unknown>
}
export interface ModelEvent {
model: ModelId
event: ModelLoadingEvent
metadata: Record<string, unknown>
}
export const EmptyModelEvent = {}
export type StatusAndEvent = {
status: Record<ModelId, ModelStatus>
event: ModelEvent | typeof EmptyModelEvent
}
export interface ModelStatusAndEvent {
data: StatusAndEvent
}

21
core/src/types/events/resource.event.ts
View File

@ -1,21 +0,0 @@
export interface ResourceEvent {
data: ResourceStatus
}
export interface ResourceStatus {
mem: UsedMemInfo
cpu: {
usage: number
}
gpus: GpuInfo[]
}
export interface UsedMemInfo {
total: number
used: number
}
export interface GpuInfo {
name: string | undefined
vram: UsedMemInfo
}

79
core/src/types/file/index.ts
View File

@ -52,82 +52,3 @@ type DownloadSize = {
total: number
transferred: number
}
export interface DownloadState2 {
/**
* The id of a particular download. Being used to prevent duplication of downloads.
*/
id: string
/**
* For displaying purposes.
*/
title: string
/**
* The type of download.
*/
type: DownloadType2
/**
* Percentage of the download.
*/
progress: number
/**
* The status of the download.
*/
status: DownloadStatus
/**
* Explanation of the error if the download failed.
*/
error?: string
/**
* The actual downloads. [DownloadState] is just a group to supporting for download multiple files.
*/
children: DownloadItem[]
}
export enum DownloadStatus {
Pending = 'pending',
Downloading = 'downloading',
Error = 'error',
Downloaded = 'downloaded',
}
export interface DownloadItem {
/**
* Filename of the download.
*/
id: string
time: {
elapsed: number
remaining: number
}
size: {
total: number
transferred: number
}
checksum?: string
status: DownloadStatus
error?: string
metadata?: Record<string, unknown>
}
export interface DownloadStateEvent {
data: DownloadState[]
}
export enum DownloadType2 {
Model = 'model',
Miscelanous = 'miscelanous',
Engine = 'engine',
}

13
core/src/types/huggingface/huggingfaceEntity.ts
View File

@ -40,11 +40,6 @@ export type CardDataKeysTuple = typeof CardDataKeys
export type CardDataKeys = CardDataKeysTuple[number]
export const AllQuantizations = [
'IQ1_M',
'IQ1_S',
'IQ3_S',
'Q3_K_XL',
'IQ4_NL',
'Q3_K_S',
'Q3_K_M',
'Q3_K_L',
@ -56,16 +51,8 @@ export const AllQuantizations = [
'Q4_1',
'Q5_0',
'Q5_1',
'Q5_K_L',
'Q4_K_L',
'IQ2_XXS',
'IQ2_XS',
'IQ2_S',
'IQ2_M',
'IQ3_M',
'IQ3_XS',
'IQ3_XXS',
'IQ4_XS',
'Q2_K',
'Q2_K_S',
'Q6_K',

2
core/src/types/index.ts
View File

@ -2,6 +2,7 @@ export * from './assistant'
export * from './model'
export * from './thread'
export * from './message'
export * from './inference'
export * from './monitoring'
export * from './file'
export * from './config'
@ -9,4 +10,3 @@ export * from './huggingface'
export * from './miscellaneous'
export * from './api'
export * from './setting'
export * from './events'

3
core/src/types/inference/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * from './inferenceEntity'
export * from './inferenceInterface'
export * from './inferenceEvent'

46
core/src/types/inference/inferenceEntity.ts Normal file
View File

@ -0,0 +1,46 @@
import { ContentType, ContentValue } from '../message'
/**
* The role of the author of this message.
*/
export enum ChatCompletionRole {
System = 'system',
Assistant = 'assistant',
User = 'user',
}
/**
* The `MessageRequest` type defines the shape of a new message request object.
* @data_transfer_object
*/
export type ChatCompletionMessage = {
/** The contents of the message. **/
content?: ChatCompletionMessageContent
/** The role of the author of this message. **/
role: ChatCompletionRole
}
export type ChatCompletionMessageContent =
| string
| (ChatCompletionMessageContentText &
ChatCompletionMessageContentImage &
ChatCompletionMessageContentDoc)[]
export enum ChatCompletionMessageContentType {
Text = 'text',
Image = 'image_url',
Doc = 'doc_url',
}
export type ChatCompletionMessageContentText = {
type: ChatCompletionMessageContentType
text: string
}
export type ChatCompletionMessageContentImage = {
type: ChatCompletionMessageContentType
image_url: { url: string }
}
export type ChatCompletionMessageContentDoc = {
type: ChatCompletionMessageContentType
doc_url: { url: string }
}

7
core/src/types/inference/inferenceEvent.ts Normal file
View File

@ -0,0 +1,7 @@
/**
* The `EventName` enumeration contains the names of all the available events in the Jan platform.
*/
export enum InferenceEvent {
/** The `OnInferenceStopped` event is emitted when a inference is stopped. */
OnInferenceStopped = 'OnInferenceStopped',
}

13
core/src/types/inference/inferenceInterface.ts Normal file
View File

@ -0,0 +1,13 @@
import { MessageRequest, ThreadMessage } from '../message'
/**
* Inference extension. Start, stop and inference models.
*/
export interface InferenceInterface {
/**
* Processes an inference request.
* @param data - The data for the inference request.
* @returns The result of the inference request.
*/
inference(data: MessageRequest): Promise<ThreadMessage>
}

3
core/src/types/message/index.ts
View File

@ -1 +1,4 @@
export * from './messageEntity'
export * from './messageInterface'
export * from './messageEvent'
export * from './messageRequestType'

132
core/src/types/message/messageEntity.ts
View File

@ -1,26 +1,122 @@
import {
ChatCompletionMessageParam as OpenAiChatCompletionMessageParam,
ChatCompletionMessage as OpenAiChatCompletionMessage,
} from 'openai/resources'
import {
MessageCreateParams as OpenAiMessageCreateParams,
Message as OpenAiMessage,
MessageContent as OpenAiMessageContent,
TextContentBlock as OpenAiTextContentBlock,
} from 'openai/resources/beta/threads/messages'
import { ChatCompletionMessage, ChatCompletionRole } from '../inference'
import { ModelInfo } from '../model'
import { Thread } from '../thread'
export interface Message extends OpenAiMessage {}
/**
* The `ThreadMessage` type defines the shape of a thread's message object.
* @stored
*/
export type ThreadMessage = {
/** Unique identifier for the message, generated by default using the ULID method. **/
id: string
/** Object name **/
object: string
/** Thread id, default is a ulid. **/
thread_id: string
/** The assistant id of this thread. **/
assistant_id?: string
/** The role of the author of this message. **/
role: ChatCompletionRole
/** The content of this message. **/
content: ThreadContent[]
/** The status of this message. **/
status: MessageStatus
/** The timestamp indicating when this message was created. Represented in Unix time. **/
created: number
/** The timestamp indicating when this message was updated. Represented in Unix time. **/
updated: number
/** The additional metadata of this message. **/
metadata?: Record<string, unknown>
export type MessageContent = OpenAiMessageContent
type?: string
export type TextContentBlock = OpenAiTextContentBlock
/** The error code which explain what error type. Used in conjunction with MessageStatus.Error */
error_code?: ErrorCode
}
export interface MessageIncompleteDetails extends OpenAiMessage.IncompleteDetails {}
/**
* The `MessageRequest` type defines the shape of a new message request object.
* @data_transfer_object
*/
export type MessageRequest = {
id?: string
export interface MessageAttachment extends OpenAiMessage.Attachment {}
/**
* @deprecated Use thread object instead
* The thread id of the message request.
*/
threadId: string
export interface ChatCompletionMessage extends OpenAiChatCompletionMessage {}
/**
* The assistant id of the message request.
*/
assistantId?: string
export type ChatCompletionMessageParam = OpenAiChatCompletionMessageParam
/** Messages for constructing a chat completion request **/
messages?: ChatCompletionMessage[]
export interface MessageCreateParams extends OpenAiMessageCreateParams {}
/** Settings for constructing a chat completion request **/
model?: ModelInfo
/** The thread of this message is belong to. **/
// TODO: deprecate threadId field
thread?: Thread
type?: string
}
/**
* The status of the message.
* @data_transfer_object
*/
export enum MessageStatus {
/** Message is fully loaded. **/
Ready = 'ready',
/** Message is not fully loaded. **/
Pending = 'pending',
/** Message loaded with error. **/
Error = 'error',
/** Message is cancelled streaming */
Stopped = 'stopped',
}
export enum ErrorCode {
InvalidApiKey = 'invalid_api_key',
AuthenticationError = 'authentication_error',
InsufficientQuota = 'insufficient_quota',
InvalidRequestError = 'invalid_request_error',
Unknown = 'unknown',
}
/**
* The content type of the message.
*/
export enum ContentType {
Text = 'text',
Image = 'image',
Pdf = 'pdf',
}
/**
* The `ContentValue` type defines the shape of a content value object
* @data_transfer_object
*/
export type ContentValue = {
value: string
annotations: string[]
name?: string
size?: number
}
/**
* The `ThreadContent` type defines the shape of a message's content object
* @data_transfer_object
*/
export type ThreadContent = {
type: ContentType
text: ContentValue
}

8
core/src/types/message/messageEvent.ts Normal file
View File

@ -0,0 +1,8 @@
export enum MessageEvent {
/** The `OnMessageSent` event is emitted when a message is sent. */
OnMessageSent = 'OnMessageSent',
/** The `OnMessageResponse` event is emitted when a message is received. */
OnMessageResponse = 'OnMessageResponse',
/** The `OnMessageUpdate` event is emitted when a message is updated. */
OnMessageUpdate = 'OnMessageUpdate',
}

30
core/src/types/message/messageInterface.ts Normal file
View File

@ -0,0 +1,30 @@
import { ThreadMessage } from './messageEntity'
/**
* Conversational extension. Persists and retrieves conversations.
* @abstract
* @extends BaseExtension
*/
export interface MessageInterface {
/**
* Adds a new message to the thread.
* @param {ThreadMessage} message - The message to be added.
* @returns {Promise<void>} A promise that resolves when the message has been added.
*/
addNewMessage(message: ThreadMessage): Promise<void>
/**
* Writes an array of messages to a specific thread.
* @param {string} threadId - The ID of the thread to write the messages to.
* @param {ThreadMessage[]} messages - The array of messages to be written.
* @returns {Promise<void>} A promise that resolves when the messages have been written.
*/
writeMessages(threadId: string, messages: ThreadMessage[]): Promise<void>
/**
* Retrieves all messages from a specific thread.
* @param {string} threadId - The ID of the thread to retrieve the messages from.
* @returns {Promise<ThreadMessage[]>} A promise that resolves to an array of messages from the thread.
*/
getAllMessages(threadId: string): Promise<ThreadMessage[]>
}

5
core/src/types/message/messageRequestType.ts Normal file
View File

@ -0,0 +1,5 @@
export enum MessageRequestType {
Thread = 'Thread',
Assistant = 'Assistant',
Summary = 'Summary',
}

10
core/src/types/model/chatCompletion.ts
View File

@ -1,10 +0,0 @@
import {
ChatCompletionCreateParamsNonStreaming as OpenAiChatCompletionCreateParamsNonStreaming,
ChatCompletionCreateParamsStreaming as OpenAiChatCompletionCreateParamsStreaming,
} from 'openai/resources/chat/completions'
export interface ChatCompletionCreateParamsNonStreaming
extends OpenAiChatCompletionCreateParamsNonStreaming {}
export interface ChatCompletionCreateParamsStreaming
extends OpenAiChatCompletionCreateParamsStreaming {}

3
core/src/types/model/index.ts
View File

@ -1,3 +1,4 @@
export * from './modelEntity'
export * from './modelInterface'
export * from './modelEvent'
export * from './modelImport'
export * from './chatCompletion'

275
core/src/types/model/modelEntity.ts
View File

@ -1,97 +1,119 @@
import { Model as OpenAiModel } from 'openai/resources'
export const LocalEngines = ['cortex.llamacpp', 'cortex.onnx', 'cortex.tensorrt-llm'] as const
export const RemoteEngines = [
'anthropic',
'mistral',
'martian',
'openrouter',
'openai',
'groq',
'triton_trtllm',
'cohere',
'nvidia',
] as const
export const LlmEngines = [...LocalEngines, ...RemoteEngines] as const
export type LlmEngine = (typeof LlmEngines)[number]
export type LocalEngine = (typeof LocalEngines)[number]
export type RemoteEngine = (typeof RemoteEngines)[number]
/**
* Represents the information about a model.
* @stored
*/
export type ModelInfo = {
id: string
settings: ModelSettingParams
parameters: ModelRuntimeParams
engine?: InferenceEngine
}
/**
* The available engine statuses.
* Represents the inference engine.
* @stored
*/
export enum EngineStatus {
Ready = 'ready',
MissingConfiguration = 'missing_configuration',
NotInitialized = 'not_initialized',
NotSupported = 'not_supported',
Error = 'error',
}
export enum InferenceEngine {
anthropic = 'anthropic',
mistral = 'mistral',
martian = 'martian',
openrouter = 'openrouter',
nitro = 'nitro',
openai = 'openai',
groq = 'groq',
triton_trtllm = 'triton_trtllm',
nitro_tensorrt_llm = 'nitro-tensorrt-llm',
cohere = 'cohere',
}
export type ModelArtifact = {
filename: string
url: string
}
export interface Model extends OpenAiModel, ModelSettingParams, ModelRuntimeParams {
/**
* Model type defines the shape of a model object.
* @stored
*/
export type Model = {
/**
* Model identifier.
* The type of the object.
* Default: "model"
*/
model: string
object: string
/**
* GGUF metadata: general.name
* The version of the model.
*/
name?: string
version: string
/**
* GGUF metadata: version
* The format of the model.
*/
version?: string
/**
* Currently we only have 'embedding' | 'llm'
*/
model_type?: string
format: string
/**
* The model download source. It can be an external url or a local filepath.
*/
files: string[] | ModelArtifact
sources: ModelArtifact[]
metadata?: Record<string, any>
/**
* The model identifier, which can be referenced in the API endpoints.
*/
id: string
/**
* Human-readable name that is used for UI.
*/
name: string
/**
* The Unix timestamp (in seconds) for when the model was created
*/
created: number
/**
* Default: "A cool model from Huggingface"
*/
description: string
/**
* The model settings.
*/
settings: ModelSettingParams
/**
* The model runtime parameters.
*/
parameters: ModelRuntimeParams
/**
* Metadata of the model.
*/
metadata: ModelMetadata
/**
* The model engine.
*/
engine: InferenceEngine
}
export type ModelMetadata = {
author: string
tags: string[]
size: number
cover?: string
}
/**
* The available model settings.
*/
export interface ModelSettingParams {
/**
* The context length for model operations varies; the maximum depends on the specific model used.
*/
export type ModelSettingParams = {
ctx_len?: number
/**
* The number of layers to load onto the GPU for acceleration.
*/
ngl?: number
embedding?: boolean
/**
* Number of parallel sequences to decode
*/
n_parallel?: number
/**
* Determines CPU inference threads, limited by hardware and OS. (Maximum determined by system)
*/
cpu_threads?: number
/**
* GGUF metadata: tokenizer.chat_template
*/
prompt_template?: string
system_prompt?: string
ai_prompt?: string
@ -99,139 +121,26 @@ export interface ModelSettingParams {
llama_model_path?: string
mmproj?: string
cont_batching?: boolean
/**
* The model engine.
*/
engine?: LlmEngine
/**
* The prompt to use for internal configuration
*/
pre_prompt?: string
/**
* The batch size for prompt eval step
*/
n_batch?: number
/**
* To enable prompt caching or not
*/
caching_enabled?: boolean
/**
* Group attention factor in self-extend
*/
grp_attn_n?: number
/**
* Group attention width in self-extend
*/
grp_attn_w?: number
/**
* Prevent system swapping of the model to disk in macOS
*/
mlock?: boolean
/**
* You can constrain the sampling using GBNF grammars by providing path to a grammar file
*/
grammar_file?: string
/**
* To enable Flash Attention, default is true
*/
flash_attn?: boolean
/**
* KV cache type: f16, q8_0, q4_0, default is f16
*/
cache_type?: string
/**
* To enable mmap, default is true
*/
use_mmap?: boolean
vision_model?: boolean
text_model?: boolean
}
type ModelSettingParamsKeys = keyof ModelSettingParams
export const modelSettingParamsKeys: ModelSettingParamsKeys[] = [
'ctx_len',
'ngl',
'embedding',
'n_parallel',
'cpu_threads',
'prompt_template',
'system_prompt',
'ai_prompt',
'user_prompt',
'llama_model_path',
'mmproj',
'cont_batching',
'engine',
'pre_prompt',
'n_batch',
'caching_enabled',
'grp_attn_n',
'grp_attn_w',
'mlock',
'grammar_file',
'flash_attn',
'cache_type',
'use_mmap',
]
/**
* The available model runtime parameters.
*/
export interface ModelRuntimeParams {
/**
* Controls the randomness of the model’s output.
*/
export type ModelRuntimeParams = {
temperature?: number
token_limit?: number
top_k?: number
/**
* Set probability threshold for more relevant outputs.
*/
top_p?: number
/**
* Enable real-time data processing for faster predictions.
*/
stream?: boolean
/*
* The maximum number of tokens the model will generate in a single response.
*/
max_tokens?: number
/**
* Defines specific tokens or phrases at which the model will stop generating further output.
*/
stop?: string[]
/**
* Adjusts the likelihood of the model repeating words or phrases in its output.
*/
frequency_penalty?: number
/**
* Influences the generation of new and varied concepts in the model’s output.
*/
presence_penalty?: number
engine?: string
}
export type ModelInitFailed = Model & {
error: Error
}
type ModelRuntimeParamsKeys = keyof ModelRuntimeParams
export const modelRuntimeParamsKeys: ModelRuntimeParamsKeys[] = [
'temperature',
'token_limit',
'top_k',
'top_p',
'stream',
'max_tokens',
'stop',
'frequency_penalty',
'presence_penalty',
]

17
core/src/types/model/modelEvent.ts Normal file
View File

@ -0,0 +1,17 @@
/**
* The `EventName` enumeration contains the names of all the available events in the Jan platform.
*/
export enum ModelEvent {
/** The `OnModelInit` event is emitted when a model inits. */
OnModelInit = 'OnModelInit',
/** The `OnModelReady` event is emitted when a model ready. */
OnModelReady = 'OnModelReady',
/** The `OnModelFail` event is emitted when a model fails loading. */
OnModelFail = 'OnModelFail',
/** The `OnModelStop` event is emitted when a model start to stop. */
OnModelStop = 'OnModelStop',
/** The `OnModelStopped` event is emitted when a model stopped ok. */
OnModelStopped = 'OnModelStopped',
/** The `OnModelUpdate` event is emitted when the model list is updated. */
OnModelsUpdate = 'OnModelsUpdate',
}

52
core/src/types/model/modelInterface.ts Normal file
View File

@ -0,0 +1,52 @@
import { GpuSetting } from '../miscellaneous'
import { Model } from './modelEntity'
/**
* Model extension for managing models.
*/
export interface ModelInterface {
/**
* Downloads a model.
* @param model - The model to download.
* @param network - Optional object to specify proxy/whether to ignore SSL certificates.
* @returns A Promise that resolves when the model has been downloaded.
*/
downloadModel(
model: Model,
gpuSettings?: GpuSetting,
network?: { ignoreSSL?: boolean; proxy?: string }
): Promise<void>
/**
* Cancels the download of a specific model.
* @param {string} modelId - The ID of the model to cancel the download for.
* @returns {Promise<void>} A promise that resolves when the download has been cancelled.
*/
cancelModelDownload(modelId: string): Promise<void>
/**
* Deletes a model.
* @param modelId - The ID of the model to delete.
* @returns A Promise that resolves when the model has been deleted.
*/
deleteModel(modelId: string): Promise<void>
/**
* Saves a model.
* @param model - The model to save.
* @returns A Promise that resolves when the model has been saved.
*/
saveModel(model: Model): Promise<void>
/**
* Gets a list of downloaded models.
* @returns A Promise that resolves with an array of downloaded models.
*/
getDownloadedModels(): Promise<Model[]>
/**
* Gets a list of configured models.
* @returns A Promise that resolves with an array of configured models.
*/
getConfiguredModels(): Promise<Model[]>
}

2
core/src/types/thread/index.ts
View File

@ -1 +1,3 @@
export * from './threadEntity'
export * from './threadInterface'
export * from './threadEvent'

Some files were not shown because too many files have changed in this diff Show More