diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f980b9df7..db1eed38d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ { - "name": "jan", - "image": "node:20" -} \ No newline at end of file + "name": "jan", + "image": "node:20" +} diff --git a/.github/workflows/jan-electron-linter-and-test.yml b/.github/workflows/jan-electron-linter-and-test.yml index 6d5aaf150..40085391f 100644 --- a/.github/workflows/jan-electron-linter-and-test.yml +++ b/.github/workflows/jan-electron-linter-and-test.yml @@ -1,5 +1,6 @@ name: Jan Electron Linter & Test on: + workflow_dispatch: push: branches: - main diff --git a/.github/workflows/jan-server-build-nightly.yml b/.github/workflows/jan-server-build-nightly.yml new file mode 100644 index 000000000..0d1bc3ca8 --- /dev/null +++ b/.github/workflows/jan-server-build-nightly.yml @@ -0,0 +1,40 @@ +name: Jan Build Docker Nightly or Manual + +on: + push: + branches: + - main + - feature/helmchart-and-ci-jan-server + paths-ignore: + - 'README.md' + - 'docs/**' + schedule: + - cron: '0 20 * * 1,2,3' # At 8 PM UTC on Monday, Tuesday, and Wednesday which is 3 AM UTC+7 Tuesday, Wednesday, and Thursday + workflow_dispatch: + +jobs: + # Job create Update app version based on latest release tag with build number and save to output + get-update-version: + uses: ./.github/workflows/template-get-update-version.yml + + build-cpu: + uses: ./.github/workflows/template-build-jan-server.yml + permissions: + packages: write + secrets: inherit + needs: [get-update-version] + with: + dockerfile_path: ./Dockerfile + docker_image_tag: "ghcr.io/janhq/jan-server:dev-cpu-latest,ghcr.io/janhq/jan-server:dev-cpu-${{ needs.get-update-version.outputs.new_version }}" + + build-gpu: + uses: ./.github/workflows/template-build-jan-server.yml + permissions: + packages: write + secrets: inherit + needs: [get-update-version] + with: + dockerfile_path: ./Dockerfile.gpu + docker_image_tag: "ghcr.io/janhq/jan-server:dev-cuda-12.2-latest,ghcr.io/janhq/jan-server:dev-cuda-12.2-${{ needs.get-update-version.outputs.new_version }}" + + diff --git a/.github/workflows/jan-server-build.yml b/.github/workflows/jan-server-build.yml new file mode 100644 index 000000000..0665838d6 --- /dev/null +++ b/.github/workflows/jan-server-build.yml @@ -0,0 +1,30 @@ +name: Jan Build Docker + +on: + push: + tags: ["v[0-9]+.[0-9]+.[0-9]+"] + +jobs: + # Job create Update app version based on latest release tag with build number and save to output + get-update-version: + uses: ./.github/workflows/template-get-update-version.yml + + build-cpu: + permissions: + packages: write + uses: ./.github/workflows/template-build-jan-server.yml + secrets: inherit + needs: [get-update-version] + with: + dockerfile_path: ./Dockerfile + docker_image_tag: "ghcr.io/janhq/jan-server:cpu-latest,ghcr.io/janhq/jan-server:cpu-${{ needs.get-update-version.outputs.new_version }}" + + build-gpu: + permissions: + packages: write + uses: ./.github/workflows/template-build-jan-server.yml + secrets: inherit + needs: [get-update-version] + with: + dockerfile_path: ./Dockerfile.gpu + docker_image_tag: "ghcr.io/janhq/jan-server:cuda-12.2-latest,ghcr.io/janhq/jan-server:cuda-12.2-${{ needs.get-update-version.outputs.new_version }}" diff --git a/.github/workflows/template-build-jan-server.yml b/.github/workflows/template-build-jan-server.yml new file mode 100644 index 000000000..9bb772605 --- /dev/null +++ b/.github/workflows/template-build-jan-server.yml @@ -0,0 +1,39 @@ +name: build-jan-server +on: + workflow_call: + inputs: + dockerfile_path: + required: false + type: string + default: './Dockerfile' + docker_image_tag: + required: true + type: string + default: 'ghcr.io/janhq/jan-server:dev-latest' + +jobs: + build: + runs-on: ubuntu-latest + env: + REGISTRY: ghcr.io + IMAGE_NAME: janhq/jan-server + permissions: + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: . + file: ${{ inputs.dockerfile_path }} + push: true + tags: ${{ inputs.docker_image_tag }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4540e5c7a..62878011e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ error.log node_modules *.tgz +!charts/server/charts/*.tgz yarn.lock dist build @@ -28,4 +29,5 @@ extensions/inference-nitro-extension/bin/*/*.exp extensions/inference-nitro-extension/bin/*/*.lib extensions/inference-nitro-extension/bin/saved-* extensions/inference-nitro-extension/bin/*.tar.gz - +extensions/inference-nitro-extension/bin/vulkaninfoSDK.exe +extensions/inference-nitro-extension/bin/vulkaninfo diff --git a/core/.prettierignore b/.prettierignore similarity index 100% rename from core/.prettierignore rename to .prettierignore diff --git a/core/.prettierrc b/.prettierrc similarity index 100% rename from core/.prettierrc rename to .prettierrc diff --git a/Dockerfile b/Dockerfile index 949a92673..48b2d254f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,39 +1,61 @@ -FROM node:20-bullseye AS base +FROM node:20-bookworm AS base # 1. Install dependencies only when needed -FROM base AS deps +FROM base AS builder + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel && rm -rf /var/lib/apt/lists/* + WORKDIR /app # Install dependencies based on the preferred package manager -COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./ -RUN yarn install +COPY . ./ + +RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \ + jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json +RUN make install-and-build # # 2. Rebuild the source code only when needed -FROM base AS builder -WORKDIR /app -COPY --from=deps /app/node_modules ./node_modules -COPY . . -# This will do the trick, use the corresponding env file for each environment. -RUN yarn workspace server install -RUN yarn server:prod - -# 3. Production image, copy all the files and run next FROM base AS runner + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel && rm -rf /var/lib/apt/lists/* + WORKDIR /app -ENV NODE_ENV=production +# Copy the package.json and yarn.lock of root yarn space to leverage Docker cache +COPY --from=builder /app/package.json ./package.json +COPY --from=builder /app/node_modules ./node_modules/ +COPY --from=builder /app/yarn.lock ./yarn.lock -# RUN addgroup -g 1001 -S nodejs; -COPY --from=builder /app/server/build ./ +# Copy the package.json, yarn.lock, and build output of server yarn space to leverage Docker cache +COPY --from=builder /app/core ./core/ +COPY --from=builder /app/server ./server/ +RUN cd core && yarn install && yarn run build +RUN yarn workspace @janhq/server install && yarn workspace @janhq/server build +COPY --from=builder /app/docs/openapi ./docs/openapi/ -# Automatically leverage output traces to reduce image size -# https://nextjs.org/docs/advanced-features/output-file-tracing -COPY --from=builder /app/server/node_modules ./node_modules -COPY --from=builder /app/server/package.json ./package.json +# Copy pre-install dependencies +COPY --from=builder /app/pre-install ./pre-install/ -EXPOSE 4000 3928 +# Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache +COPY --from=builder /app/uikit ./uikit/ +COPY --from=builder /app/web ./web/ +COPY --from=builder /app/models ./models/ -ENV PORT 4000 -ENV APPDATA /app/data +RUN yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build +RUN yarn workspace jan-web install -CMD ["node", "main.js"] \ No newline at end of file +RUN npm install -g serve@latest + +EXPOSE 1337 3000 3928 + +ENV JAN_API_HOST 0.0.0.0 +ENV JAN_API_PORT 1337 + +ENV API_BASE_URL http://localhost:1337 + +CMD ["sh", "-c", "export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out & cd server && node build/main.js"] + +# docker build -t jan . +# docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 jan diff --git a/Dockerfile.gpu b/Dockerfile.gpu new file mode 100644 index 000000000..832e2c18c --- /dev/null +++ b/Dockerfile.gpu @@ -0,0 +1,88 @@ +# Please change the base image to the appropriate CUDA version base on NVIDIA Driver Compatibility +# Run nvidia-smi to check the CUDA version and the corresponding driver version +# Then update the base image to the appropriate CUDA version refer https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags + +FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base + +# 1. Install dependencies only when needed +FROM base AS builder + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt install nodejs -y && rm -rf /var/lib/apt/lists/* + +# Update alternatives for GCC and related tools +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \ + update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 + +RUN npm install -g yarn + +WORKDIR /app + +# Install dependencies based on the preferred package manager +COPY . ./ + +RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \ + jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json +RUN make install-and-build + +# # 2. Rebuild the source code only when needed +FROM base AS runner + +# Install g++ 11 +RUN apt update && apt install -y gcc-11 g++-11 cpp-11 jq xsel curl gnupg make python3-dev && curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt-get install nodejs -y && rm -rf /var/lib/apt/lists/* + +# Update alternatives for GCC and related tools +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 \ + --slave /usr/bin/g++ g++ /usr/bin/g++-11 \ + --slave /usr/bin/gcov gcov /usr/bin/gcov-11 \ + --slave /usr/bin/gcc-ar gcc-ar /usr/bin/gcc-ar-11 \ + --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-11 && \ + update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-11 110 + +RUN npm install -g yarn + +WORKDIR /app + +# Copy the package.json and yarn.lock of root yarn space to leverage Docker cache +COPY --from=builder /app/package.json ./package.json +COPY --from=builder /app/node_modules ./node_modules/ +COPY --from=builder /app/yarn.lock ./yarn.lock + +# Copy the package.json, yarn.lock, and build output of server yarn space to leverage Docker cache +COPY --from=builder /app/core ./core/ +COPY --from=builder /app/server ./server/ +RUN cd core && yarn install && yarn run build +RUN yarn workspace @janhq/server install && yarn workspace @janhq/server build +COPY --from=builder /app/docs/openapi ./docs/openapi/ + +# Copy pre-install dependencies +COPY --from=builder /app/pre-install ./pre-install/ + +# Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache +COPY --from=builder /app/uikit ./uikit/ +COPY --from=builder /app/web ./web/ +COPY --from=builder /app/models ./models/ + +RUN yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build +RUN yarn workspace jan-web install + +RUN npm install -g serve@latest + +EXPOSE 1337 3000 3928 + +ENV LD_LIBRARY_PATH=/usr/local/cuda/targets/x86_64-linux/lib:/usr/local/cuda-12.0/compat${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + +ENV JAN_API_HOST 0.0.0.0 +ENV JAN_API_PORT 1337 + +ENV API_BASE_URL http://localhost:1337 + +CMD ["sh", "-c", "export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out & cd server && node build/main.js"] + +# pre-requisites: nvidia-docker +# docker build -t jan-gpu . -f Dockerfile.gpu +# docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 --gpus all jan-gpu diff --git a/Makefile b/Makefile index 905a68321..a45477b29 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,9 @@ endif check-file-counts: install-and-build ifeq ($(OS),Windows_NT) - powershell -Command "if ((Get-ChildItem -Path electron/pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in electron/pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }" + powershell -Command "if ((Get-ChildItem -Path pre-install -Filter *.tgz | Measure-Object | Select-Object -ExpandProperty Count) -ne (Get-ChildItem -Path extensions -Directory | Measure-Object | Select-Object -ExpandProperty Count)) { Write-Host 'Number of .tgz files in pre-install does not match the number of subdirectories in extension'; exit 1 } else { Write-Host 'Extension build successful' }" else - @tgz_count=$$(find electron/pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in electron/pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi + @tgz_count=$$(find pre-install -type f -name "*.tgz" | wc -l); dir_count=$$(find extensions -mindepth 1 -maxdepth 1 -type d | wc -l); if [ $$tgz_count -ne $$dir_count ]; then echo "Number of .tgz files in pre-install ($$tgz_count) does not match the number of subdirectories in extension ($$dir_count)"; exit 1; else echo "Extension build successful"; fi endif dev: check-file-counts @@ -52,18 +52,28 @@ build: check-file-counts clean: ifeq ($(OS),Windows_NT) - powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist -Recurse -Directory | Remove-Item -Recurse -Force" + powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out -Recurse -Directory | Remove-Item -Recurse -Force" + powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz" + powershell -Command "Remove-Item -Recurse -Force ./electron/pre-install/*.tgz" rmdir /s /q "%USERPROFILE%\jan\extensions" else ifeq ($(shell uname -s),Linux) find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name ".next" -type d -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + + find . -name "build" -type d -exec rm -rf '{}' + + find . -name "out" -type d -exec rm -rf '{}' + + rm -rf ./pre-install/*.tgz + rm -rf ./electron/pre-install/*.tgz rm -rf "~/jan/extensions" rm -rf "~/.cache/jan*" else find . -name "node_modules" -type d -prune -exec rm -rf '{}' + find . -name ".next" -type d -exec rm -rf '{}' + find . -name "dist" -type d -exec rm -rf '{}' + + find . -name "build" -type d -exec rm -rf '{}' + + find . -name "out" -type d -exec rm -rf '{}' + + rm -rf ./pre-install/*.tgz + rm -rf ./electron/pre-install/*.tgz rm -rf ~/jan/extensions rm -rf ~/Library/Caches/jan* endif diff --git a/README.md b/README.md index e1f74ef23..425ea69be 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage @@ -167,6 +167,7 @@ To reset your installation: - Clear Application cache in `~/Library/Caches/jan` ## Requirements for running Jan + - MacOS: 13 or higher - Windows: - Windows 10 or higher @@ -194,17 +195,17 @@ Contributions are welcome! Please read the [CONTRIBUTING.md](CONTRIBUTING.md) fi 1. **Clone the repository and prepare:** - ```bash - git clone https://github.com/janhq/jan - cd jan - git checkout -b DESIRED_BRANCH - ``` + ```bash + git clone https://github.com/janhq/jan + cd jan + git checkout -b DESIRED_BRANCH + ``` 2. **Run development and use Jan Desktop** - ```bash - make dev - ``` + ```bash + make dev + ``` This will start the development server and open the desktop app. @@ -218,6 +219,101 @@ make build This will build the app MacOS m1/m2 for production (with code signing already done) and put the result in `dist` folder. +### Docker mode + +- Supported OS: Linux, WSL2 Docker +- Pre-requisites: + + - Docker Engine and Docker Compose are required to run Jan in Docker mode. Follow the [instructions](https://docs.docker.com/engine/install/ubuntu/) below to get started with Docker Engine on Ubuntu. + + ```bash + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh ./get-docker.sh --dry-run + ``` + + - If you intend to run Jan in GPU mode, you need to install `nvidia-driver` and `nvidia-docker2`. Follow the instruction [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for installation. + +- Run Jan in Docker mode + +| Docker compose Profile | Description | +| ---------------------- | -------------------------------------------- | +| `cpu-fs` | Run Jan in CPU mode with default file system | +| `cpu-s3fs` | Run Jan in CPU mode with S3 file system | +| `gpu-fs` | Run Jan in GPU mode with default file system | +| `gpu-s3fs` | Run Jan in GPU mode with S3 file system | + +| Environment Variable | Description | +| ----------------------- | ------------------------------------------------------------------------------------------------------- | +| `S3_BUCKET_NAME` | S3 bucket name - leave blank for default file system | +| `AWS_ACCESS_KEY_ID` | AWS access key ID - leave blank for default file system | +| `AWS_SECRET_ACCESS_KEY` | AWS secret access key - leave blank for default file system | +| `AWS_ENDPOINT` | AWS endpoint URL - leave blank for default file system | +| `AWS_REGION` | AWS region - leave blank for default file system | +| `API_BASE_URL` | Jan Server URL, please modify it as your public ip address or domain name default http://localhost:1377 | + +- **Option 1**: Run Jan in CPU mode + + ```bash + # cpu mode with default file system + docker compose --profile cpu-fs up -d + + # cpu mode with S3 file system + docker compose --profile cpu-s3fs up -d + ``` + +- **Option 2**: Run Jan in GPU mode + + - **Step 1**: Check CUDA compatibility with your NVIDIA driver by running `nvidia-smi` and check the CUDA version in the output + + ```bash + nvidia-smi + + # Output + +---------------------------------------------------------------------------------------+ + | NVIDIA-SMI 531.18 Driver Version: 531.18 CUDA Version: 12.1 | + |-----------------------------------------+----------------------+----------------------+ + | GPU Name TCC/WDDM | Bus-Id Disp.A | Volatile Uncorr. ECC | + | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | + | | | MIG M. | + |=========================================+======================+======================| + | 0 NVIDIA GeForce RTX 4070 Ti WDDM | 00000000:01:00.0 On | N/A | + | 0% 44C P8 16W / 285W| 1481MiB / 12282MiB | 2% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + | 1 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:02:00.0 Off | N/A | + | 0% 49C P8 14W / 120W| 0MiB / 6144MiB | 0% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + | 2 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:05:00.0 Off | N/A | + | 29% 38C P8 11W / 120W| 0MiB / 6144MiB | 0% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + + +---------------------------------------------------------------------------------------+ + | Processes: | + | GPU GI CI PID Type Process name GPU Memory | + | ID ID Usage | + |=======================================================================================| + ``` + + - **Step 2**: Visit [NVIDIA NGC Catalog ](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags) and find the smallest minor version of image tag that matches your CUDA version (e.g., 12.1 -> 12.1.0) + + - **Step 3**: Update the `Dockerfile.gpu` line number 5 with the latest minor version of the image tag from step 2 (e.g. change `FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base` to `FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS base`) + + - **Step 4**: Run command to start Jan in GPU mode + + ```bash + # GPU mode with default file system + docker compose --profile gpu up -d + + # GPU mode with S3 file system + docker compose --profile gpu-s3fs up -d + ``` + +This will start the web server and you can access Jan at `http://localhost:3000`. + +> Note: RAG feature is not supported in Docker mode with s3fs yet. + ## Acknowledgements Jan builds on top of other open-source projects: diff --git a/charts/server/Chart.lock b/charts/server/Chart.lock new file mode 100644 index 000000000..915788d61 --- /dev/null +++ b/charts/server/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: oci://ghcr.io/janhq/charts + version: 0.1.2 +digest: sha256:35e98bde174130787755b0f8ea2359b7b6790d965a7157c2f7cabf1bc8c04471 +generated: "2024-02-20T16:20:37.6530108+07:00" diff --git a/charts/server/Chart.yaml b/charts/server/Chart.yaml new file mode 100644 index 000000000..fb2e1c91b --- /dev/null +++ b/charts/server/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: jan-server +description: A Helm chart for Kubernetes +type: application +version: 0.1.0 +appVersion: '1.0.0' +dependencies: + - name: common + version: 0.1.2 # common-chart-version + repository: oci://ghcr.io/janhq/charts diff --git a/charts/server/charts/common-0.1.2.tgz b/charts/server/charts/common-0.1.2.tgz new file mode 100644 index 000000000..946617eab Binary files /dev/null and b/charts/server/charts/common-0.1.2.tgz differ diff --git a/charts/server/config.json b/charts/server/config.json new file mode 100644 index 000000000..62e9682fa --- /dev/null +++ b/charts/server/config.json @@ -0,0 +1,4 @@ +{ + "image-list": "server=ghcr.io/janhq/jan-server", + "platforms": "linux/amd64" +} \ No newline at end of file diff --git a/charts/server/values.yaml b/charts/server/values.yaml new file mode 100644 index 000000000..70f463174 --- /dev/null +++ b/charts/server/values.yaml @@ -0,0 +1,256 @@ +common: + imageTag: v0.4.6-cpu + # DO NOT CHANGE THE LINE ABOVE. MAKE ALL CHANGES BELOW + + # Global pvc for all workload + pvc: + enabled: false + name: 'janroot' + accessModes: 'ReadWriteOnce' + storageClassName: '' + capacity: '50Gi' + + # Global image pull secret + imagePullSecrets: [] + + externalSecret: + create: false + name: '' + annotations: {} + + nameOverride: 'jan-server' + fullnameOverride: 'jan-server' + + serviceAccount: + create: true + annotations: {} + name: 'jan-server-service-account' + + podDisruptionBudget: + create: false + minAvailable: 1 + + workloads: + - name: server + image: + repository: ghcr.io/janhq/jan-server + pullPolicy: Always + + command: ['/bin/sh', '-c'] + args: ['cd server && node build/main.js'] + + replicaCount: 1 + ports: + containerPort: 1337 + + strategy: + canary: + steps: + - setWeight: 50 + - pause: { duration: 1m } + + ingress: + enabled: true + className: 'nginx' + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: '100m' + nginx.ingress.kubernetes.io/proxy-read-timeout: '1800' + nginx.ingress.kubernetes.io/proxy-send-timeout: '1800' + # cert-manager.io/cluster-issuer: 'jan-ai-dns01-cluster-issuer' + # nginx.ingress.kubernetes.io/force-ssl-redirect: 'true' + nginx.ingress.kubernetes.io/backend-protocol: HTTP + hosts: + - host: server.local + paths: + - path: / + pathType: Prefix + tls: + [] + # - hosts: + # - server-dev.jan.ai + # secretName: jan-server-prod-tls-v2 + + instrumentation: + enabled: false + podAnnotations: {} + + podSecurityContext: {} + + securityContext: {} + + service: + extenalLabel: {} + type: ClusterIP + port: 1337 + targetPort: 1337 + + # If you want to use GPU, please uncomment the following lines and change imageTag to the one with GPU support + resources: + # limits: + # nvidia.com/gpu: 1 + requests: + cpu: 2000m + memory: 8192M + + # If you want to use pv, please uncomment the following lines and enable pvc.enabled + volumes: + [] + # - name: janroot + # persistentVolumeClaim: + # claimName: janroot + + volumeMounts: + [] + # - name: janroot + # mountPath: /app/server/build/jan + + # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME, AWS_ENDPOINT, AWS_REGION should mount as a secret env instead of plain text here + # Change API_BASE_URL to your server's public domain + env: + - name: API_BASE_URL + value: 'http://server.local' + + lifecycle: {} + autoscaling: + enabled: false + minReplicas: 2 + maxReplicas: 3 + targetCPUUtilizationPercentage: 95 + targetMemoryUtilizationPercentage: 95 + + kedaScaling: + enabled: false # ignore if autoscaling.enable = true + cooldownPeriod: 30 + pollingInterval: 2 + minReplicas: 1 + maxReplicas: 5 + metricName: celery_queue_length + query: celery_queue_length{queue_name="myqueue"} # change queue_name here + serverAddress: http://prometheus-prod-kube-prome-prometheus.monitoring.svc:9090 + threshold: '3' + + nodeSelector: {} + + tolerations: [] + + podSecurityGroup: + enabled: false + securitygroupid: [] + + # Reloader Option + reloader: 'false' + vpa: + enabled: false + + - name: web + image: + repository: ghcr.io/janhq/jan-server + pullPolicy: Always + + command: ['/bin/sh', '-c'] + args: + [ + 'export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out', + ] + + replicaCount: 1 + ports: + containerPort: 3000 + + strategy: + canary: + steps: + - setWeight: 50 + - pause: { duration: 1m } + + ingress: + enabled: true + className: 'nginx' + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: '100m' + nginx.ingress.kubernetes.io/proxy-read-timeout: '1800' + nginx.ingress.kubernetes.io/proxy-send-timeout: '1800' + # cert-manager.io/cluster-issuer: 'jan-ai-dns01-cluster-issuer' + # nginx.ingress.kubernetes.io/force-ssl-redirect: 'true' + nginx.ingress.kubernetes.io/backend-protocol: HTTP + hosts: + - host: web.local + paths: + - path: / + pathType: Prefix + tls: + [] + # - hosts: + # - server-dev.jan.ai + # secretName: jan-server-prod-tls-v2 + + instrumentation: + enabled: false + podAnnotations: {} + + podSecurityContext: {} + + securityContext: {} + + service: + extenalLabel: {} + type: ClusterIP + port: 3000 + targetPort: 3000 + + resources: + limits: + cpu: 1000m + memory: 2048M + requests: + cpu: 50m + memory: 500M + + volumes: + [] + # - name: janroot + # persistentVolumeClaim: + # claimName: janroot + + volumeMounts: + [] + # - name: janroot + # mountPath: /app/server/build/jan + + # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME, AWS_ENDPOINT, AWS_REGION should mount as a secret env instead of plain text here + # Change API_BASE_URL to your server's public domain + env: + - name: API_BASE_URL + value: 'http://server.local' + + lifecycle: {} + autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 95 + targetMemoryUtilizationPercentage: 95 + + kedaScaling: + enabled: false # ignore if autoscaling.enable = true + cooldownPeriod: 30 + pollingInterval: 2 + minReplicas: 1 + maxReplicas: 5 + metricName: celery_queue_length + query: celery_queue_length{queue_name="myqueue"} # change queue_name here + serverAddress: http://prometheus-prod-kube-prome-prometheus.monitoring.svc:9090 + threshold: '3' + + nodeSelector: {} + + tolerations: [] + + podSecurityGroup: + enabled: false + securitygroupid: [] + + # Reloader Option + reloader: 'false' + vpa: + enabled: false diff --git a/core/jest.config.js b/core/jest.config.js index fb03768fe..c18f55091 100644 --- a/core/jest.config.js +++ b/core/jest.config.js @@ -4,4 +4,4 @@ module.exports = { moduleNameMapper: { '@/(.*)': '/src/$1', }, -} \ No newline at end of file +} diff --git a/core/package.json b/core/package.json index 437e6d0a6..c3abe2d56 100644 --- a/core/package.json +++ b/core/package.json @@ -57,6 +57,7 @@ "rollup-plugin-typescript2": "^0.36.0", "ts-jest": "^26.1.1", "tslib": "^2.6.2", - "typescript": "^5.2.2" + "typescript": "^5.2.2", + "rimraf": "^3.0.2" } } diff --git a/core/rollup.config.ts b/core/rollup.config.ts index d78130a4d..ebea8e237 100644 --- a/core/rollup.config.ts +++ b/core/rollup.config.ts @@ -54,7 +54,8 @@ export default [ 'url', 'http', 'os', - 'util' + 'util', + 'child_process', ], watch: { include: 'src/node/**', diff --git a/core/src/api/index.ts b/core/src/api/index.ts index 0d7cc51f7..676020758 100644 --- a/core/src/api/index.ts +++ b/core/src/api/index.ts @@ -1,15 +1,22 @@ +/** + * Native Route APIs + * @description Enum of all the routes exposed by the app + */ +export enum NativeRoute { + openExternalUrl = 'openExternalUrl', + openAppDirectory = 'openAppDirectory', + openFileExplore = 'openFileExplorer', + selectDirectory = 'selectDirectory', + relaunch = 'relaunch', +} + /** * App Route APIs * @description Enum of all the routes exposed by the app */ export enum AppRoute { - openExternalUrl = 'openExternalUrl', - openAppDirectory = 'openAppDirectory', - openFileExplore = 'openFileExplorer', - selectDirectory = 'selectDirectory', getAppConfigurations = 'getAppConfigurations', updateAppConfiguration = 'updateAppConfiguration', - relaunch = 'relaunch', joinPath = 'joinPath', isSubdirectory = 'isSubdirectory', baseName = 'baseName', @@ -30,6 +37,7 @@ export enum DownloadRoute { downloadFile = 'downloadFile', pauseDownload = 'pauseDownload', resumeDownload = 'resumeDownload', + getDownloadProgress = 'getDownloadProgress', } export enum DownloadEvent { @@ -68,6 +76,10 @@ export enum FileManagerRoute { export type ApiFunction = (...args: any[]) => any +export type NativeRouteFunctions = { + [K in NativeRoute]: ApiFunction +} + export type AppRouteFunctions = { [K in AppRoute]: ApiFunction } @@ -96,7 +108,8 @@ export type FileManagerRouteFunctions = { [K in FileManagerRoute]: ApiFunction } -export type APIFunctions = AppRouteFunctions & +export type APIFunctions = NativeRouteFunctions & + AppRouteFunctions & AppEventFunctions & DownloadRouteFunctions & DownloadEventFunctions & @@ -104,11 +117,13 @@ export type APIFunctions = AppRouteFunctions & FileSystemRouteFunctions & FileManagerRoute -export const APIRoutes = [ +export const CoreRoutes = [ ...Object.values(AppRoute), ...Object.values(DownloadRoute), ...Object.values(ExtensionRoute), ...Object.values(FileSystemRoute), ...Object.values(FileManagerRoute), ] + +export const APIRoutes = [...CoreRoutes, ...Object.values(NativeRoute)] export const APIEvents = [...Object.values(AppEvent), ...Object.values(DownloadEvent)] diff --git a/core/src/extension.ts b/core/src/extension.ts index 0b7f9b7fc..3528f581c 100644 --- a/core/src/extension.ts +++ b/core/src/extension.ts @@ -1,13 +1,13 @@ export enum ExtensionTypeEnum { - Assistant = "assistant", - Conversational = "conversational", - Inference = "inference", - Model = "model", - SystemMonitoring = "systemMonitoring", + Assistant = 'assistant', + Conversational = 'conversational', + Inference = 'inference', + Model = 'model', + SystemMonitoring = 'systemMonitoring', } export interface ExtensionType { - type(): ExtensionTypeEnum | undefined; + type(): ExtensionTypeEnum | undefined } /** * Represents a base extension. @@ -20,16 +20,16 @@ export abstract class BaseExtension implements ExtensionType { * Undefined means its not extending any known extension by the application. */ type(): ExtensionTypeEnum | undefined { - return undefined; + return undefined } /** * Called when the extension is loaded. * Any initialization logic for the extension should be put here. */ - abstract onLoad(): void; + abstract onLoad(): void /** * Called when the extension is unloaded. * Any cleanup logic for the extension should be put here. */ - abstract onUnload(): void; + abstract onUnload(): void } diff --git a/core/src/extensions/assistant.ts b/core/src/extensions/assistant.ts index ba345711a..5c3114f41 100644 --- a/core/src/extensions/assistant.ts +++ b/core/src/extensions/assistant.ts @@ -1,5 +1,5 @@ -import { Assistant, AssistantInterface } from "../index"; -import { BaseExtension, ExtensionTypeEnum } from "../extension"; +import { Assistant, AssistantInterface } from '../index' +import { BaseExtension, ExtensionTypeEnum } from '../extension' /** * Assistant extension for managing assistants. @@ -10,10 +10,10 @@ export abstract class AssistantExtension extends BaseExtension implements Assist * Assistant extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Assistant; + return ExtensionTypeEnum.Assistant } - abstract createAssistant(assistant: Assistant): Promise; - abstract deleteAssistant(assistant: Assistant): Promise; - abstract getAssistants(): Promise; + abstract createAssistant(assistant: Assistant): Promise + abstract deleteAssistant(assistant: Assistant): Promise + abstract getAssistants(): Promise } diff --git a/core/src/extensions/conversational.ts b/core/src/extensions/conversational.ts index 4319784c3..a49a4e689 100644 --- a/core/src/extensions/conversational.ts +++ b/core/src/extensions/conversational.ts @@ -14,7 +14,7 @@ export abstract class ConversationalExtension * Conversation extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Conversational; + return ExtensionTypeEnum.Conversational } abstract getThreads(): Promise diff --git a/core/src/extensions/index.ts b/core/src/extensions/index.ts index 1796c1618..522334548 100644 --- a/core/src/extensions/index.ts +++ b/core/src/extensions/index.ts @@ -2,24 +2,24 @@ * Conversational extension. Persists and retrieves conversations. * @module */ -export { ConversationalExtension } from "./conversational"; +export { ConversationalExtension } from './conversational' /** * Inference extension. Start, stop and inference models. */ -export { InferenceExtension } from "./inference"; +export { InferenceExtension } from './inference' /** * Monitoring extension for system monitoring. */ -export { MonitoringExtension } from "./monitoring"; +export { MonitoringExtension } from './monitoring' /** * Assistant extension for managing assistants. */ -export { AssistantExtension } from "./assistant"; +export { AssistantExtension } from './assistant' /** * Model extension for managing models. */ -export { ModelExtension } from "./model"; +export { ModelExtension } from './model' diff --git a/core/src/extensions/inference.ts b/core/src/extensions/inference.ts index c551d108f..e8e51f9eb 100644 --- a/core/src/extensions/inference.ts +++ b/core/src/extensions/inference.ts @@ -1,5 +1,5 @@ -import { InferenceInterface, MessageRequest, ThreadMessage } from "../index"; -import { BaseExtension, ExtensionTypeEnum } from "../extension"; +import { InferenceInterface, MessageRequest, ThreadMessage } from '../index' +import { BaseExtension, ExtensionTypeEnum } from '../extension' /** * Inference extension. Start, stop and inference models. @@ -9,8 +9,8 @@ export abstract class InferenceExtension extends BaseExtension implements Infere * Inference extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Inference; + return ExtensionTypeEnum.Inference } - abstract inference(data: MessageRequest): Promise; + abstract inference(data: MessageRequest): Promise } diff --git a/core/src/extensions/model.ts b/core/src/extensions/model.ts index 30aa5b6ba..df7d14f42 100644 --- a/core/src/extensions/model.ts +++ b/core/src/extensions/model.ts @@ -1,5 +1,5 @@ -import { BaseExtension, ExtensionTypeEnum } from "../extension"; -import { Model, ModelInterface } from "../index"; +import { BaseExtension, ExtensionTypeEnum } from '../extension' +import { Model, ModelInterface } from '../index' /** * Model extension for managing models. @@ -9,16 +9,16 @@ export abstract class ModelExtension extends BaseExtension implements ModelInter * Model extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.Model; + return ExtensionTypeEnum.Model } abstract downloadModel( model: Model, - network?: { proxy: string; ignoreSSL?: boolean }, - ): Promise; - abstract cancelModelDownload(modelId: string): Promise; - abstract deleteModel(modelId: string): Promise; - abstract saveModel(model: Model): Promise; - abstract getDownloadedModels(): Promise; - abstract getConfiguredModels(): Promise; + network?: { proxy: string; ignoreSSL?: boolean } + ): Promise + abstract cancelModelDownload(modelId: string): Promise + abstract deleteModel(modelId: string): Promise + abstract saveModel(model: Model): Promise + abstract getDownloadedModels(): Promise + abstract getConfiguredModels(): Promise } diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts index 2de9b9ae5..ba193f0f4 100644 --- a/core/src/extensions/monitoring.ts +++ b/core/src/extensions/monitoring.ts @@ -1,5 +1,5 @@ -import { BaseExtension, ExtensionTypeEnum } from "../extension"; -import { MonitoringInterface } from "../index"; +import { BaseExtension, ExtensionTypeEnum } from '../extension' +import { MonitoringInterface } from '../index' /** * Monitoring extension for system monitoring. @@ -10,9 +10,9 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit * Monitoring extension type. */ type(): ExtensionTypeEnum | undefined { - return ExtensionTypeEnum.SystemMonitoring; + return ExtensionTypeEnum.SystemMonitoring } - abstract getResourcesInfo(): Promise; - abstract getCurrentLoad(): Promise; + abstract getResourcesInfo(): Promise + abstract getCurrentLoad(): Promise } diff --git a/core/src/index.ts b/core/src/index.ts index a56b6f0e1..3505797b1 100644 --- a/core/src/index.ts +++ b/core/src/index.ts @@ -38,3 +38,10 @@ export * from './extension' * @module */ export * from './extensions/index' + +/** + * Declare global object + */ +declare global { + var core: any | undefined +} diff --git a/core/src/node/api/common/adapter.ts b/core/src/node/api/common/adapter.ts new file mode 100644 index 000000000..56f4cedb3 --- /dev/null +++ b/core/src/node/api/common/adapter.ts @@ -0,0 +1,43 @@ +import { + AppRoute, + DownloadRoute, + ExtensionRoute, + FileManagerRoute, + FileSystemRoute, +} from '../../../api' +import { Downloader } from '../processors/download' +import { FileSystem } from '../processors/fs' +import { Extension } from '../processors/extension' +import { FSExt } from '../processors/fsExt' +import { App } from '../processors/app' + +export class RequestAdapter { + downloader: Downloader + fileSystem: FileSystem + extension: Extension + fsExt: FSExt + app: App + + constructor(observer?: Function) { + this.downloader = new Downloader(observer) + this.fileSystem = new FileSystem() + this.extension = new Extension() + this.fsExt = new FSExt() + this.app = new App() + } + + // TODO: Clearer Factory pattern here + process(route: string, ...args: any) { + if (route in DownloadRoute) { + return this.downloader.process(route, ...args) + } else if (route in FileSystemRoute) { + return this.fileSystem.process(route, ...args) + } else if (route in ExtensionRoute) { + return this.extension.process(route, ...args) + } else if (route in FileManagerRoute) { + return this.fsExt.process(route, ...args) + } else if (route in AppRoute) { + return this.app.process(route, ...args) + } + } +} diff --git a/core/src/node/api/common/handler.ts b/core/src/node/api/common/handler.ts new file mode 100644 index 000000000..4a39ae52a --- /dev/null +++ b/core/src/node/api/common/handler.ts @@ -0,0 +1,23 @@ +import { CoreRoutes } from '../../../api' +import { RequestAdapter } from './adapter' + +export type Handler = (route: string, args: any) => any + +export class RequestHandler { + handler: Handler + adataper: RequestAdapter + + constructor(handler: Handler, observer?: Function) { + this.handler = handler + this.adataper = new RequestAdapter(observer) + } + + handle() { + CoreRoutes.map((route) => { + this.handler(route, async (...args: any[]) => { + const values = await this.adataper.process(route, ...args) + return values + }) + }) + } +} diff --git a/core/src/node/api/index.ts b/core/src/node/api/index.ts index 4c3041ba3..ab0c51656 100644 --- a/core/src/node/api/index.ts +++ b/core/src/node/api/index.ts @@ -1,2 +1,3 @@ export * from './HttpServer' -export * from './routes' +export * from './restful/v1' +export * from './common/handler' diff --git a/core/src/node/api/processors/Processor.ts b/core/src/node/api/processors/Processor.ts new file mode 100644 index 000000000..8ef0c6e19 --- /dev/null +++ b/core/src/node/api/processors/Processor.ts @@ -0,0 +1,3 @@ +export abstract class Processor { + abstract process(key: string, ...args: any[]): any +} diff --git a/core/src/node/api/processors/app.ts b/core/src/node/api/processors/app.ts new file mode 100644 index 000000000..c62b5011d --- /dev/null +++ b/core/src/node/api/processors/app.ts @@ -0,0 +1,96 @@ +import { basename, isAbsolute, join, relative } from 'path' + +import { Processor } from './Processor' +import { getAppConfigurations as appConfiguration, updateAppConfiguration } from '../../helper' +import { log as writeLog, logServer as writeServerLog } from '../../helper/log' +import { appResourcePath } from '../../helper/path' + +export class App implements Processor { + observer?: Function + + constructor(observer?: Function) { + this.observer = observer + } + + process(key: string, ...args: any[]): any { + const instance = this as any + const func = instance[key] + return func(...args) + } + + /** + * Joins multiple paths together, respect to the current OS. + */ + joinPath(args: any[]) { + return join(...args) + } + + /** + * Checks if the given path is a subdirectory of the given directory. + * + * @param _event - The IPC event object. + * @param from - The path to check. + * @param to - The directory to check against. + * + * @returns {Promise} - A promise that resolves with the result. + */ + isSubdirectory(from: any, to: any) { + const rel = relative(from, to) + const isSubdir = rel && !rel.startsWith('..') && !isAbsolute(rel) + + if (isSubdir === '') return false + else return isSubdir + } + + /** + * Retrieve basename from given path, respect to the current OS. + */ + baseName(args: any) { + return basename(args) + } + + /** + * Log message to log file. + */ + log(args: any) { + writeLog(args) + } + + /** + * Log message to log file. + */ + logServer(args: any) { + writeServerLog(args) + } + + getAppConfigurations() { + return appConfiguration() + } + + async updateAppConfiguration(args: any) { + await updateAppConfiguration(args) + } + + /** + * Start Jan API Server. + */ + async startServer(args?: any) { + const { startServer } = require('@janhq/server') + return startServer({ + host: args?.host, + port: args?.port, + isCorsEnabled: args?.isCorsEnabled, + isVerboseEnabled: args?.isVerboseEnabled, + schemaPath: join(await appResourcePath(), 'docs', 'openapi', 'jan.yaml'), + baseDir: join(await appResourcePath(), 'docs', 'openapi'), + }) + } + + /** + * Stop Jan API Server. + */ + stopServer() { + const { stopServer } = require('@janhq/server') + return stopServer() + } +} diff --git a/core/src/node/api/processors/download.ts b/core/src/node/api/processors/download.ts new file mode 100644 index 000000000..686ba58a1 --- /dev/null +++ b/core/src/node/api/processors/download.ts @@ -0,0 +1,106 @@ +import { resolve, sep } from 'path' +import { DownloadEvent } from '../../../api' +import { normalizeFilePath } from '../../helper/path' +import { getJanDataFolderPath } from '../../helper' +import { DownloadManager } from '../../helper/download' +import { createWriteStream, renameSync } from 'fs' +import { Processor } from './Processor' +import { DownloadState } from '../../../types' + +export class Downloader implements Processor { + observer?: Function + + constructor(observer?: Function) { + this.observer = observer + } + + process(key: string, ...args: any[]): any { + const instance = this as any + const func = instance[key] + return func(this.observer, ...args) + } + + downloadFile(observer: any, url: string, localPath: string, network: any) { + const request = require('request') + const progress = require('request-progress') + + const strictSSL = !network?.ignoreSSL + const proxy = network?.proxy?.startsWith('http') ? network.proxy : undefined + if (typeof localPath === 'string') { + localPath = normalizeFilePath(localPath) + } + const array = localPath.split(sep) + const fileName = array.pop() ?? '' + const modelId = array.pop() ?? '' + + const destination = resolve(getJanDataFolderPath(), localPath) + const rq = request({ url, strictSSL, proxy }) + + // Put request to download manager instance + DownloadManager.instance.setRequest(localPath, rq) + + // Downloading file to a temp file first + const downloadingTempFile = `${destination}.download` + + progress(rq, {}) + .on('progress', (state: any) => { + const downloadState: DownloadState = { + ...state, + modelId, + fileName, + downloadState: 'downloading', + } + console.log('progress: ', downloadState) + observer?.(DownloadEvent.onFileDownloadUpdate, downloadState) + DownloadManager.instance.downloadProgressMap[modelId] = downloadState + }) + .on('error', (error: Error) => { + const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] + const downloadState: DownloadState = { + ...currentDownloadState, + error: error.message, + downloadState: 'error', + } + if (currentDownloadState) { + DownloadManager.instance.downloadProgressMap[modelId] = downloadState + } + + observer?.(DownloadEvent.onFileDownloadError, downloadState) + }) + .on('end', () => { + const currentDownloadState = DownloadManager.instance.downloadProgressMap[modelId] + if (currentDownloadState && DownloadManager.instance.networkRequests[localPath]) { + // Finished downloading, rename temp file to actual file + renameSync(downloadingTempFile, destination) + const downloadState: DownloadState = { + ...currentDownloadState, + downloadState: 'end', + } + observer?.(DownloadEvent.onFileDownloadSuccess, downloadState) + DownloadManager.instance.downloadProgressMap[modelId] = downloadState + } + }) + .pipe(createWriteStream(downloadingTempFile)) + } + + abortDownload(observer: any, fileName: string) { + const rq = DownloadManager.instance.networkRequests[fileName] + if (rq) { + DownloadManager.instance.networkRequests[fileName] = undefined + rq?.abort() + } else { + observer?.(DownloadEvent.onFileDownloadError, { + fileName, + error: 'aborted', + }) + } + } + + resumeDownload(observer: any, fileName: any) { + DownloadManager.instance.networkRequests[fileName]?.resume() + } + + pauseDownload(observer: any, fileName: any) { + DownloadManager.instance.networkRequests[fileName]?.pause() + } +} diff --git a/core/src/node/api/processors/extension.ts b/core/src/node/api/processors/extension.ts new file mode 100644 index 000000000..df5d2d945 --- /dev/null +++ b/core/src/node/api/processors/extension.ts @@ -0,0 +1,88 @@ +import { readdirSync } from 'fs' +import { join, extname } from 'path' + +import { Processor } from './Processor' +import { ModuleManager } from '../../helper/module' +import { getJanExtensionsPath as getPath } from '../../helper' +import { + getActiveExtensions as getExtensions, + getExtension, + removeExtension, + installExtensions, +} from '../../extension/store' +import { appResourcePath } from '../../helper/path' + +export class Extension implements Processor { + observer?: Function + + constructor(observer?: Function) { + this.observer = observer + } + + process(key: string, ...args: any[]): any { + const instance = this as any + const func = instance[key] + return func(...args) + } + + invokeExtensionFunc(modulePath: string, method: string, ...params: any[]) { + const module = require(join(getPath(), modulePath)) + ModuleManager.instance.setModule(modulePath, module) + + if (typeof module[method] === 'function') { + return module[method](...params) + } else { + console.debug(module[method]) + console.error(`Function "${method}" does not exist in the module.`) + } + } + + /** + * Returns the paths of the base extensions. + * @returns An array of paths to the base extensions. + */ + async baseExtensions() { + const baseExtensionPath = join(await appResourcePath(), 'pre-install') + return readdirSync(baseExtensionPath) + .filter((file) => extname(file) === '.tgz') + .map((file) => join(baseExtensionPath, file)) + } + + /**MARK: Extension Manager handlers */ + async installExtension(extensions: any) { + // Install and activate all provided extensions + const installed = await installExtensions(extensions) + return JSON.parse(JSON.stringify(installed)) + } + + // Register IPC route to uninstall a extension + async uninstallExtension(extensions: any) { + // Uninstall all provided extensions + for (const ext of extensions) { + const extension = getExtension(ext) + await extension.uninstall() + if (extension.name) removeExtension(extension.name) + } + + // Reload all renderer pages if needed + return true + } + + // Register IPC route to update a extension + async updateExtension(extensions: any) { + // Update all provided extensions + const updated: any[] = [] + for (const ext of extensions) { + const extension = getExtension(ext) + const res = await extension.update() + if (res) updated.push(extension) + } + + // Reload all renderer pages if needed + return JSON.parse(JSON.stringify(updated)) + } + + getActiveExtensions() { + return JSON.parse(JSON.stringify(getExtensions())) + } +} diff --git a/core/src/node/api/processors/fs.ts b/core/src/node/api/processors/fs.ts new file mode 100644 index 000000000..93a5f1905 --- /dev/null +++ b/core/src/node/api/processors/fs.ts @@ -0,0 +1,25 @@ +import { join } from 'path' +import { normalizeFilePath } from '../../helper/path' +import { getJanDataFolderPath } from '../../helper' +import { Processor } from './Processor' + +export class FileSystem implements Processor { + observer?: Function + private static moduleName = 'fs' + + constructor(observer?: Function) { + this.observer = observer + } + + process(route: string, ...args: any[]): any { + return import(FileSystem.moduleName).then((mdl) => + mdl[route]( + ...args.map((arg: any) => + typeof arg === 'string' && (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) + ? join(getJanDataFolderPath(), normalizeFilePath(arg)) + : arg + ) + ) + ) + } +} diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts new file mode 100644 index 000000000..71e07ae57 --- /dev/null +++ b/core/src/node/api/processors/fsExt.ts @@ -0,0 +1,78 @@ +import { join } from 'path' +import fs from 'fs' +import { FileManagerRoute } from '../../../api' +import { appResourcePath, normalizeFilePath } from '../../helper/path' +import { getJanDataFolderPath, getJanDataFolderPath as getPath } from '../../helper' +import { Processor } from './Processor' +import { FileStat } from '../../../types' + +export class FSExt implements Processor { + observer?: Function + + constructor(observer?: Function) { + this.observer = observer + } + + process(key: string, ...args: any): any { + const instance = this as any + const func = instance[key] + return func(...args) + } + + // Handles the 'syncFile' IPC event. This event is triggered to synchronize a file from a source path to a destination path. + syncFile(src: string, dest: string) { + const reflect = require('@alumna/reflect') + return reflect({ + src, + dest, + recursive: true, + delete: false, + overwrite: true, + errorOnExist: false, + }) + } + + // Handles the 'getJanDataFolderPath' IPC event. This event is triggered to get the user space path. + getJanDataFolderPath() { + return Promise.resolve(getPath()) + } + + // Handles the 'getResourcePath' IPC event. This event is triggered to get the resource path. + getResourcePath() { + return appResourcePath() + } + + // Handles the 'getUserHomePath' IPC event. This event is triggered to get the user home path. + getUserHomePath() { + return process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'] + } + + // handle fs is directory here + fileStat(path: string) { + const normalizedPath = normalizeFilePath(path) + + const fullPath = join(getJanDataFolderPath(), normalizedPath) + const isExist = fs.existsSync(fullPath) + if (!isExist) return undefined + + const isDirectory = fs.lstatSync(fullPath).isDirectory() + const size = fs.statSync(fullPath).size + + const fileStat: FileStat = { + isDirectory, + size, + } + + return fileStat + } + + writeBlob(path: string, data: any) { + try { + const normalizedPath = normalizeFilePath(path) + const dataBuffer = Buffer.from(data, 'base64') + fs.writeFileSync(join(getJanDataFolderPath(), normalizedPath), dataBuffer) + } catch (err) { + console.error(`writeFile ${path} result: ${err}`) + } + } +} diff --git a/core/src/node/api/restful/app/download.ts b/core/src/node/api/restful/app/download.ts new file mode 100644 index 000000000..b5919659b --- /dev/null +++ b/core/src/node/api/restful/app/download.ts @@ -0,0 +1,23 @@ +import { DownloadRoute } from '../../../../api' +import { DownloadManager } from '../../../helper/download' +import { HttpServer } from '../../HttpServer' + +export const downloadRouter = async (app: HttpServer) => { + app.get(`/download/${DownloadRoute.getDownloadProgress}/:modelId`, async (req, res) => { + const modelId = req.params.modelId + + console.debug(`Getting download progress for model ${modelId}`) + console.debug( + `All Download progress: ${JSON.stringify(DownloadManager.instance.downloadProgressMap)}` + ) + + // check if null DownloadManager.instance.downloadProgressMap + if (!DownloadManager.instance.downloadProgressMap[modelId]) { + return res.status(404).send({ + message: 'Download progress not found', + }) + } else { + return res.status(200).send(DownloadManager.instance.downloadProgressMap[modelId]) + } + }) +} diff --git a/core/src/node/api/restful/app/handlers.ts b/core/src/node/api/restful/app/handlers.ts new file mode 100644 index 000000000..43c3f7add --- /dev/null +++ b/core/src/node/api/restful/app/handlers.ts @@ -0,0 +1,13 @@ +import { HttpServer } from '../../HttpServer' +import { Handler, RequestHandler } from '../../common/handler' + +export function handleRequests(app: HttpServer) { + const restWrapper: Handler = (route: string, listener: (...args: any[]) => any) => { + app.post(`/app/${route}`, async (request: any, reply: any) => { + const args = JSON.parse(request.body) as any[] + reply.send(JSON.stringify(await listener(...args))) + }) + } + const handler = new RequestHandler(restWrapper) + handler.handle() +} diff --git a/core/src/node/api/routes/common.ts b/core/src/node/api/restful/common.ts similarity index 54% rename from core/src/node/api/routes/common.ts rename to core/src/node/api/restful/common.ts index 8887755fe..433632989 100644 --- a/core/src/node/api/routes/common.ts +++ b/core/src/node/api/restful/common.ts @@ -1,24 +1,34 @@ -import { AppRoute } from '../../../api' import { HttpServer } from '../HttpServer' -import { basename, join } from 'path' import { chatCompletions, deleteBuilder, downloadModel, getBuilder, retrieveBuilder, -} from '../common/builder' + createMessage, + createThread, + getMessages, + retrieveMessage, + updateThread, +} from './helper/builder' -import { JanApiRouteConfiguration } from '../common/configuration' -import { startModel, stopModel } from '../common/startStopModel' +import { JanApiRouteConfiguration } from './helper/configuration' +import { startModel, stopModel } from './helper/startStopModel' import { ModelSettingParams } from '../../../types' -import { getJanDataFolderPath } from '../../utils' -import { normalizeFilePath } from '../../path' export const commonRouter = async (app: HttpServer) => { + const normalizeData = (data: any) => { + return { + object: 'list', + data, + } + } // Common Routes + // Read & Delete :: Threads | Models | Assistants Object.keys(JanApiRouteConfiguration).forEach((key) => { - app.get(`/${key}`, async (_request) => getBuilder(JanApiRouteConfiguration[key])) + app.get(`/${key}`, async (_request) => + getBuilder(JanApiRouteConfiguration[key]).then(normalizeData) + ) app.get(`/${key}/:id`, async (request: any) => retrieveBuilder(JanApiRouteConfiguration[key], request.params.id) @@ -29,7 +39,26 @@ export const commonRouter = async (app: HttpServer) => { ) }) - // Download Model Routes + // Threads + app.post(`/threads/`, async (req, res) => createThread(req.body)) + + app.get(`/threads/:threadId/messages`, async (req, res) => + getMessages(req.params.threadId).then(normalizeData) + ) + + app.get(`/threads/:threadId/messages/:messageId`, async (req, res) => + retrieveMessage(req.params.threadId, req.params.messageId) + ) + + app.post(`/threads/:threadId/messages`, async (req, res) => + createMessage(req.params.threadId as any, req.body as any) + ) + + app.patch(`/threads/:threadId`, async (request: any) => + updateThread(request.params.threadId, request.body) + ) + + // Models app.get(`/models/download/:modelId`, async (request: any) => downloadModel(request.params.modelId, { ignoreSSL: request.query.ignoreSSL === 'true', @@ -48,24 +77,6 @@ export const commonRouter = async (app: HttpServer) => { app.put(`/models/:modelId/stop`, async (request: any) => stopModel(request.params.modelId)) - // Chat Completion Routes + // Chat Completion app.post(`/chat/completions`, async (request: any, reply: any) => chatCompletions(request, reply)) - - // App Routes - app.post(`/app/${AppRoute.joinPath}`, async (request: any, reply: any) => { - const args = JSON.parse(request.body) as any[] - - const paths = args[0].map((arg: string) => - typeof arg === 'string' && (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) - ? join(getJanDataFolderPath(), normalizeFilePath(arg)) - : arg - ) - - reply.send(JSON.stringify(join(...paths))) - }) - - app.post(`/app/${AppRoute.baseName}`, async (request: any, reply: any) => { - const args = JSON.parse(request.body) as any[] - reply.send(JSON.stringify(basename(args[0]))) - }) } diff --git a/core/src/node/api/common/builder.ts b/core/src/node/api/restful/helper/builder.ts similarity index 96% rename from core/src/node/api/common/builder.ts rename to core/src/node/api/restful/helper/builder.ts index 5c99cf4d8..7001c0c76 100644 --- a/core/src/node/api/common/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -1,10 +1,11 @@ import fs from 'fs' import { JanApiRouteConfiguration, RouteConfiguration } from './configuration' import { join } from 'path' -import { ContentType, MessageStatus, Model, ThreadMessage } from './../../../index' -import { getEngineConfiguration, getJanDataFolderPath } from '../../utils' +import { ContentType, MessageStatus, Model, ThreadMessage } from '../../../../index' +import { getEngineConfiguration, getJanDataFolderPath } from '../../../helper' import { DEFAULT_CHAT_COMPLETION_URL } from './consts' +// TODO: Refactor these export const getBuilder = async (configuration: RouteConfiguration) => { const directoryPath = join(getJanDataFolderPath(), configuration.dirName) try { @@ -124,7 +125,7 @@ export const getMessages = async (threadId: string): Promise => } } -export const retrieveMesasge = async (threadId: string, messageId: string) => { +export const retrieveMessage = async (threadId: string, messageId: string) => { const messages = await getMessages(threadId) const filteredMessages = messages.filter((m) => m.id === messageId) if (!filteredMessages || filteredMessages.length === 0) { @@ -317,13 +318,6 @@ export const chatCompletions = async (request: any, reply: any) => { apiUrl = engineConfiguration.full_url } - reply.raw.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - 'Access-Control-Allow-Origin': '*', - }) - const headers: Record = { 'Content-Type': 'application/json', } @@ -342,8 +336,14 @@ export const chatCompletions = async (request: any, reply: any) => { }) if (response.status !== 200) { console.error(response) - return + reply.code(400).send(response) } else { + reply.raw.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Access-Control-Allow-Origin': '*', + }) response.body.pipe(reply.raw) } } diff --git a/core/src/node/api/common/configuration.ts b/core/src/node/api/restful/helper/configuration.ts similarity index 100% rename from core/src/node/api/common/configuration.ts rename to core/src/node/api/restful/helper/configuration.ts diff --git a/core/src/node/api/common/consts.ts b/core/src/node/api/restful/helper/consts.ts similarity index 100% rename from core/src/node/api/common/consts.ts rename to core/src/node/api/restful/helper/consts.ts diff --git a/core/src/node/api/common/startStopModel.ts b/core/src/node/api/restful/helper/startStopModel.ts similarity index 99% rename from core/src/node/api/common/startStopModel.ts rename to core/src/node/api/restful/helper/startStopModel.ts index 0d4934e1c..0e6972b0b 100644 --- a/core/src/node/api/common/startStopModel.ts +++ b/core/src/node/api/restful/helper/startStopModel.ts @@ -1,9 +1,9 @@ import fs from 'fs' import { join } from 'path' -import { getJanDataFolderPath, getJanExtensionsPath, getSystemResourceInfo } from '../../utils' -import { logServer } from '../../log' +import { getJanDataFolderPath, getJanExtensionsPath, getSystemResourceInfo } from '../../../helper' +import { logServer } from '../../../helper/log' import { ChildProcessWithoutNullStreams, spawn } from 'child_process' -import { Model, ModelSettingParams, PromptTemplate } from '../../../types' +import { Model, ModelSettingParams, PromptTemplate } from '../../../../types' import { LOCAL_HOST, NITRO_DEFAULT_PORT, diff --git a/core/src/node/api/restful/v1.ts b/core/src/node/api/restful/v1.ts new file mode 100644 index 000000000..5eb8f5067 --- /dev/null +++ b/core/src/node/api/restful/v1.ts @@ -0,0 +1,16 @@ +import { HttpServer } from '../HttpServer' +import { commonRouter } from './common' +import { downloadRouter } from './app/download' +import { handleRequests } from './app/handlers' + +export const v1Router = async (app: HttpServer) => { + // MARK: Public API Routes + app.register(commonRouter) + + // MARK: Internal Application Routes + handleRequests(app) + + // Expanded route for tracking download progress + // TODO: Replace by Observer Wrapper (ZeroMQ / Vanilla Websocket) + app.register(downloadRouter) +} diff --git a/core/src/node/api/routes/download.ts b/core/src/node/api/routes/download.ts deleted file mode 100644 index ab8c0bd37..000000000 --- a/core/src/node/api/routes/download.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { DownloadRoute } from '../../../api' -import { join } from 'path' -import { DownloadManager } from '../../download' -import { HttpServer } from '../HttpServer' -import { createWriteStream } from 'fs' -import { getJanDataFolderPath } from '../../utils' -import { normalizeFilePath } from '../../path' - -export const downloadRouter = async (app: HttpServer) => { - app.post(`/${DownloadRoute.downloadFile}`, async (req, res) => { - const strictSSL = !(req.query.ignoreSSL === 'true') - const proxy = req.query.proxy?.startsWith('http') ? req.query.proxy : undefined - const body = JSON.parse(req.body as any) - const normalizedArgs = body.map((arg: any) => { - if (typeof arg === 'string' && arg.startsWith('file:')) { - return join(getJanDataFolderPath(), normalizeFilePath(arg)) - } - return arg - }) - - const localPath = normalizedArgs[1] - const fileName = localPath.split('/').pop() ?? '' - - const request = require('request') - const progress = require('request-progress') - - const rq = request({ url: normalizedArgs[0], strictSSL, proxy }) - progress(rq, {}) - .on('progress', function (state: any) { - console.log('download onProgress', state) - }) - .on('error', function (err: Error) { - console.log('download onError', err) - }) - .on('end', function () { - console.log('download onEnd') - }) - .pipe(createWriteStream(normalizedArgs[1])) - - DownloadManager.instance.setRequest(fileName, rq) - }) - - app.post(`/${DownloadRoute.abortDownload}`, async (req, res) => { - const body = JSON.parse(req.body as any) - const normalizedArgs = body.map((arg: any) => { - if (typeof arg === 'string' && arg.startsWith('file:')) { - return join(getJanDataFolderPath(), normalizeFilePath(arg)) - } - return arg - }) - - const localPath = normalizedArgs[0] - const fileName = localPath.split('/').pop() ?? '' - const rq = DownloadManager.instance.networkRequests[fileName] - DownloadManager.instance.networkRequests[fileName] = undefined - rq?.abort() - }) -} diff --git a/core/src/node/api/routes/extension.ts b/core/src/node/api/routes/extension.ts deleted file mode 100644 index 02bc54eb3..000000000 --- a/core/src/node/api/routes/extension.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { join, extname } from 'path' -import { ExtensionRoute } from '../../../api/index' -import { ModuleManager } from '../../module' -import { getActiveExtensions, installExtensions } from '../../extension/store' -import { HttpServer } from '../HttpServer' - -import { readdirSync } from 'fs' -import { getJanExtensionsPath } from '../../utils' - -export const extensionRouter = async (app: HttpServer) => { - // TODO: Share code between node projects - app.post(`/${ExtensionRoute.getActiveExtensions}`, async (_req, res) => { - const activeExtensions = await getActiveExtensions() - res.status(200).send(activeExtensions) - }) - - app.post(`/${ExtensionRoute.baseExtensions}`, async (_req, res) => { - const baseExtensionPath = join(__dirname, '..', '..', '..', 'pre-install') - const extensions = readdirSync(baseExtensionPath) - .filter((file) => extname(file) === '.tgz') - .map((file) => join(baseExtensionPath, file)) - - res.status(200).send(extensions) - }) - - app.post(`/${ExtensionRoute.installExtension}`, async (req) => { - const extensions = req.body as any - const installed = await installExtensions(JSON.parse(extensions)[0]) - return JSON.parse(JSON.stringify(installed)) - }) - - app.post(`/${ExtensionRoute.invokeExtensionFunc}`, async (req, res) => { - const args = JSON.parse(req.body as any) - console.debug(args) - const module = await import(join(getJanExtensionsPath(), args[0])) - - ModuleManager.instance.setModule(args[0], module) - const method = args[1] - if (typeof module[method] === 'function') { - // remove first item from args - const newArgs = args.slice(2) - console.log(newArgs) - return module[method](...args.slice(2)) - } else { - console.debug(module[method]) - console.error(`Function "${method}" does not exist in the module.`) - } - }) -} diff --git a/core/src/node/api/routes/fileManager.ts b/core/src/node/api/routes/fileManager.ts deleted file mode 100644 index 66056444e..000000000 --- a/core/src/node/api/routes/fileManager.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { FileManagerRoute } from '../../../api' -import { HttpServer } from '../../index' - -export const fsRouter = async (app: HttpServer) => { - app.post(`/app/${FileManagerRoute.syncFile}`, async (request: any, reply: any) => {}) - - app.post(`/app/${FileManagerRoute.getJanDataFolderPath}`, async (request: any, reply: any) => {}) - - app.post(`/app/${FileManagerRoute.getResourcePath}`, async (request: any, reply: any) => {}) - - app.post(`/app/${FileManagerRoute.getUserHomePath}`, async (request: any, reply: any) => {}) - - app.post(`/app/${FileManagerRoute.fileStat}`, async (request: any, reply: any) => {}) -} diff --git a/core/src/node/api/routes/fs.ts b/core/src/node/api/routes/fs.ts deleted file mode 100644 index c5404ccce..000000000 --- a/core/src/node/api/routes/fs.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { FileSystemRoute } from '../../../api' -import { join } from 'path' -import { HttpServer } from '../HttpServer' -import { getJanDataFolderPath } from '../../utils' -import { normalizeFilePath } from '../../path' - -export const fsRouter = async (app: HttpServer) => { - const moduleName = 'fs' - // Generate handlers for each fs route - Object.values(FileSystemRoute).forEach((route) => { - app.post(`/${route}`, async (req, res) => { - const body = JSON.parse(req.body as any) - try { - const result = await import(moduleName).then((mdl) => { - return mdl[route]( - ...body.map((arg: any) => - typeof arg === 'string' && (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) - ? join(getJanDataFolderPath(), normalizeFilePath(arg)) - : arg - ) - ) - }) - res.status(200).send(result) - } catch (ex) { - console.log(ex) - } - }) - }) -} diff --git a/core/src/node/api/routes/index.ts b/core/src/node/api/routes/index.ts deleted file mode 100644 index e6edc62f7..000000000 --- a/core/src/node/api/routes/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './download' -export * from './extension' -export * from './fs' -export * from './thread' -export * from './common' -export * from './v1' diff --git a/core/src/node/api/routes/thread.ts b/core/src/node/api/routes/thread.ts deleted file mode 100644 index 4066d2716..000000000 --- a/core/src/node/api/routes/thread.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { HttpServer } from '../HttpServer' -import { - createMessage, - createThread, - getMessages, - retrieveMesasge, - updateThread, -} from '../common/builder' - -export const threadRouter = async (app: HttpServer) => { - // create thread - app.post(`/`, async (req, res) => createThread(req.body)) - - app.get(`/:threadId/messages`, async (req, res) => getMessages(req.params.threadId)) - - // retrieve message - app.get(`/:threadId/messages/:messageId`, async (req, res) => - retrieveMesasge(req.params.threadId, req.params.messageId), - ) - - // create message - app.post(`/:threadId/messages`, async (req, res) => - createMessage(req.params.threadId as any, req.body as any), - ) - - // modify thread - app.patch(`/:threadId`, async (request: any) => - updateThread(request.params.threadId, request.body), - ) -} diff --git a/core/src/node/api/routes/v1.ts b/core/src/node/api/routes/v1.ts deleted file mode 100644 index a2a48cd8b..000000000 --- a/core/src/node/api/routes/v1.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { HttpServer } from '../HttpServer' -import { commonRouter } from './common' -import { threadRouter } from './thread' -import { fsRouter } from './fs' -import { extensionRouter } from './extension' -import { downloadRouter } from './download' - -export const v1Router = async (app: HttpServer) => { - // MARK: External Routes - app.register(commonRouter) - app.register(threadRouter, { - prefix: '/threads', - }) - - // MARK: Internal Application Routes - app.register(fsRouter, { - prefix: '/fs', - }) - app.register(extensionRouter, { - prefix: '/extension', - }) - app.register(downloadRouter, { - prefix: '/download', - }) -} diff --git a/core/src/node/extension/extension.ts b/core/src/node/extension/extension.ts index aeb0277c0..1f8dfa3ec 100644 --- a/core/src/node/extension/extension.ts +++ b/core/src/node/extension/extension.ts @@ -104,7 +104,7 @@ export default class Extension { await pacote.extract( this.specifier, join(ExtensionManager.instance.getExtensionsPath() ?? '', this.name ?? ''), - this.installOptions, + this.installOptions ) // Set the url using the custom extensions protocol diff --git a/core/src/node/extension/index.ts b/core/src/node/extension/index.ts index ed8544773..994fc97f2 100644 --- a/core/src/node/extension/index.ts +++ b/core/src/node/extension/index.ts @@ -41,8 +41,8 @@ async function registerExtensionProtocol() { console.error('Electron is not available') } const extensionPath = ExtensionManager.instance.getExtensionsPath() - if (electron) { - return electron.protocol.registerFileProtocol('extension', (request: any, callback: any) => { + if (electron && electron.protocol) { + return electron.protocol?.registerFileProtocol('extension', (request: any, callback: any) => { const entry = request.url.substr('extension://'.length - 1) const url = normalize(extensionPath + entry) @@ -69,7 +69,7 @@ export function useExtensions(extensionsPath: string) { // Read extension list from extensions folder const extensions = JSON.parse( - readFileSync(ExtensionManager.instance.getExtensionsFile(), 'utf-8'), + readFileSync(ExtensionManager.instance.getExtensionsFile(), 'utf-8') ) try { // Create and store a Extension instance for each extension in list @@ -82,7 +82,7 @@ export function useExtensions(extensionsPath: string) { throw new Error( 'Could not successfully rebuild list of installed extensions.\n' + error + - '\nPlease check the extensions.json file in the extensions folder.', + '\nPlease check the extensions.json file in the extensions folder.' ) } @@ -122,7 +122,7 @@ function loadExtension(ext: any) { export function getStore() { if (!ExtensionManager.instance.getExtensionsFile()) { throw new Error( - 'The extension path has not yet been set up. Please run useExtensions before accessing the store', + 'The extension path has not yet been set up. Please run useExtensions before accessing the store' ) } diff --git a/core/src/node/extension/store.ts b/core/src/node/extension/store.ts index 84b1f9caf..93b1aeb2b 100644 --- a/core/src/node/extension/store.ts +++ b/core/src/node/extension/store.ts @@ -1,6 +1,6 @@ -import { writeFileSync } from "fs"; -import Extension from "./extension"; -import { ExtensionManager } from "./manager"; +import { writeFileSync } from 'fs' +import Extension from './extension' +import { ExtensionManager } from './manager' /** * @module store @@ -11,7 +11,7 @@ import { ExtensionManager } from "./manager"; * Register of installed extensions * @type {Object.} extension - List of installed extensions */ -const extensions: Record = {}; +const extensions: Record = {} /** * Get a extension from the stored extensions. @@ -21,10 +21,10 @@ const extensions: Record = {}; */ export function getExtension(name: string) { if (!Object.prototype.hasOwnProperty.call(extensions, name)) { - throw new Error(`Extension ${name} does not exist`); + throw new Error(`Extension ${name} does not exist`) } - return extensions[name]; + return extensions[name] } /** @@ -33,7 +33,7 @@ export function getExtension(name: string) { * @alias extensionManager.getAllExtensions */ export function getAllExtensions() { - return Object.values(extensions); + return Object.values(extensions) } /** @@ -42,7 +42,7 @@ export function getAllExtensions() { * @alias extensionManager.getActiveExtensions */ export function getActiveExtensions() { - return Object.values(extensions).filter((extension) => extension.active); + return Object.values(extensions).filter((extension) => extension.active) } /** @@ -53,9 +53,9 @@ export function getActiveExtensions() { * @alias extensionManager.removeExtension */ export function removeExtension(name: string, persist = true) { - const del = delete extensions[name]; - if (persist) persistExtensions(); - return del; + const del = delete extensions[name] + if (persist) persistExtensions() + return del } /** @@ -65,10 +65,10 @@ export function removeExtension(name: string, persist = true) { * @returns {void} */ export function addExtension(extension: Extension, persist = true) { - if (extension.name) extensions[extension.name] = extension; + if (extension.name) extensions[extension.name] = extension if (persist) { - persistExtensions(); - extension.subscribe("pe-persist", persistExtensions); + persistExtensions() + extension.subscribe('pe-persist', persistExtensions) } } @@ -77,14 +77,11 @@ export function addExtension(extension: Extension, persist = true) { * @returns {void} */ export function persistExtensions() { - const persistData: Record = {}; + const persistData: Record = {} for (const name in extensions) { - persistData[name] = extensions[name]; + persistData[name] = extensions[name] } - writeFileSync( - ExtensionManager.instance.getExtensionsFile(), - JSON.stringify(persistData), - ); + writeFileSync(ExtensionManager.instance.getExtensionsFile(), JSON.stringify(persistData)) } /** @@ -94,26 +91,29 @@ export function persistExtensions() { * @returns {Promise.>} New extension * @alias extensionManager.installExtensions */ -export async function installExtensions(extensions: any, store = true) { - const installed: Extension[] = []; +export async function installExtensions(extensions: any) { + const installed: Extension[] = [] for (const ext of extensions) { // Set install options and activation based on input type - const isObject = typeof ext === "object"; - const spec = isObject ? [ext.specifier, ext] : [ext]; - const activate = isObject ? ext.activate !== false : true; + const isObject = typeof ext === 'object' + const spec = isObject ? [ext.specifier, ext] : [ext] + const activate = isObject ? ext.activate !== false : true // Install and possibly activate extension - const extension = new Extension(...spec); - await extension._install(); - if (activate) extension.setActive(true); + const extension = new Extension(...spec) + if (!extension.origin) { + continue + } + await extension._install() + if (activate) extension.setActive(true) // Add extension to store if needed - if (store) addExtension(extension); - installed.push(extension); + addExtension(extension) + installed.push(extension) } // Return list of all installed extensions - return installed; + return installed } /** diff --git a/core/src/node/utils/index.ts b/core/src/node/helper/config.ts similarity index 91% rename from core/src/node/utils/index.ts rename to core/src/node/helper/config.ts index 4bcbf13b1..71e721578 100644 --- a/core/src/node/utils/index.ts +++ b/core/src/node/helper/config.ts @@ -1,8 +1,7 @@ -import { AppConfiguration, SystemResourceInfo } from '../../types' +import { AppConfiguration } from '../../types' import { join } from 'path' import fs from 'fs' import os from 'os' -import { log, logServer } from '../log' import childProcess from 'child_process' // TODO: move this to core @@ -56,34 +55,6 @@ export const updateAppConfiguration = (configuration: AppConfiguration): Promise return Promise.resolve() } -/** - * Utility function to get server log path - * - * @returns {string} The log path. - */ -export const getServerLogPath = (): string => { - const appConfigurations = getAppConfigurations() - const logFolderPath = join(appConfigurations.data_folder, 'logs') - if (!fs.existsSync(logFolderPath)) { - fs.mkdirSync(logFolderPath, { recursive: true }) - } - return join(logFolderPath, 'server.log') -} - -/** - * Utility function to get app log path - * - * @returns {string} The log path. - */ -export const getAppLogPath = (): string => { - const appConfigurations = getAppConfigurations() - const logFolderPath = join(appConfigurations.data_folder, 'logs') - if (!fs.existsSync(logFolderPath)) { - fs.mkdirSync(logFolderPath, { recursive: true }) - } - return join(logFolderPath, 'app.log') -} - /** * Utility function to get data folder path * @@ -146,18 +117,6 @@ const exec = async (command: string): Promise => { }) } -export const getSystemResourceInfo = async (): Promise => { - const cpu = await physicalCpuCount() - const message = `[NITRO]::CPU informations - ${cpu}` - log(message) - logServer(message) - - return { - numCpuPhysicalCore: cpu, - memAvailable: 0, // TODO: this should not be 0 - } -} - export const getEngineConfiguration = async (engineId: string) => { if (engineId !== 'openai') { return undefined @@ -167,3 +126,31 @@ export const getEngineConfiguration = async (engineId: string) => { const data = fs.readFileSync(filePath, 'utf-8') return JSON.parse(data) } + +/** + * Utility function to get server log path + * + * @returns {string} The log path. + */ +export const getServerLogPath = (): string => { + const appConfigurations = getAppConfigurations() + const logFolderPath = join(appConfigurations.data_folder, 'logs') + if (!fs.existsSync(logFolderPath)) { + fs.mkdirSync(logFolderPath, { recursive: true }) + } + return join(logFolderPath, 'server.log') +} + +/** + * Utility function to get app log path + * + * @returns {string} The log path. + */ +export const getAppLogPath = (): string => { + const appConfigurations = getAppConfigurations() + const logFolderPath = join(appConfigurations.data_folder, 'logs') + if (!fs.existsSync(logFolderPath)) { + fs.mkdirSync(logFolderPath, { recursive: true }) + } + return join(logFolderPath, 'app.log') +} diff --git a/core/src/node/download.ts b/core/src/node/helper/download.ts similarity index 67% rename from core/src/node/download.ts rename to core/src/node/helper/download.ts index 6d15fc344..b9fb88bb5 100644 --- a/core/src/node/download.ts +++ b/core/src/node/helper/download.ts @@ -1,15 +1,18 @@ +import { DownloadState } from '../../types' /** * Manages file downloads and network requests. */ export class DownloadManager { - public networkRequests: Record = {}; + public networkRequests: Record = {} - public static instance: DownloadManager = new DownloadManager(); + public static instance: DownloadManager = new DownloadManager() + + public downloadProgressMap: Record = {} constructor() { if (DownloadManager.instance) { - return DownloadManager.instance; + return DownloadManager.instance } } /** @@ -18,6 +21,6 @@ export class DownloadManager { * @param {Request | undefined} request - The network request to set, or undefined to clear the request. */ setRequest(fileName: string, request: any | undefined) { - this.networkRequests[fileName] = request; + this.networkRequests[fileName] = request } } diff --git a/core/src/node/helper/index.ts b/core/src/node/helper/index.ts new file mode 100644 index 000000000..6fc54fc6b --- /dev/null +++ b/core/src/node/helper/index.ts @@ -0,0 +1,6 @@ +export * from './config' +export * from './download' +export * from './log' +export * from './module' +export * from './path' +export * from './resource' diff --git a/core/src/node/log.ts b/core/src/node/helper/log.ts similarity index 93% rename from core/src/node/log.ts rename to core/src/node/helper/log.ts index 6f2c2f80f..8ff196943 100644 --- a/core/src/node/log.ts +++ b/core/src/node/helper/log.ts @@ -1,6 +1,6 @@ import fs from 'fs' import util from 'util' -import { getAppLogPath, getServerLogPath } from './utils' +import { getAppLogPath, getServerLogPath } from './config' export const log = (message: string) => { const path = getAppLogPath() diff --git a/core/src/node/module.ts b/core/src/node/helper/module.ts similarity index 100% rename from core/src/node/module.ts rename to core/src/node/helper/module.ts diff --git a/core/src/node/helper/path.ts b/core/src/node/helper/path.ts new file mode 100644 index 000000000..c20889f4c --- /dev/null +++ b/core/src/node/helper/path.ts @@ -0,0 +1,35 @@ +import { join } from 'path' + +/** + * Normalize file path + * Remove all file protocol prefix + * @param path + * @returns + */ +export function normalizeFilePath(path: string): string { + return path.replace(/^(file:[\\/]+)([^:\s]+)$/, '$2') +} + +export async function appResourcePath(): Promise { + let electron: any = undefined + + try { + const moduleName = 'electron' + electron = await import(moduleName) + } catch (err) { + console.error('Electron is not available') + } + + // electron + if (electron && electron.protocol) { + let appPath = join(electron.app.getAppPath(), '..', 'app.asar.unpacked') + + if (!electron.app.isPackaged) { + // for development mode + appPath = join(electron.app.getAppPath()) + } + return appPath + } + // server + return join(global.core.appPath(), '../../..') +} diff --git a/core/src/node/helper/resource.ts b/core/src/node/helper/resource.ts new file mode 100644 index 000000000..c79a63688 --- /dev/null +++ b/core/src/node/helper/resource.ts @@ -0,0 +1,14 @@ +import { SystemResourceInfo } from '../../types' +import { physicalCpuCount } from './config' +import { log, logServer } from './log' + +export const getSystemResourceInfo = async (): Promise => { + const cpu = await physicalCpuCount() + const message = `[NITRO]::CPU informations - ${cpu}` + log(message) + + return { + numCpuPhysicalCore: cpu, + memAvailable: 0, // TODO: this should not be 0 + } +} diff --git a/core/src/node/index.ts b/core/src/node/index.ts index 10385ecfc..31f2f076e 100644 --- a/core/src/node/index.ts +++ b/core/src/node/index.ts @@ -2,9 +2,5 @@ export * from './extension/index' export * from './extension/extension' export * from './extension/manager' export * from './extension/store' -export * from './download' -export * from './module' export * from './api' -export * from './log' -export * from './utils' -export * from './path' +export * from './helper' diff --git a/core/src/node/path.ts b/core/src/node/path.ts deleted file mode 100644 index adbc38c6c..000000000 --- a/core/src/node/path.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** - * Normalize file path - * Remove all file protocol prefix - * @param path - * @returns - */ -export function normalizeFilePath(path: string): string { - return path.replace(/^(file:[\\/]+)([^:\s]+)$/, "$2"); -} diff --git a/core/src/types/assistant/assistantEvent.ts b/core/src/types/assistant/assistantEvent.ts new file mode 100644 index 000000000..8c32f5d37 --- /dev/null +++ b/core/src/types/assistant/assistantEvent.ts @@ -0,0 +1,7 @@ +/** + * The `EventName` enumeration contains the names of all the available events in the Jan platform. + */ +export enum AssistantEvent { + /** The `OnAssistantsUpdate` event is emitted when the assistant list is updated. */ + OnAssistantsUpdate = 'OnAssistantsUpdate', +} diff --git a/core/src/types/assistant/index.ts b/core/src/types/assistant/index.ts index 83ea73f85..e18589551 100644 --- a/core/src/types/assistant/index.ts +++ b/core/src/types/assistant/index.ts @@ -1,2 +1,3 @@ export * from './assistantEntity' +export * from './assistantEvent' export * from './assistantInterface' diff --git a/core/src/types/file/index.ts b/core/src/types/file/index.ts index 6526cfc6d..cc7274a28 100644 --- a/core/src/types/file/index.ts +++ b/core/src/types/file/index.ts @@ -2,3 +2,26 @@ export type FileStat = { isDirectory: boolean size: number } + +export type DownloadState = { + modelId: string + fileName: string + time: DownloadTime + speed: number + percent: number + + size: DownloadSize + children?: DownloadState[] + error?: string + downloadState: 'downloading' | 'error' | 'end' +} + +type DownloadTime = { + elapsed: number + remaining: number +} + +type DownloadSize = { + total: number + transferred: number +} diff --git a/core/src/types/message/index.ts b/core/src/types/message/index.ts index e8d78deda..ebb4c363d 100644 --- a/core/src/types/message/index.ts +++ b/core/src/types/message/index.ts @@ -1,3 +1,4 @@ export * from './messageEntity' export * from './messageInterface' export * from './messageEvent' +export * from './messageRequestType' diff --git a/core/src/types/message/messageEntity.ts b/core/src/types/message/messageEntity.ts index 87e4b1997..e9211d550 100644 --- a/core/src/types/message/messageEntity.ts +++ b/core/src/types/message/messageEntity.ts @@ -27,6 +27,8 @@ export type ThreadMessage = { updated: number /** The additional metadata of this message. **/ metadata?: Record + + type?: string } /** @@ -56,6 +58,8 @@ export type MessageRequest = { /** The thread of this message is belong to. **/ // TODO: deprecate threadId field thread?: Thread + + type?: string } /** diff --git a/core/src/types/message/messageRequestType.ts b/core/src/types/message/messageRequestType.ts new file mode 100644 index 000000000..cbb4cf421 --- /dev/null +++ b/core/src/types/message/messageRequestType.ts @@ -0,0 +1,5 @@ +export enum MessageRequestType { + Thread = 'Thread', + Assistant = 'Assistant', + Summary = 'Summary', +} diff --git a/core/src/types/model/modelEvent.ts b/core/src/types/model/modelEvent.ts index 978a48724..443f3a34f 100644 --- a/core/src/types/model/modelEvent.ts +++ b/core/src/types/model/modelEvent.ts @@ -12,4 +12,6 @@ export enum ModelEvent { OnModelStop = 'OnModelStop', /** The `OnModelStopped` event is emitted when a model stopped ok. */ OnModelStopped = 'OnModelStopped', + /** The `OnModelUpdate` event is emitted when the model list is updated. */ + OnModelsUpdate = 'OnModelsUpdate', } diff --git a/core/src/types/model/modelInterface.ts b/core/src/types/model/modelInterface.ts index 74a479f3c..93d5867ee 100644 --- a/core/src/types/model/modelInterface.ts +++ b/core/src/types/model/modelInterface.ts @@ -10,7 +10,7 @@ export interface ModelInterface { * @param network - Optional object to specify proxy/whether to ignore SSL certificates. * @returns A Promise that resolves when the model has been downloaded. */ - downloadModel(model: Model, network?: { ignoreSSL?: boolean, proxy?: string }): Promise + downloadModel(model: Model, network?: { ignoreSSL?: boolean; proxy?: string }): Promise /** * Cancels the download of a specific model. diff --git a/core/tests/node/path.test.ts b/core/tests/node/path.test.ts index 9f8a557bb..5390df119 100644 --- a/core/tests/node/path.test.ts +++ b/core/tests/node/path.test.ts @@ -1,4 +1,4 @@ -import { normalizeFilePath } from "../../src/node/path"; +import { normalizeFilePath } from "../../src/node/helper/path"; describe("Test file normalize", () => { test("returns no file protocol prefix on Unix", async () => { diff --git a/core/tslint.json b/core/tslint.json index 398a41670..6543a641a 100644 --- a/core/tslint.json +++ b/core/tslint.json @@ -1,6 +1,3 @@ { - "extends": [ - "tslint-config-standard", - "tslint-config-prettier" - ] -} \ No newline at end of file + "extends": ["tslint-config-standard", "tslint-config-prettier"] +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..1691a841a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,172 @@ +# Docker Compose file for setting up Minio, createbuckets, app_cpu, and app_gpu services + +version: '3.7' + +services: + # Minio service for object storage + minio: + image: minio/minio + volumes: + - minio_data:/data + ports: + - "9000:9000" + - "9001:9001" + environment: + # Set the root user and password for Minio + MINIO_ROOT_USER: minioadmin # This acts as AWS_ACCESS_KEY + MINIO_ROOT_PASSWORD: minioadmin # This acts as AWS_SECRET_ACCESS_KEY + command: server --console-address ":9001" /data + restart: always + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + vpcbr: + ipv4_address: 10.5.0.2 + + # createbuckets service to create a bucket and set its policy + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb myminio/mybucket; + /usr/bin/mc policy set public myminio/mybucket; + exit 0; + " + networks: + vpcbr: + + # app_cpu service for running the CPU version of the application + app_cpu_s3fs: + image: jan:latest + volumes: + - app_data_cpu_s3fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile + environment: + # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_cpu + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + S3_BUCKET_NAME: mybucket + AWS_ENDPOINT: http://10.5.0.2:9000 + AWS_REGION: us-east-1 + API_BASE_URL: http://localhost:1337 + restart: always + profiles: + - cpu-s3fs + ports: + - "3000:3000" + - "1337:1337" + - "3928:3928" + networks: + vpcbr: + ipv4_address: 10.5.0.3 + + # app_gpu service for running the GPU version of the application + app_gpu_s3fs: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + image: jan-gpu:latest + volumes: + - app_data_gpu_s3fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile.gpu + restart: always + environment: + # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_gpu + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + S3_BUCKET_NAME: mybucket + AWS_ENDPOINT: http://10.5.0.2:9000 + AWS_REGION: us-east-1 + API_BASE_URL: http://localhost:1337 + profiles: + - gpu-s3fs + ports: + - "3000:3000" + - "1337:1337" + - "3928:3928" + networks: + vpcbr: + ipv4_address: 10.5.0.4 + + app_cpu_fs: + image: jan:latest + volumes: + - app_data_cpu_fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile + environment: + API_BASE_URL: http://localhost:1337 + restart: always + profiles: + - cpu-fs + ports: + - "3000:3000" + - "1337:1337" + - "3928:3928" + networks: + vpcbr: + ipv4_address: 10.5.0.5 + + # app_gpu service for running the GPU version of the application + app_gpu_fs: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + image: jan-gpu:latest + volumes: + - app_data_gpu_fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile.gpu + restart: always + environment: + API_BASE_URL: http://localhost:1337 + profiles: + - gpu-fs + ports: + - "3000:3000" + - "1337:1337" + - "3928:3928" + networks: + vpcbr: + ipv4_address: 10.5.0.6 + +volumes: + minio_data: + app_data_cpu_s3fs: + app_data_gpu_s3fs: + app_data_cpu_fs: + app_data_gpu_fs: + +networks: + vpcbr: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 + +# Usage: +# - Run 'docker compose --profile cpu-s3fs up -d' to start the app_cpu service +# - Run 'docker compose --profile gpu-s3fs up -d' to start the app_gpu service +# - Run 'docker compose --profile cpu-fs up -d' to start the app_cpu service +# - Run 'docker compose --profile gpu-fs up -d' to start the app_gpu service diff --git a/docs/docs/about/03-engineering/05-postmortems/01-january-10-2024-bitdefender-false-positive-flag.mdx b/docs/blog/01-january-10-2024-bitdefender-false-positive-flag.mdx similarity index 96% rename from docs/docs/about/03-engineering/05-postmortems/01-january-10-2024-bitdefender-false-positive-flag.mdx rename to docs/blog/01-january-10-2024-bitdefender-false-positive-flag.mdx index 4d0189c60..ef418ff97 100644 --- a/docs/docs/about/03-engineering/05-postmortems/01-january-10-2024-bitdefender-false-positive-flag.mdx +++ b/docs/blog/01-january-10-2024-bitdefender-false-positive-flag.mdx @@ -1,22 +1,8 @@ --- -title: | - 10/1/24: Bitdefender False Positive Flag (Resolved) +title: "Post Mortem: Bitdefender False Positive Flag" +description: "10th January 2024, Jan's 0.4.4 Release on Windows triggered Bitdefender to incorrectly flag it as infected with Gen:Variant.Tedy.258323, leading to automatic quarantine warnings on users' computers." slug: /postmortems/january-10-2024-bitdefender-false-positive-flag -description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - postmortem, - incident, - flagging issue, - ] +tags: [Postmortem] --- Following the recent incident related to Jan version 0.4.4 triggering Bitdefender on Windows with Gen:Variant.Tedy.258323 on January 10, 2024, we wanted to provide a comprehensive postmortem and outline the necessary follow-up actions. diff --git a/docs/blog/README.md b/docs/blog/README.md deleted file mode 100644 index 464090415..000000000 --- a/docs/blog/README.md +++ /dev/null @@ -1 +0,0 @@ -# TODO diff --git a/docs/blog/authors.yml b/docs/blog/authors.yml index f30d4610d..ec58002e4 100644 --- a/docs/blog/authors.yml +++ b/docs/blog/authors.yml @@ -1,6 +1,76 @@ dan-jan: name: Daniel Onggunhao title: Co-Founder - url: https://github.com/dan-jan + url: https://github.com/dan-jan image_url: https://avatars.githubusercontent.com/u/101145494?v=4 - email: daniel@jan.ai \ No newline at end of file + email: daniel@jan.ai + +namchuai: + name: Nam Nguyen + title: Developer + url: https://github.com/namchuai + image_url: https://avatars.githubusercontent.com/u/10397206?v=4 + email: james@jan.ai + +hiro-v: + name: Hiro Vuong + title: MLE + url: https://github.com/hiro-v + image_url: https://avatars.githubusercontent.com/u/22463238?v=4 + email: hiro@jan.ai + +ashley-jan: + name: Ashley Tran + title: Product Designer + url: https://github.com/imtuyethan + image_url: https://avatars.githubusercontent.com/u/89722390?v=4 + email: ashley@jan.ai + +hientominh: + name: Hien To + title: DevOps Engineer + url: https://github.com/hientominh + image_url: https://avatars.githubusercontent.com/u/37921427?v=4 + email: hien@jan.ai + +Van-QA: + name: Van Pham + title: QA & Release Manager + url: https://github.com/Van-QA + image_url: https://avatars.githubusercontent.com/u/64197333?v=4 + email: van@jan.ai + +louis-jan: + name: Louis Le + title: Software Engineer + url: https://github.com/louis-jan + image_url: https://avatars.githubusercontent.com/u/133622055?v=4 + email: louis@jan.ai + +hahuyhoang411: + name: Rex Ha + title: LLM Researcher & Content Writer + url: https://github.com/hahuyhoang411 + image_url: https://avatars.githubusercontent.com/u/64120343?v=4 + email: rex@jan.ai + +automaticcat: + name: Alan Dao + title: AI Engineer + url: https://github.com/tikikun + image_url: https://avatars.githubusercontent.com/u/22268502?v=4 + email: alan@jan.ai + +hieu-jan: + name: Henry Ho + title: Software Engineer + url: https://github.com/hieu-jan + image_url: https://avatars.githubusercontent.com/u/150573299?v=4 + email: hieu@jan.ai + +0xsage: + name: Nicole Zhu + title: Co-Founder + url: https://github.com/0xsage + image_url: https://avatars.githubusercontent.com/u/69952136?v=4 + email: nicole@jan.ai diff --git a/docs/docs/about/01-README.md b/docs/docs/about/01-README.md deleted file mode 100644 index d5d3b8dc2..000000000 --- a/docs/docs/about/01-README.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -title: About Jan -slug: /about -description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -Jan believes in the need for an **open source AI ecosystem**. We are focused on building the infra and tooling to allow open source AIs to compete on a level playing field with proprietary ones. - -Jan's long-term technical endeavor is to build a cognitive framework for future robots, who are practical, useful assistants for humans and businesses in everyday life. - -## Quicklinks - -- Core product vision for [Jan Framework](../docs) -- R&D and model training efforts [Discord](https://discord.gg/9NfUSyzp3y) (via our small data-center which is `free & open to all researchers who lack GPUs`!) -- Current implementations of Jan Framework: [Jan Desktop](https://jan.ai/), [Nitro](https://nitro.jan.ai/) - -## Why does Jan Exist? - -### Mission - -Our current mission is to allow humans and businesses to **own their AI, with the right to tinker, repair and innovate**. - -:::tip -Our life-long mission is to **eliminate work - so human can focus on creation, invention, and moral governance over robots**. -::: - -### Ideal Customer - -Our ideal customer is an AI enthusiast or business who has experienced some limitations with current AI solutions and is keen to find open source alternatives. - -### Problems - -Our ideal customer would use Jan to solve one of these problems. - -_Control_ - -- Control (e.g. preventing vendor lock-in) -- Stability (e.g. runs predictably every time) -- Local-use (e.g. for speed, or for airgapped environments) - -_Privacy_ - -- Data protection (e.g. personal data or company data) -- Privacy (e.g. nsfw) - -_Customisability_ - -- Tinkerability (e.g. ability to change model, experiment) -- Niche Models (e.g. fine-tuned, domain-specific models that outperform OpenAI) - -Sources: [^1] [^2] [^3] [^4] - -[^1]: [What are you guys doing that can't be done with ChatGPT?](https://www.reddit.com/r/LocalLLaMA/comments/17mghqr/comment/k7ksti6/?utm_source=share&utm_medium=web2x&context=3) -[^2]: [What's your main interest in running a local LLM instead of an existing API?](https://www.reddit.com/r/LocalLLaMA/comments/1718a9o/whats_your_main_interest_in_running_a_local_llm/) -[^3]: [Ask HN: What's the best self-hosted/local alternative to GPT-4?](https://news.ycombinator.com/item?id=36138224) -[^4]: [LoRAs](https://www.reddit.com/r/LocalLLaMA/comments/17mghqr/comment/k7mdz1i/?utm_source=share&utm_medium=web2x&context=3) - -### Solution - -Jan is a seamless user experience that runs on your personal computer, that glues the different pieces of the open source AI ecosystem to provide an alternative to OpenAI's closed platform. - -- We build a comprehensive, seamless platform that takes care of the technical chores across the stack required to run open source AI -- We run on top of a local folder of non-proprietary files, that anyone can tinker with (yes, even other apps!) -- We provide open formats for packaging and distributing AI to run reproducibly across devices - -## How Jan Works - -### Open Source - -Jan is a startup with an open source business model. We believe in the need for an open source AI ecosystem, and are committed to building it. - -- [Jan Framework](https://github.com/janhq/jan) (AGPLv3) -- [Jan Desktop Client & Local server](https://jan.ai) (AGPLv3, built on Jan Framework) -- [Nitro: run Local AI](https://github.com/janhq/nitro) (AGPLv3) - -### Build in Public - -We use GitHub to build in public and welcome anyone to join in. - -- [Jan's Kanban](https://github.com/orgs/janhq/projects/5) -- [Jan's Roadmap](https://github.com/orgs/janhq/projects/5/views/29) -- [Jan's Newsletter](https://newsletter.jan.ai) - -### Remote Team - -Jan has a fully-remote team. We are mainly based in the APAC timezone. We use [Discord](https://discord.gg/af6SaTdzpx) and [Github](https://github.com/janhq) to work. - -### Bootstrapped - -Jan is currently a bootstrapped, founder funded startup. - -We balance technical invention with the search for a sustainable business model. Thus, we appreciate any business inquiries that can balance growth with cashflow. - -**We invite you to join us on our journey to find PMF**. Join our [Discord here](https://discord.gg/BnHRr3Q7Ms) - -### Analytics - -Adhering to Jan's privacy preserving philosophy, our analytics philosophy is to get `barely-enough-to-function`. - -#### What is tracked - -1. By default, Github tracks downloads and device metadata for all public GitHub repositories. This helps us troubleshoot & ensure cross-platform support. -2. We use [Umami](https://umami.is/) to collect, analyze, and understand application data while maintaining visitor privacy and data ownership. We are using the Umami Cloud in Europe to ensure GDPR compliance. Please see [Umami Privacy Policy](https://umami.is/privacy) for more details. -3. We use Umami to track a single `app.opened` event without additional user metadata, in order to understand retention. In addition, we track `app.event` to understand app version usage. -4. Additionally, we plan to enable a `Settings` feature for users to turn off all tracking. - -#### Request for help - -As a result, our feature prioritization can feel a bit black box at times. - -We'd appreciate high quality insights and volunteers for user interviews through [Discord](https://discord.gg/af6SaTdzpx) and [Github](https://github.com/janhq). - -## Contact - -### General Enquiries - -Drop us a message in our [Discord](https://discord.gg/af6SaTdzpx) and we'll get back to you. - -- `#general`: for general discussion -- `#jan-dev`: for Jan-related questions -- `#nitro-dev`: for Nitro-related questions -- '#bd-inquiries': for enterprise license queries - -### Careers - -Jan has a culture of ownership, independent thought, and lightning fast execution. If you'd like to join us, we have open positions on our [careers page](https://janai.bamboohr.com/careers). - -## Footnotes diff --git a/docs/docs/about/02-onboarding/README.md b/docs/docs/about/02-onboarding/README.md deleted file mode 100644 index bf232c07d..000000000 --- a/docs/docs/about/02-onboarding/README.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -title: Onboarding -description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -slug: /onboarding -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -# Onboarding - -Welcome to Jan! We’re really excited to bring you onboard. - -## Expectations - -- **Take Initiative** Take ownership of an area. If you see a problem, take it and own it to completion. Your work will often not be defined, or poorly defined. Take the initiative to figure out what needs to be done, seek others out for clarification, and then communicate what you will be doing to the team. - -- **Bias to Action** There are many problem-filled areas. There is no need to ask for permission or try to build consensus: just take action. - -- **Speak Up** We require clear, effective and timely communication, which enables others to coordinate with you to be effective. We are a fully distributed, remote team of people from different cultures and languages. If conflicts do arise, first assume Hanlon’s Razor: “Never attribute to malice that which is adequately explained by ~~stupidity~~ lack of communication or too much ambiguity”. Don’t take things personally, be a professional. - -- **Mastery** We are working in a frontier industry, where there are no playbooks, and expertise is developed by doing. Own your area, and build mastery. - -## Code of conduct - -- We operate on the basis of trust. -- We expect you to be available and communicative during scheduled meetings or work hours. -- Turning on video during meetings is encouraged. -- Casual dress during meetings is acceptable; however, use discretion (No nudity, pajamas, etc.) -- While it’s natural for people to disagree at times, disagreement is no excuse for poor behavior and poor manners. We cannot allow that frustration to turn into a personal attack. -- Respect other people's cultures. Especially since we are working in a diverse working culture. -- Sexual harassment is a specific type of prohibited conduct. Sexual harassment is any unwelcome conduct of a sexual nature that might reasonably be expected or be perceived to cause offense or humiliation. Sexual harassment may involve any conduct of a verbal, nonverbal, or physical nature, including written and electronic communications, and may occur between persons of the same or different genders. - -## Onboarding Checklist - -### HR - -- [ ] Service Agreement -- [ ] Equipment Loan Agreement -- [ ] Calendar events - - [ ] Add to Google Team - - [ ] Add to Standup & TGIF -- [ ] `#hr-*` channel -- [ ] BambooHr Log-in -- [ ] Add Emergency Contact in BambooHR - -### Apps you will need - -:::info - -💡 In order to feel connected in a remote workplace, we encourage you to add your profile photo to all the accounts. - -::: - -- Company-wide - - [ ] Google:`[first_name]@jan.ai` - - Recommended: setup on Mobile too (i.e. Calendar, Mail) - - We use Google Calendar for tracking meetings, etc. - - [ ] Discord: - - [Invite link](https://discord.gg/sZb6qxfgyx) to Jan’s Discord - - We use Discord for day-to-day Comms in the company (ala Slack) - - Recommended: setup on Mobile with Notifications - - Download the desktop app - - [ ] 1Password - - [ ] [Jan](https://jan.ai/) - Desktop App -- Engineering - - [ ] Code Editor (such as VSCode, Vim, ect) - - [ ] Github -- Communications - - [ ] Fill in your contact details [here](https://docs.google.com/spreadsheets/d/1KAxya29_wb1bEESiFJeCrOec4pCG3uA2D4_VPgAn89U/edit#gid=0) - - [ ] To make sure everyone in the remote working environment understands more about each other, we encourage you to share your `How to work with me` in the [Drive Tab](https://docs.google.com/spreadsheets/d/1KAxya29_wb1bEESiFJeCrOec4pCG3uA2D4_VPgAn89U/edit#gid=0) under your name and the Discord `Internal` channel. diff --git a/docs/docs/about/03-engineering/03-mlops.md b/docs/docs/about/03-engineering/03-mlops.md deleted file mode 100644 index d3e768a58..000000000 --- a/docs/docs/about/03-engineering/03-mlops.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: MLOps -description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -slug: /engineering/mlops -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -## Connecting to Rigs - -We have a small data rig you can remote into for R&D and CI. - -### Pritunl Setup - -1. **Install Pritunl**: [Download here](https://client.pritunl.com/#install) -2. **Import .ovpn file** -3. **VSCode**: Install the "Remote-SSH" extension for connection - -### Llama.cpp Setup - -1. **Clone Repo**: `git clone https://github.com/ggerganov/llama.cpp && cd llama.cpp` -2. **Build**: - -```bash -mkdir build && cd build -cmake .. -DLLAMA_CUBLAS=ON -DLLAMA_CUDA_F16=ON -DLLAMA_CUDA_MMV_Y=8 -cmake --build . --config Release -``` - -3. **Download Model:** - -```bash -cd ../models && wget https://huggingface.co/TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q8_0.gguf -``` - -4. **Run:** - -```bash -cd ../build/bin/ -./main -m ./models/llama-2-7b.Q8_0.gguf -p "Writing a thesis proposal can be done in 10 simple steps:\nStep 1:" -n 2048 -e -ngl 100 -t 48 -``` - -For the llama.cpp CLI arguments you can see here: - -| Short Option | Long Option | Param Value | Description | -| --------------- | --------------------- | ----------- | ---------------------------------------------------------------- | -| `-h` | `--help` | | Show this help message and exit | -| `-i` | `--interactive` | | Run in interactive mode | -| | `--interactive-first` | | Run in interactive mode and wait for input right away | -| | `-ins`, `--instruct` | | Run in instruction mode (use with Alpaca models) | -| `-r` | `--reverse-prompt` | `PROMPT` | Run in interactive mode and poll user input upon seeing `PROMPT` | -| | `--color` | | Colorise output to distinguish prompt and user input from | -| **Generations** | -| `-s` | `--seed` | `SEED` | Seed for random number generator | -| `-t` | `--threads` | `N` | Number of threads to use during computation | -| `-p` | `--prompt` | `PROMPT` | Prompt to start generation with | -| | `--random-prompt` | | Start with a randomized prompt | -| | `--in-prefix` | `STRING` | String to prefix user inputs with | -| `-f` | `--file` | `FNAME` | Prompt file to start generation | -| `-n` | `--n_predict` | `N` | Number of tokens to predict | -| | `--top_k` | `N` | Top-k sampling | -| | `--top_p` | `N` | Top-p sampling | -| | `--repeat_last_n` | `N` | Last n tokens to consider for penalize | -| | `--repeat_penalty` | `N` | Penalize repeat sequence of tokens | -| `-c` | `--ctx_size` | `N` | Size of the prompt context | -| | `--ignore-eos` | | Ignore end of stream token and continue generating | -| | `--memory_f32` | | Use `f32` instead of `f16` for memory key+value | -| | `--temp` | `N` | Temperature | -| | `--n_parts` | `N` | Number of model parts | -| `-b` | `--batch_size` | `N` | Batch size for prompt processing | -| | `--perplexity` | | Compute perplexity over the prompt | -| | `--keep` | | Number of tokens to keep from the initial prompt | -| | `--mlock` | | Force system to keep model in RAM | -| | `--mtest` | | Determine the maximum memory usage | -| | `--verbose-prompt` | | Print prompt before generation | -| `-m` | `--model` | `FNAME` | Model path | - -### TensorRT-LLM Setup - -#### **Docker and TensorRT-LLM build** - -> Note: You should run with admin permission to make sure everything works fine - -1. **Docker Image:** - -```bash -sudo make -C docker build -``` - -2. **Run Container:** - -```bash -sudo make -C docker run -``` - -Once in the container, TensorRT-LLM can be built from the source using the following: - -3. **Build:** - -```bash -# To build the TensorRT-LLM code. -python3 ./scripts/build_wheel.py --trt_root /usr/local/tensorrt -# Deploy TensorRT-LLM in your environment. -pip install ./build/tensorrt_llm*.whl -``` - -> Note: You can specify the GPU architecture (e.g. for 4090 is ADA) for compilation time reduction -> The list of supported architectures can be found in the `CMakeLists.txt` file. - -```bash -python3 ./scripts/build_wheel.py --cuda_architectures "89-real;90-real" -``` - -#### Running TensorRT-LLM - -1. **Requirements:** - -```bash -pip install -r examples/bloom/requirements.txt && git lfs install -``` - -2. **Download Weights:** - -```bash -cd examples/llama && rm -rf ./llama/7B && mkdir -p ./llama/7B && git clone https://huggingface.co/NousResearch/Llama-2-7b-hf ./llama/7B -``` - -3. **Build Engine:** - -```bash -python build.py --model_dir ./llama/7B/ --dtype float16 --remove_input_padding --use_gpt_attention_plugin float16 --enable_context_fmha --use_gemm_plugin float16 --use_weight_only --output_dir ./llama/7B/trt_engines/weight_only/1-gpu/ -``` - -4. **Run Inference:** - -```bash -python3 run.py --max_output_len=2048 --tokenizer_dir ./llama/7B/ --engine_dir=./llama/7B/trt_engines/weight_only/1-gpu/ --input_text "Writing a thesis proposal can be done in 10 simple steps:\nStep 1:" -``` - -For the tensorRT-LLM CLI arguments, you can see in the `run.py`. diff --git a/docs/docs/about/03-engineering/04-rd.md b/docs/docs/about/03-engineering/04-rd.md deleted file mode 100644 index f173d27de..000000000 --- a/docs/docs/about/03-engineering/04-rd.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: R&D -description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -slug: /engineering/research -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -## Foundry Best Practices - -@alan/rex TODO diff --git a/docs/docs/about/03-engineering/05-postmortems/README.md b/docs/docs/about/03-engineering/05-postmortems/README.md deleted file mode 100644 index fda1e6c2f..000000000 --- a/docs/docs/about/03-engineering/05-postmortems/README.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Postmortems -description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -slug: /postmortems -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -import DocCardList from "@theme/DocCardList"; - - diff --git a/docs/docs/about/04-product/README.md b/docs/docs/about/04-product/README.md deleted file mode 100644 index c07109800..000000000 --- a/docs/docs/about/04-product/README.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: Product -slug: /product ---- - -## Prerequisites - -- [Figma](https://figma.com) -- [ScreenStudio](https://www.screen.studio/) diff --git a/docs/docs/about/05-community/events/README.mdx b/docs/docs/about/05-community/events/README.mdx deleted file mode 100644 index 92d9cd5f1..000000000 --- a/docs/docs/about/05-community/events/README.mdx +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Events -slug: /events -description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -import DocCardList from "@theme/DocCardList"; - - \ No newline at end of file diff --git a/docs/docs/about/06-careers/README.md b/docs/docs/about/06-careers/README.md deleted file mode 100644 index a0a579e02..000000000 --- a/docs/docs/about/06-careers/README.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Careers -slug: /careers -description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -## We're hiring - -[Careers on Bamboo](https://janai.bamboohr.com/careers) diff --git a/docs/docs/about/2035.mdx b/docs/docs/about/2035.mdx new file mode 100644 index 000000000..3af7a3197 --- /dev/null +++ b/docs/docs/about/2035.mdx @@ -0,0 +1,12 @@ +--- +title: Jan's Vision for 2035 +--- + +[Jan 2035: A Robotics Company](https://hackmd.io/QIWyYbNNQVWVbupuI3kjAA) + +We only have 2 planning parameters: + +- 10 year vision +- 2 week sprint + +And we measure our success on Quarterly OKRs \ No newline at end of file diff --git a/docs/docs/about/about.md b/docs/docs/about/about.md new file mode 100644 index 000000000..3066f3003 --- /dev/null +++ b/docs/docs/about/about.md @@ -0,0 +1,94 @@ +--- +title: About Jan +slug: /about +description: Jan is a productivity tool to customize AI to your needs and workflows. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + ] +--- + +Jan is a [open-source](https://en.wikipedia.org/wiki/Open_source), [local-first](https://www.inkandswitch.com/local-first/) tool to [create, customize and use AI](https://www.gatesnotes.com/AI-agents) for everyday tasks. + +You can: + +- Run locally using [open-source LLMs](https://huggingface.co/models?pipeline_tag=text-generation) or connect to cloud AIs like [ChatGPT](https://openai.com/blog/openai-api) or [Google](https://ai.google.dev/) +- Fine-tune AI with specific knowledge +- Search the web and other databases +- Connect AI to your everyday tools and (with your permission) do work on your behalf + +Longer-term, Jan is building a cognitive framework for future robots. We envision a world where we have personal or company robots that we continually improve and customize, growing together with us. + +![Human repairing a Droid](/img/star-wars-droids.png) + +## Why do we exist + +At Jan, our mission is to advance human-machine collaboration. We achieve this through delivering the best open-source, local-first tools to allow users to run, customize and tinker with AI. + +## What's different about it? + +| | Status Quo | Jan | +| ---------------------------------------------------------- | -------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| Ownership | AI Monopolies owned by Big Tech | AI that you own and control | +| Openness? | Closed-source | [Open-source (AGPLv3)](https://github.com/janhq/jan/blob/main/LICENSE) | +| Your role | Consume | Create, Tinker and Customize | +| Approach | Cloud | [Local-first](https://www.inkandswitch.com/local-first/), running 100% on your devices | +| Data | Data stored on their servers | Data stored in your local filesystem in open, non-proprietary file formats | +| Privacy | 😂 | Runs 100% on your own machine, predictably, privately and offline | +| Transparency | "Black Box" | Runs predictability with code available to tinker and customize | +| What happens if there's an outage or goes out of business? | Your life's work held hostage in the cloud in proprietary data formats[^1] | Continues to run 100% on your computer, your data is safe in your local folder | +| Driving Philosophy | Monetize your users | [Privacy as a human right](https://en.wikipedia.org/wiki/Right_to_privacy) and the [Right to Repair](https://www.repair.org/) | + +## How do I get it? + +You can install and start using Jan in less than 5 minutes, from [jan.ai](https://jan.ai) or our [Github repo](https://github.com/janhq/jan). + +You can read the [User Guide](/docs/user-guide) if you need some help to get started. + +## What license is the code under? + +Jan is licensed under the [AGPLv3 License](https://github.com/janhq/jan/blob/main/LICENSE). + +We happily accept pull requests, however, we do ask that you sign a [Contributor License Agreement](https://en.wikipedia.org/wiki/Contributor_License_Agreement) so that we have the right to relicense your contributions[^2]. + +We also have a [Contributor Program](/docs/team/contributor-program) to provide ownership and upside to contributors who have made significant contributions to the project. + +## What was it built with? + +[Jan](https://github.com/janhq/jan) is pragmatically built using `Typescript` at the application level and `C++` at the Inference level (which we have refactored into [Nitro](https://nitro.jan.ai)[^3]). + +We follow [clean architecture](https://blog.cleancoder.com/uncle-bob/2012/08/13/the-clean-architecture.html) and currently support multiple frameworks and runtimes: + +- A desktop client with [Electron](https://www.electronjs.org/) +- A headless server-mode with [Nodejs](https://nodejs.org/en) +- Planned support for mobile with [Capacitor](https://capacitorjs.com/) +- Planned support for Python runtime + +Architecturally, we have made similar choices to the [Next.js Enterprise Javascript Stack](https://vercel.com/templates/next.js/nextjs-enterprise-boilerplate), which is a [battle-tested](https://nextjs.org/showcase/enterprise) framework for building enterprise-grade applications that scale. + +:::tip + +**At its core, Jan is a software development kit to build and run copilots on personal devices**. The Desktop Client many folks use is, rather, a specific set of extensions packaged by default. We're excited to see what developers do with the SDK (once its in better shape). + +::: + +## Contact + +Drop us a message in our [Discord](https://discord.gg/af6SaTdzpx) and we'll get back to you. + +- `#general`: for general discussion +- `#get-help`: for bug reports and troubleshooting +- `#roadmap`: for feature requests and ideas + +## Footnotes + +[^1]: Credit to Obsidian's original website +[^2]: Credit to [Discourse's About Page](https://www.discourse.org/about) +[^3]: Credit to [Llama.cpp](https://github.com/ggerganov/llama.cpp), [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM), [vLLM](https://github.com/vllm-project/vllm), [LMDeploy](https://github.com/InternLM/lmdeploy) and more. diff --git a/docs/docs/about/roadmap.md b/docs/docs/about/roadmap.md new file mode 100644 index 000000000..1c789d733 --- /dev/null +++ b/docs/docs/about/roadmap.md @@ -0,0 +1,6 @@ +--- +title: Roadmap +--- + +- [ ] [Immediate Roadmap on Github](https://github.com/orgs/janhq/projects/5/views/16) +- [ ] [Longer-term Roadmap on Discord](https://discord.gg/Ey62mynnYr) \ No newline at end of file diff --git a/docs/docs/about/05-community/README.md b/docs/docs/community/community.md similarity index 96% rename from docs/docs/about/05-community/README.md rename to docs/docs/community/community.md index 849cac61e..24a87daf0 100644 --- a/docs/docs/about/05-community/README.md +++ b/docs/docs/community/community.md @@ -1,5 +1,5 @@ --- -title: Community +title: Jan's Community slug: /community description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. keywords: diff --git a/docs/docs/docs/02-integrations/README.md b/docs/docs/docs/02-integrations/README.md deleted file mode 100644 index a1fe45e0c..000000000 --- a/docs/docs/docs/02-integrations/README.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: Integrations ---- - -Existing and upcoming 3rd party integrations on top of Jan Framework. - -From both the core development team, and core contributors. - -Suggestions? File an [issue here](https://github.com/janhq/jan/issues) diff --git a/docs/docs/ecosystem/ecosystem.md b/docs/docs/ecosystem/ecosystem.md new file mode 100644 index 000000000..38f939b3a --- /dev/null +++ b/docs/docs/ecosystem/ecosystem.md @@ -0,0 +1,3 @@ +--- +title: Ecosystem +--- \ No newline at end of file diff --git a/docs/docs/about/05-community/events/hcmc-oct23.md b/docs/docs/events/hcmc-oct23.md similarity index 100% rename from docs/docs/about/05-community/events/hcmc-oct23.md rename to docs/docs/events/hcmc-oct23.md diff --git a/docs/docs/about/05-community/events/nvidia-llm-day-nov-23.md b/docs/docs/events/nvidia-llm-day-nov-23.md similarity index 100% rename from docs/docs/about/05-community/events/nvidia-llm-day-nov-23.md rename to docs/docs/events/nvidia-llm-day-nov-23.md diff --git a/docs/docs/features/agents-framework.md b/docs/docs/features/agents-framework.md new file mode 100644 index 000000000..2ba312854 --- /dev/null +++ b/docs/docs/features/agents-framework.md @@ -0,0 +1,3 @@ +--- +title: Agents Framework +--- \ No newline at end of file diff --git a/docs/docs/features/api-server.md b/docs/docs/features/api-server.md new file mode 100644 index 000000000..36f697cb2 --- /dev/null +++ b/docs/docs/features/api-server.md @@ -0,0 +1,3 @@ +--- +title: API Server +--- \ No newline at end of file diff --git a/docs/docs/features/data-security.md b/docs/docs/features/data-security.md new file mode 100644 index 000000000..c5c1e6c7c --- /dev/null +++ b/docs/docs/features/data-security.md @@ -0,0 +1,3 @@ +--- +title: Data Security +--- \ No newline at end of file diff --git a/docs/docs/features/extensions-framework.md b/docs/docs/features/extensions-framework.md new file mode 100644 index 000000000..6c173ee53 --- /dev/null +++ b/docs/docs/features/extensions-framework.md @@ -0,0 +1,3 @@ +--- +title: Extensions Framework +--- \ No newline at end of file diff --git a/docs/docs/features/features.md b/docs/docs/features/features.md new file mode 100644 index 000000000..d68e9a7ad --- /dev/null +++ b/docs/docs/features/features.md @@ -0,0 +1,3 @@ +--- +title: Features +--- \ No newline at end of file diff --git a/docs/docs/features/local.md b/docs/docs/features/local.md new file mode 100644 index 000000000..6c80c0378 --- /dev/null +++ b/docs/docs/features/local.md @@ -0,0 +1,3 @@ +--- +title: Local & Self-Hosted AI +--- \ No newline at end of file diff --git a/docs/docs/features/remote.md b/docs/docs/features/remote.md new file mode 100644 index 000000000..4145a0f0a --- /dev/null +++ b/docs/docs/features/remote.md @@ -0,0 +1,3 @@ +--- +title: OpenAI API Support (and others) +--- \ No newline at end of file diff --git a/docs/docs/foundry/foundry.md b/docs/docs/foundry/foundry.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/docs/guides/02-installation/05-docker.md b/docs/docs/guides/02-installation/05-docker.md new file mode 100644 index 000000000..6236ed92e --- /dev/null +++ b/docs/docs/guides/02-installation/05-docker.md @@ -0,0 +1,102 @@ +--- +title: Docker +slug: /install/docker +description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + docker installation, + ] +--- + +# Installing Jan using Docker + +## Installation + +### Pre-requisites + +:::note + +**Supported OS**: Linux, WSL2 Docker + +::: + +- Docker Engine and Docker Compose are required to run Jan in Docker mode. Follow the [instructions](https://docs.docker.com/engine/install/ubuntu/) below to get started with Docker Engine on Ubuntu. + +```bash +curl -fsSL https://get.docker.com -o get-docker.sh +sudo sh ./get-docker.sh --dry-run +``` + +- If you intend to run Jan in GPU mode, you need to install `nvidia-driver` and `nvidia-docker2`. Follow the instruction [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for installation. + +### Instructions + +- Run Jan in Docker mode + + - **Option 1**: Run Jan in CPU mode + + ```bash + docker compose --profile cpu up -d + ``` + + - **Option 2**: Run Jan in GPU mode + + - **Step 1**: Check CUDA compatibility with your NVIDIA driver by running `nvidia-smi` and check the CUDA version in the output + + ```bash + nvidia-smi + + # Output + +---------------------------------------------------------------------------------------+ + | NVIDIA-SMI 531.18 Driver Version: 531.18 CUDA Version: 12.1 | + |-----------------------------------------+----------------------+----------------------+ + | GPU Name TCC/WDDM | Bus-Id Disp.A | Volatile Uncorr. ECC | + | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | + | | | MIG M. | + |=========================================+======================+======================| + | 0 NVIDIA GeForce RTX 4070 Ti WDDM | 00000000:01:00.0 On | N/A | + | 0% 44C P8 16W / 285W| 1481MiB / 12282MiB | 2% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + | 1 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:02:00.0 Off | N/A | + | 0% 49C P8 14W / 120W| 0MiB / 6144MiB | 0% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + | 2 NVIDIA GeForce GTX 1660 Ti WDDM | 00000000:05:00.0 Off | N/A | + | 29% 38C P8 11W / 120W| 0MiB / 6144MiB | 0% Default | + | | | N/A | + +-----------------------------------------+----------------------+----------------------+ + + +---------------------------------------------------------------------------------------+ + | Processes: | + | GPU GI CI PID Type Process name GPU Memory | + | ID ID Usage | + |=======================================================================================| + ``` + + - **Step 2**: Visit [NVIDIA NGC Catalog ](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags) and find the smallest minor version of image tag that matches your CUDA version (e.g., 12.1 -> 12.1.0) + + - **Step 3**: Update the `Dockerfile.gpu` line number 5 with the latest minor version of the image tag from step 2 (e.g. change `FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base` to `FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS base`) + + - **Step 4**: Run command to start Jan in GPU mode + + ```bash + # GPU mode + docker compose --profile gpu up -d + ``` + + This will start the web server and you can access Jan at `http://localhost:3000`. + +:::warning + +- Docker mode is currently only suitable for development and localhost. Production is not supported yet, and the RAG feature is not available in Docker mode. + +::: diff --git a/docs/docs/guides/02-installation/05-nightly-build.md b/docs/docs/guides/02-installation/07-nightly-build.md similarity index 100% rename from docs/docs/guides/02-installation/05-nightly-build.md rename to docs/docs/guides/02-installation/07-nightly-build.md diff --git a/docs/docs/guides/02-installation/07-antivirus-compatibility-testing.md b/docs/docs/guides/02-installation/08-antivirus-compatibility-testing.md similarity index 100% rename from docs/docs/guides/02-installation/07-antivirus-compatibility-testing.md rename to docs/docs/guides/02-installation/08-antivirus-compatibility-testing.md diff --git a/docs/docs/guides/04-using-models/02-import-manually.mdx b/docs/docs/guides/04-using-models/02-import-manually.mdx index 68142a8af..7c446ea1c 100644 --- a/docs/docs/guides/04-using-models/02-import-manually.mdx +++ b/docs/docs/guides/04-using-models/02-import-manually.mdx @@ -29,6 +29,10 @@ In this section, we will show you how to import a GGUF model from [HuggingFace]( > We are fast shipping a UI to make this easier, but it's a bit manual for now. Apologies. +## Import Models Using Absolute Filepath (version 0.4.7) + +Starting from version 0.4.7, Jan has introduced the capability to import models using an absolute file path. It allows you to import models from any directory on your computer. Please check the [import models using absolute filepath](../import-models-using-absolute-filepath) guide for more information. + ## Manually Importing a Downloaded Model (nightly versions and v0.4.4+) ### 1. Create a Model Folder @@ -186,7 +190,6 @@ This means that you can easily reconfigure your models, export them, and share y Edit `model.json` and include the following configurations: -- Ensure the filename must be `model.json`. - Ensure the `id` property matches the folder name you created. - Ensure the GGUF filename should match the `id` property exactly. - Ensure the `source.url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the direct links in the `Files and versions` tab. diff --git a/docs/docs/guides/04-using-models/03-import-models-using-absolute-filepath.mdx b/docs/docs/guides/04-using-models/03-import-models-using-absolute-filepath.mdx new file mode 100644 index 000000000..490f68cd6 --- /dev/null +++ b/docs/docs/guides/04-using-models/03-import-models-using-absolute-filepath.mdx @@ -0,0 +1,84 @@ +--- +title: Import Models Using Absolute Filepath +slug: /guides/using-models/import-models-using-absolute-filepath +description: Guide to import model using absolute filepath in Jan. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + import-models-manually, + absolute-filepath, + ] +--- + +In this guide, we will walk you through the process of importing a model using an absolute filepath in Jan, using our latest model, [Trinity](https://huggingface.co/janhq/trinity-v1-GGUF), as an example. + +### 1. Get the Absolute Filepath of the Model + +After downloading .gguf model, you can get the absolute filepath of the model file. + +### 2. Configure the Model JSON + +1. Navigate to the `~/jan/models` folder. +2. Create a folder named ``, for example, `tinyllama`. +3. Create a `model.json` file inside the folder, including the following configurations: + +- Ensure the `id` property matches the folder name you created. +- Ensure the `url` property is the direct binary download link ending in `.gguf`. Now, you can use the absolute filepath of the model file. +- Ensure the `engine` property is set to `nitro`. + +```json +{ + "sources": [ + { + "filename": "tinyllama.gguf", + // highlight-next-line + "url": "" + } + ], + "id": "tinyllama-1.1b", + "object": "model", + "name": "(Absolute Path) TinyLlama Chat 1.1B Q4", + "version": "1.0", + "description": "TinyLlama is a tiny model with only 1.1B. It's a good model for less powerful computers.", + "format": "gguf", + "settings": { + "ctx_len": 4096, + "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>", + "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 2048, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "TinyLlama", + "tags": ["Tiny", "Foundation Model"], + "size": 669000000 + }, + "engine": "nitro" +} +``` + +:::warning + +- If you are using Windows, you need to use double backslashes in the url property, for example: `C:\\Users\\username\\filename.gguf`. + +::: + +### 3. Start the Model + +Restart Jan and navigate to the Hub. Locate your model and click the Use button. + +![Demo](assets/03-demo-absolute-filepath.gif) \ No newline at end of file diff --git a/docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx b/docs/docs/guides/04-using-models/04-integrate-with-remote-server.mdx similarity index 97% rename from docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx rename to docs/docs/guides/04-using-models/04-integrate-with-remote-server.mdx index f0db1bd55..3632a40b0 100644 --- a/docs/docs/guides/04-using-models/03-integrate-with-remote-server.mdx +++ b/docs/docs/guides/04-using-models/04-integrate-with-remote-server.mdx @@ -88,7 +88,7 @@ You can find your API keys in the [OpenAI Platform](https://platform.openai.com/ Restart Jan and navigate to the Hub. Then, select your configured model and start the model. -![image-01](assets/03-openai-platform-configuration.png) +![image-01](assets/04-openai-platform-configuration.png) ## Engines with OAI Compatible Configuration @@ -159,7 +159,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `mistral-ins-7b-q4` Restart Jan and navigate to the Hub. Locate your model and click the Use button. -![image-02](assets/03-oai-compatible-configuration.png) +![image-02](assets/04-oai-compatible-configuration.png) ## Assistance and Support diff --git a/docs/docs/guides/04-using-models/04-customize-engine-settings.mdx b/docs/docs/guides/04-using-models/05-customize-engine-settings.mdx similarity index 100% rename from docs/docs/guides/04-using-models/04-customize-engine-settings.mdx rename to docs/docs/guides/04-using-models/05-customize-engine-settings.mdx diff --git a/docs/docs/guides/04-using-models/assets/03-demo-absolute-filepath.gif b/docs/docs/guides/04-using-models/assets/03-demo-absolute-filepath.gif new file mode 100644 index 000000000..24dcc251a Binary files /dev/null and b/docs/docs/guides/04-using-models/assets/03-demo-absolute-filepath.gif differ diff --git a/docs/docs/guides/04-using-models/assets/03-oai-compatible-configuration.png b/docs/docs/guides/04-using-models/assets/04-oai-compatible-configuration.png similarity index 100% rename from docs/docs/guides/04-using-models/assets/03-oai-compatible-configuration.png rename to docs/docs/guides/04-using-models/assets/04-oai-compatible-configuration.png diff --git a/docs/docs/guides/04-using-models/assets/03-openai-platform-configuration.png b/docs/docs/guides/04-using-models/assets/04-openai-platform-configuration.png similarity index 100% rename from docs/docs/guides/04-using-models/assets/03-openai-platform-configuration.png rename to docs/docs/guides/04-using-models/assets/04-openai-platform-configuration.png diff --git a/docs/docs/guides/05-using-server/01-server.md b/docs/docs/guides/05-using-server/01-server.md deleted file mode 100644 index 952b7399f..000000000 --- a/docs/docs/guides/05-using-server/01-server.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Connect to Server -description: Connect to Jan's built-in API server. -keywords: - [ - Jan AI, - Jan, - ChatGPT alternative, - local AI, - private AI, - conversational AI, - no-subscription fee, - large language model, - ] ---- - -:::warning - -This page is under construction. - -::: - -Jan ships with a built-in API server, that can be used as a drop-in, local replacement for OpenAI's API. - -Jan runs on port `1337` by default, but this can (soon) be changed in Settings. - -1. Go to Settings > Advanced > Enable API Server - -2. Go to http://localhost:1337 for the API docs. - -3. In terminal, simply CURL... - -Note: Some UI states may be broken when in Server Mode. diff --git a/docs/docs/guides/05-using-server/01-start-server.md b/docs/docs/guides/05-using-server/01-start-server.md new file mode 100644 index 000000000..2433fd80a --- /dev/null +++ b/docs/docs/guides/05-using-server/01-start-server.md @@ -0,0 +1,72 @@ +--- +title: Start Local Server +slug: /guides/using-server/start-server +description: How to run Jan's built-in API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + local server, + api server, + ] +--- + +Jan ships with a built-in API server that can be used as a drop-in, local replacement for OpenAI's API. You can run your server by following these simple steps. + +## Open Local API Server View + +Navigate to the Local API Server view by clicking the corresponding icon on the left side of the screen. + +

+ +![01-local-api-view](./assets/01-local-api-view.gif) + +## Choosing a Model + +On the top right of your screen under `Model Settings`, set the LLM that your local server will be running. You can choose from any of the models already installed, or pick a new model by clicking `Explore the Hub`. + +

+ +![01-choose-model](./assets/01-choose-model.png) + +## Server Options + +On the left side of your screen, you can set custom server options. + +

+ +![01-server-settings](./assets/01-server-options.png) + +### Local Server Address + +By default, Jan will be accessible only on localhost `127.0.0.1`. This means a local server can only be accessed on the same machine where the server is being run. + +You can make the local server more accessible by clicking on the address and choosing `0.0.0.0` instead, which allows the server to be accessed from other devices on the local network. This is less secure than choosing localhost, and should be done with caution. + +### Port + +Jan runs on port `1337` by default. You can change the port to any other port number if needed. + +### Cross-Origin Resource Sharing (CORS) + +Cross-Origin Resource Sharing (CORS) manages resource access on the local server from external domains. Enabled for security by default, it can be disabled if needed. + +### Verbose Server Logs + +The center of the screen displays the server logs as the local server runs. This option provides extensive details about server activities. + +## Start Server + +Click the `Start Server` button on the top left of your screen. You will see the server log display a message such as `Server listening at http://127.0.0.1:1337`, and the `Start Server` button will change to a red `Stop Server` button. + +

+ +![01-running-server](./assets/01-running-server.gif) + +You server is now running and you can use the server address and port to make requests to the local server. diff --git a/docs/docs/guides/05-using-server/02-using-server.md b/docs/docs/guides/05-using-server/02-using-server.md new file mode 100644 index 000000000..3d4b004a1 --- /dev/null +++ b/docs/docs/guides/05-using-server/02-using-server.md @@ -0,0 +1,102 @@ +--- +title: Using Jan's Built-in API Server +description: How to use Jan's built-in API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + local server, + api server, + ] +--- + +Jan's built-in API server is compatible with [OpenAI's API](https://platform.openai.com/docs/api-reference) and can be used as a drop-in, local replacement. Follow these steps to use the API server. + +## Open the API Reference + +Jan contains a comprehensive API reference. This reference displays all the API endpoints available, gives you examples requests and responses, and allows you to execute them in browser. + +On the top left of your screen below the red `Stop Server` button is the blue `API Reference`. Clicking this will open the reference in your browser. + +

+ +![02-api-reference](./assets/02-api-reference.png) + +Scroll through the various available endpoints to learn what options are available and try them out by executing the example requests. In addition, you can also use the [Jan API Reference](https://jan.ai/api-reference/) on the Jan website. + +### Chat + +In the Chat section of the API reference, you will see an example JSON request body. + +

+ +![02-chat-example](./assets/02-chat-example.png) + +With your local server running, you can click the `Try it out` button on the top left, then the blue `Execute` button below the JSON. The browser will send the example request to your server, and display the response body below. + +Use the API endpoints, request and response body examples as models for your own application. + +### cURL Request Example + +Here is an example curl request with a local server running `tinyllama-1.1b`: + +

+ +```json +{ + "messages": [ + { + "content": "You are a helpful assistant.", + "role": "system" + }, + { + "content": "Hello!", + "role": "user" + } + ], + "model": "tinyllama-1.1b", + "stream": true, + "max_tokens": 2048, + "stop": [ + "hello" + ], + "frequency_penalty": 0, + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 +} +' +``` + +### Response Body Example + +```json +{ + "choices": [ + { + "finish_reason": null, + "index": 0, + "message": { + "content": "Hello user. What can I help you with?", + "role": "assistant" + } + } + ], + "created": 1700193928, + "id": "ebwd2niJvJB1Q2Whyvkz", + "model": "_", + "object": "chat.completion", + "system_fingerprint": "_", + "usage": { + "completion_tokens": 500, + "prompt_tokens": 33, + "total_tokens": 533 + } +} +``` diff --git a/docs/docs/guides/05-using-server/assets/01-choose-model.png b/docs/docs/guides/05-using-server/assets/01-choose-model.png new file mode 100644 index 000000000..9062a1e95 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/01-choose-model.png differ diff --git a/docs/docs/guides/05-using-server/assets/01-local-api-view.gif b/docs/docs/guides/05-using-server/assets/01-local-api-view.gif new file mode 100644 index 000000000..cb221fce4 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/01-local-api-view.gif differ diff --git a/docs/docs/guides/05-using-server/assets/01-running-server.gif b/docs/docs/guides/05-using-server/assets/01-running-server.gif new file mode 100644 index 000000000..a4225f3cb Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/01-running-server.gif differ diff --git a/docs/docs/guides/05-using-server/assets/01-server-options.png b/docs/docs/guides/05-using-server/assets/01-server-options.png new file mode 100644 index 000000000..c48844e40 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/01-server-options.png differ diff --git a/docs/docs/guides/05-using-server/assets/02-api-reference.png b/docs/docs/guides/05-using-server/assets/02-api-reference.png new file mode 100644 index 000000000..154d9dfc9 Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/02-api-reference.png differ diff --git a/docs/docs/guides/05-using-server/assets/02-chat-example.png b/docs/docs/guides/05-using-server/assets/02-chat-example.png new file mode 100644 index 000000000..bd7e33a6a Binary files /dev/null and b/docs/docs/guides/05-using-server/assets/02-chat-example.png differ diff --git a/docs/docs/guides/07-integrations/01-integrate-continue.mdx b/docs/docs/guides/07-integrations/01-integrate-continue.mdx index 3a0e9f282..1fa0397e2 100644 --- a/docs/docs/guides/07-integrations/01-integrate-continue.mdx +++ b/docs/docs/guides/07-integrations/01-integrate-continue.mdx @@ -35,7 +35,7 @@ To get started with Continue in VS Code, please follow this [guide to install Co ### 2. Enable Jan API Server -To configure the Continue to use Jan's Local Server, you need to enable Jan API Server with your preferred model, please follow this [guide to enable Jan API Server](../05-using-server/01-server.md) +To configure the Continue to use Jan's Local Server, you need to enable Jan API Server with your preferred model, please follow this [guide to enable Jan API Server](/guides/using-server/start-server). ### 3. Configure Continue to Use Jan's Local Server @@ -77,7 +77,7 @@ Edit the `config.json` file and include the following configuration. // highlight-start "model": "mistral-ins-7b-q4", "apiKey": "EMPTY", - "apiBase": "http://localhost:1337" + "apiBase": "http://localhost:1337/v1" // highlight-end } ] @@ -86,7 +86,7 @@ Edit the `config.json` file and include the following configuration. - Ensure that the `provider` is `openai`. - Ensure that the `model` is the same as the one you enabled in the Jan API Server. -- Ensure that the `apiBase` is `http://localhost:1337`. +- Ensure that the `apiBase` is `http://localhost:1337/v1`. - Ensure that the `apiKey` is `EMPTY`. ### 4. Ensure the Using Model Is Activated in Jan diff --git a/docs/docs/guides/07-integrations/04-integrate-mistral-ai.mdx b/docs/docs/guides/07-integrations/04-integrate-mistral-ai.mdx new file mode 100644 index 000000000..14ddeaa75 --- /dev/null +++ b/docs/docs/guides/07-integrations/04-integrate-mistral-ai.mdx @@ -0,0 +1,89 @@ +--- +title: Integrate Mistral AI with Jan +slug: /guides/integrations/mistral-ai +description: Guide to integrate Mistral AI with Jan +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + Mistral integration, + ] +--- + +## Quick Introduction + +[Mistral AI](https://docs.mistral.ai/) currently provides two ways of accessing their Large Language Models (LLM) - via their API or via open source models available on Hugging Face. In this guide, we will show you how to integrate Mistral AI with Jan using the API method. + +## Steps to Integrate Mistral AI with Jan + +### 1. Configure Mistral API key + +You can find your API keys in the [Mistral API Key](https://console.mistral.ai/user/api-keys/) and set the Mistral AI API key in `~/jan/engines/openai.json` file. + +```json title="~/jan/engines/openai.json" +{ + // highlight-start + "full_url": "https://api.mistral.ai/v1/chat/completions", + "api_key": "" + // highlight-end +} +``` + +### 2. Modify a Model JSON + +Navigate to the `~/jan/models` folder. Create a folder named ``, for example, `mistral-tiny` and create a `model.json` file inside the folder including the following configurations: + +- Ensure the filename must be `model.json`. +- Ensure the `id` property is set to the model id from Mistral AI. +- Ensure the `format` property is set to `api`. +- Ensure the `engine` property is set to `openai`. +- Ensure the `state` property is set to `ready`. + +```json title="~/jan/models/mistral-tiny/model.json" +{ + "sources": [ + { + "filename": "mistral-tiny", + "url": "https://mistral.ai/" + } + ], + "id": "mistral-tiny", + "object": "model", + "name": "Mistral-7B-v0.2 (Tiny Endpoint)", + "version": "1.0", + "description": "Currently powered by Mistral-7B-v0.2, a better fine-tuning of the initial Mistral-7B released, inspired by the fantastic work of the community.", + // highlight-next-line + "format": "api", + "settings": {}, + "parameters": {}, + "metadata": { + "author": "Mistral AI", + "tags": ["General", "Big Context Length"] + }, + // highlight-start + "engine": "openai" + // highlight-end +} +``` + +:::tip + +Mistral AI provides different endpoints. Please check out their [endpoint documentation](https://docs.mistral.ai/platform/endpoints/) to find the one that suits your needs. In this example, we will use the `mistral-tiny` model. + +::: + +### 3. Start the Model + +Restart Jan and navigate to the Hub. Locate your model and click the Use button. + +![Mitral AI Tiny Model](assets/04-mistral-ai-tiny-hub.png) + +### 4. Try Out the Integration of Jan and Mistral AI + +![Mistral AI Integration Demo](assets/04-mistral-ai-integration-demo.gif) diff --git a/docs/docs/guides/07-integrations/05-integrate-lmstudio.mdx b/docs/docs/guides/07-integrations/05-integrate-lmstudio.mdx new file mode 100644 index 000000000..58e2f0be9 --- /dev/null +++ b/docs/docs/guides/07-integrations/05-integrate-lmstudio.mdx @@ -0,0 +1,184 @@ +--- +title: Integrate LM Studio with Jan +slug: /guides/integrations/lmstudio +description: Guide to integrate LM Studio with Jan +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + LM Studio integration, + ] +--- + +## Quick Introduction + +With [LM Studio](https://lmstudio.ai/), you can discover, download, and run local Large Language Models (LLMs). In this guide, we will show you how to integrate and use your current models on LM Studio with Jan using 2 methods. The first method is integrating LM Studio server with Jan UI. The second method is migrating your downloaded model from LM Studio to Jan. We will use the [Phi 2 - GGUF](https://huggingface.co/TheBloke/phi-2-GGUF) model on Hugging Face as an example. + +## Steps to Integrate LM Studio Server with Jan UI + +### 1. Start the LM Studio Server + +1. Navigate to the `Local Inference Server` on the LM Studio application. +2. Select the model you want to use. +3. Start the server after configuring the server port and options. + +![LM Studio Server](assets/05-setting-lmstudio-server.gif) + +

+ +Modify the `openai.json` file in the `~/jan/engines` folder to include the full URL of the LM Studio server. + +```json title="~/jan/engines/openai.json" +{ + "full_url": "http://localhost:/v1/chat/completions" +} +``` + +:::tip + +- Replace `` with the port number you set in the LM Studio server. The default port is `1234`. + +::: + +### 2. Modify a Model JSON + +Navigate to the `~/jan/models` folder. Create a folder named ``, for example, `lmstudio-phi-2` and create a `model.json` file inside the folder including the following configurations: + +- Set the `format` property to `api`. +- Set the `engine` property to `openai`. +- Set the `state` property to `ready`. + +```json title="~/jan/models/lmstudio-phi-2/model.json" +{ + "sources": [ + { + "filename": "phi-2-GGUF", + "url": "https://huggingface.co/TheBloke/phi-2-GGUF" + } + ], + "id": "lmstudio-phi-2", + "object": "model", + "name": "LM Studio - Phi 2 - GGUF", + "version": "1.0", + "description": "TheBloke/phi-2-GGUF", + // highlight-next-line + "format": "api", + "settings": {}, + "parameters": {}, + "metadata": { + "author": "Microsoft", + "tags": ["General", "Big Context Length"] + }, + // highlight-start + "engine": "openai" + // highlight-end +} +``` + +### 3. Start the Model + +1. Restart Jan and navigate to the **Hub**. +2. Locate your model and click the **Use** button. + +![LM Studio Model](assets/05-lmstudio-run.png) + +### 4. Try Out the Integration of Jan and LM Studio + +![LM Studio Integration Demo](assets/05-lmstudio-integration-demo.gif) + +## Steps to Migrate Your Downloaded Model from LM Studio to Jan (version 0.4.6 and older) + +### 1. Migrate Your Downloaded Model + +1. Navigate to `My Models` in the LM Studio application and reveal the model folder. + +![Reveal-model-folder-lmstudio](assets/05-reveal-model-folder-lmstudio.gif) + +2. Copy the model folder that you want to migrate to `~/jan/models` folder. + +3. Ensure the folder name property is the same as the model name of `.gguf` filename by changing the folder name if necessary. For example, in this case, we changed foldername from `TheBloke` to `phi-2.Q4_K_S`. + +### 2. Start the Model + +1. Restart Jan and navigate to the **Hub**. Jan will automatically detect the model and display it in the **Hub**. +2. Locate your model and click the **Use** button to try the migrating model. + +![Demo](assets/05-demo-migrating-model.gif) + +## Steps to Pointing to the Downloaded Model of LM Studio from Jan (version 0.4.7+) + +Starting from version 0.4.7, Jan supports importing models using an absolute filepath, so you can directly use the model from the LM Studio folder. + +### 1. Reveal the Model Absolute Path + +Navigate to `My Models` in the LM Studio application and reveal the model folder. Then, you can get the absolute path of your model. + +![Reveal-model-folder-lmstudio](assets/05-reveal-model-folder-lmstudio.gif) + +### 2. Modify a Model JSON + +Navigate to the `~/jan/models` folder. Create a folder named ``, for example, `phi-2.Q4_K_S` and create a `model.json` file inside the folder including the following configurations: + +- Ensure the `id` property matches the folder name you created. +- Ensure the `url` property is the direct binary download link ending in `.gguf`. Now, you can use the absolute filepath of the model file. In this example, the absolute filepath is `/Users//.cache/lm-studio/models/TheBloke/phi-2-GGUF/phi-2.Q4_K_S.gguf`. +- Ensure the `engine` property is set to `nitro`. + +```json +{ + "object": "model", + "version": 1, + "format": "gguf", + "sources": [ + { + "filename": "phi-2.Q4_K_S.gguf", + "url": "" + } + ], + "id": "phi-2.Q4_K_S", + "name": "phi-2.Q4_K_S", + "created": 1708308111506, + "description": "phi-2.Q4_K_S - user self import model", + "settings": { + "ctx_len": 4096, + "embedding": false, + "prompt_template": "{system_message}\n### Instruction: {prompt}\n### Response:", + "llama_model_path": "phi-2.Q4_K_S.gguf" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 2048, + "stop": [""], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "size": 1615568736, + "author": "User", + "tags": [] + }, + "engine": "nitro" +} +``` + +:::warning + +- If you are using Windows, you need to use double backslashes in the url property, for example: `C:\\Users\\username\\filename.gguf`. + +::: + + +### 3. Start the Model + +1. Restart Jan and navigate to the **Hub**. +2. Jan will automatically detect the model and display it in the **Hub**. +3. Locate your model and click the **Use** button to try the migrating model. + +![Demo](assets/05-demo-pointing-model.gif) diff --git a/docs/docs/guides/07-integrations/06-integrate-ollama.mdx b/docs/docs/guides/07-integrations/06-integrate-ollama.mdx new file mode 100644 index 000000000..e55c3e49f --- /dev/null +++ b/docs/docs/guides/07-integrations/06-integrate-ollama.mdx @@ -0,0 +1,90 @@ +--- +title: Integrate Ollama with Jan +slug: /guides/integrations/ollama +description: Guide to integrate Ollama with Jan +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + Ollama integration, + ] +--- + +## Quick Introduction + +With [Ollama](https://ollama.com/), you can run large language models locally. In this guide, we will show you how to integrate and use your current models on Ollama with Jan using 2 methods. The first method is integrating Ollama server with Jan UI. The second method is migrating your downloaded model from Ollama to Jan. We will use the [llama2](https://ollama.com/library/llama2) model as an example. + +## Steps to Integrate Ollama Server with Jan UI + +### 1. Start the Ollama Server + +1. Select the model you want to use from the [Ollama library](https://ollama.com/library). +2. Run your model by using the following command: + +```bash +ollama run +``` + +3. According to the [Ollama documentation on OpenAI compatibility](https://github.com/ollama/ollama/blob/main/docs/openai.md), you can use the `http://localhost:11434/v1/chat/completions` endpoint to interact with the Ollama server. Thus, modify the `openai.json` file in the `~/jan/engines` folder to include the full URL of the Ollama server. + +```json title="~/jan/engines/openai.json" +{ + "full_url": "http://localhost:11434/v1/chat/completions" +} +``` + +### 2. Modify a Model JSON + +1. Navigate to the `~/jan/models` folder. +2. Create a folder named ``, for example, `lmstudio-phi-2`. +3. Create a `model.json` file inside the folder including the following configurations: + +- Set the `id` property to the model name as Ollama model name. +- Set the `format` property to `api`. +- Set the `engine` property to `openai`. +- Set the `state` property to `ready`. + +```json title="~/jan/models/llama2/model.json" +{ + "sources": [ + { + "filename": "llama2", + "url": "https://ollama.com/library/llama2" + } + ], + // highlight-next-line + "id": "llama2", + "object": "model", + "name": "Ollama - Llama2", + "version": "1.0", + "description": "Llama 2 is a collection of foundation language models ranging from 7B to 70B parameters.", + // highlight-next-line + "format": "api", + "settings": {}, + "parameters": {}, + "metadata": { + "author": "Meta", + "tags": ["General", "Big Context Length"] + }, + // highlight-next-line + "engine": "openai" +} +``` + +### 3. Start the Model + +1. Restart Jan and navigate to the **Hub**. +2. Locate your model and click the **Use** button. + +![Ollama Model](assets/06-ollama-run.png) + +### 4. Try Out the Integration of Jan and Ollama + +![Ollama Integration Demo](assets/06-ollama-integration-demo.gif) + diff --git a/docs/docs/guides/07-integrations/assets/04-mistral-ai-integration-demo.gif b/docs/docs/guides/07-integrations/assets/04-mistral-ai-integration-demo.gif new file mode 100644 index 000000000..015167e2a Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/04-mistral-ai-integration-demo.gif differ diff --git a/docs/docs/guides/07-integrations/assets/04-mistral-ai-tiny-hub.png b/docs/docs/guides/07-integrations/assets/04-mistral-ai-tiny-hub.png new file mode 100644 index 000000000..1ae377d70 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/04-mistral-ai-tiny-hub.png differ diff --git a/docs/docs/guides/07-integrations/assets/05-demo-migrating-model.gif b/docs/docs/guides/07-integrations/assets/05-demo-migrating-model.gif new file mode 100644 index 000000000..985755e47 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-demo-migrating-model.gif differ diff --git a/docs/docs/guides/07-integrations/assets/05-demo-pointing-model.gif b/docs/docs/guides/07-integrations/assets/05-demo-pointing-model.gif new file mode 100644 index 000000000..137fb955a Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-demo-pointing-model.gif differ diff --git a/docs/docs/guides/07-integrations/assets/05-lmstudio-integration-demo.gif b/docs/docs/guides/07-integrations/assets/05-lmstudio-integration-demo.gif new file mode 100644 index 000000000..445ea3416 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-lmstudio-integration-demo.gif differ diff --git a/docs/docs/guides/07-integrations/assets/05-lmstudio-run.png b/docs/docs/guides/07-integrations/assets/05-lmstudio-run.png new file mode 100644 index 000000000..721581f72 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-lmstudio-run.png differ diff --git a/docs/docs/guides/07-integrations/assets/05-reveal-model-folder-lmstudio.gif b/docs/docs/guides/07-integrations/assets/05-reveal-model-folder-lmstudio.gif new file mode 100644 index 000000000..4c1ee85fc Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-reveal-model-folder-lmstudio.gif differ diff --git a/docs/docs/guides/07-integrations/assets/05-setting-lmstudio-server.gif b/docs/docs/guides/07-integrations/assets/05-setting-lmstudio-server.gif new file mode 100644 index 000000000..63084be01 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/05-setting-lmstudio-server.gif differ diff --git a/docs/docs/guides/07-integrations/assets/06-ollama-integration-demo.gif b/docs/docs/guides/07-integrations/assets/06-ollama-integration-demo.gif new file mode 100644 index 000000000..708f2058a Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/06-ollama-integration-demo.gif differ diff --git a/docs/docs/guides/07-integrations/assets/06-ollama-run.png b/docs/docs/guides/07-integrations/assets/06-ollama-run.png new file mode 100644 index 000000000..7f18e1b15 Binary files /dev/null and b/docs/docs/guides/07-integrations/assets/06-ollama-run.png differ diff --git a/docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx b/docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx index d35993ab6..53638027b 100644 --- a/docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx +++ b/docs/docs/guides/08-troubleshooting/03-gpu-not-used.mdx @@ -188,4 +188,6 @@ Troubleshooting tips: 2. If the issue persists, ensure your (V)RAM is accessible by the application. Some folks have virtual RAM and need additional configuration. -3. Get help in [Jan Discord](https://discord.gg/mY69SZaMaC). +3. If you are facing issues with the installation of RTX issues, please update the NVIDIA driver that supports CUDA 11.7 or higher. Ensure that the CUDA path is added to the environment variable. + +4. Get help in [Jan Discord](https://discord.gg/mY69SZaMaC). diff --git a/docs/docs/guides/09-advanced-settings/01-https-proxy.mdx b/docs/docs/guides/09-advanced-settings/01-https-proxy.mdx new file mode 100644 index 000000000..35f4c30f9 --- /dev/null +++ b/docs/docs/guides/09-advanced-settings/01-https-proxy.mdx @@ -0,0 +1,101 @@ +--- +title: HTTPS Proxy +slug: /guides/advanced-settings/https-proxy +description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + advanced-settings, + https-proxy, + ] +--- + +In this guide, we will show you how to set up your own HTTPS proxy server and configure Jan to use it. + +## Why HTTPS Proxy? +An HTTPS proxy helps you to maintain your privacy and security while still being able to browser the internet circumventing geographical restrictions. + +## Setting Up Your Own HTTPS Proxy Server +In this section, we will show you a high-level overview of how to set up your own HTTPS proxy server. This guide focus on using Squid as a popular and open-source proxy server software, but there are other software options you might consider based on your needs and preferences. + +### Step 1: Choosing a Server +Firstly, you need to choose a server to host your proxy server. We recommend using a cloud provider like Amazon AWS, Google Cloud, Microsoft Azure, Digital Ocean, etc. Ensure that your server has a public IP address and is accessible from the internet. + +### Step 2: Installing Squid +```bash +sudo apt-get update +sudo apt-get install squid +``` + +### Step 3: Configure Squid for HTTPS + +To enable HTTPS, you will need to configure Squid with SSL support. + +- Generate SSL certificate + +Squid requires an SSL certificate to be able to handle HTTPS traffic. You can generate a self-signed certificate or obtain one from a Certificate Authority (CA). For a self-signed certificate, you can use OpenSSL: + +```bash +openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout squid-proxy.pem -out squid-proxy.pem +``` + +- Configure Squid to use the SSL certificate: Edit the Squid configuration file `/etc/squid/squid.conf` to include the path to your SSL certificate and enable the HTTPS port: + +```bash +http_port 3128 ssl-bump cert=/path/to/your/squid-proxy.pem +ssl_bump server-first all +ssl_bump bump all +``` + +- Enable SSL Bumping: To intercept HTTPS traffic, Squid uses a process called SSL Bumping. This process allows Squid to decrypt and re-encrypt HTTPS traffic. To enable SSL Bumping, ensure the `ssl_bump` directives are configured correctly in your `squid.conf` file. + +### Step 4 (Optional): Configure ACLs and Authentication + +- Access Control Lists (ACLs): You can define rules to control who can access your proxy. This is done by editing the squid.conf file and defining ACLs: + +```bash +acl allowed_ips src "/etc/squid/allowed_ips.txt" +http_access allow allowed_ips +``` + +- Authentication: If you want to add an authentication layer, Squid supports several authentication schemes. Basic authentication setup might look like this: + +```bash +auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/passwords +acl authenticated proxy_auth REQUIRED +http_access allow authenticated +``` + +### Step 5: Restart and Test Your Proxy + +After configuring, restart Squid to apply the changes: + +```bash +sudo systemctl restart squid +``` + +To test, configure your browser or another client to use the proxy server with its IP address and port (default is 3128). Check if you can access the internet through your proxy. + +:::tip + +Tips for Secure Your Proxy: +- Firewall rules: Ensure that only intended users or IP addresses can connect to your proxy server. This can be achieved by setting up appropriate firewall rules. +- Regular updates: Keep your server and proxy software updated to ensure that you are protected against known vulnerabilities. +- Monitoring and logging: Monitor your proxy server for unusual activity and enable logging to keep track of the traffic passing through your proxy. + +::: + +## Setting Up Jan to Use Your HTTPS Proxy + +Once you have your HTTPS proxy server set up, you can configure Jan to use it. Navigate to `Settings` > `Advanced Settings` and specify the HTTPS proxy (proxy auto-configuration and SOCKS not supported). + +You can turn on the feature `Ignore SSL Certificates` if you are using a self-signed certificate. This feature allows self-signed or unverified certificates. + +![01-https-proxy-jan-configure](./assets/01-https-proxy-jan-configure.png) \ No newline at end of file diff --git a/docs/docs/guides/09-advanced-settings/README.mdx b/docs/docs/guides/09-advanced-settings/README.mdx new file mode 100644 index 000000000..ba3da9bb1 --- /dev/null +++ b/docs/docs/guides/09-advanced-settings/README.mdx @@ -0,0 +1,65 @@ +--- +title: Advanced Settings +slug: /guides/advanced-settings/ +description: Jan is a ChatGPT-alternative that runs on your own computer, with a local API server. +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + advanced-settings, + ] +--- + +This guide will show you how to use the advanced settings in Jan. + +## Keyboard Shortcuts + +Keyboard shortcuts are a great way to speed up your workflow. Here are some of the keyboard shortcuts that you can use in Jan. + +| Combination | Description | +| --------------- | -------------------------------------------------- | +| `⌘ E` | Show list your models | +| `⌘ K` | Show list navigation pages | +| `⌘ B` | Toggle collapsible left panel | +| `⌘ ,` | Navigate to setting page | +| `Enter` | Send a message | +| `Shift + Enter` | Insert new line in input box | +| `Arrow Up` | Navigate to previous option (within search dialog) | +| `Arrow Down` | Navigate to next option (within search dialog) | + +

+ +:::note +`⌘` is the command key on macOS, and `Ctrl` on Windows. +::: + +## Experimental Mode + +Experimental mode allows you to enable experimental features that may be unstable tested. + +## Jan Data Folder + +The Jan data folder is the location where messages, model configurations, and other user data are placed. You can change the location of the data folder to a different location. + +![00-changing-folder](./assets/00-changing-folder.gif) + +## HTTPS Proxy & Ignore SSL Certificate + +HTTPS Proxy allows you to use a proxy server to connect to the internet. You can also ignore SSL certificates if you are using a self-signed certificate. +Please check out the guide on [how to set up your own HTTPS proxy server and configure Jan to use it](../advanced-settings/https-proxy) for more information. + +## Clear Logs + +Clear logs will remove all logs from the Jan application. + +## Reset To Factory Default + +Reset the application to its original state, deleting all your usage data, including model customizations and conversation history. This action is irreversible and recommended only if the application is in a corrupted state. + +![00-reset-factory-settings](./assets/00-reset-factory-settings.gif) diff --git a/docs/docs/guides/09-advanced-settings/assets/00-changing-folder.gif b/docs/docs/guides/09-advanced-settings/assets/00-changing-folder.gif new file mode 100644 index 000000000..ac280a5c3 Binary files /dev/null and b/docs/docs/guides/09-advanced-settings/assets/00-changing-folder.gif differ diff --git a/docs/docs/guides/09-advanced-settings/assets/00-reset-factory-settings.gif b/docs/docs/guides/09-advanced-settings/assets/00-reset-factory-settings.gif new file mode 100644 index 000000000..81760848d Binary files /dev/null and b/docs/docs/guides/09-advanced-settings/assets/00-reset-factory-settings.gif differ diff --git a/docs/docs/guides/09-advanced-settings/assets/01-https-proxy-jan-configure.png b/docs/docs/guides/09-advanced-settings/assets/01-https-proxy-jan-configure.png new file mode 100644 index 000000000..25e0f7660 Binary files /dev/null and b/docs/docs/guides/09-advanced-settings/assets/01-https-proxy-jan-configure.png differ diff --git a/docs/docs/how-we-work.md b/docs/docs/how-we-work.md new file mode 100644 index 000000000..e81099d18 --- /dev/null +++ b/docs/docs/how-we-work.md @@ -0,0 +1,23 @@ +--- +title: How We Work +--- + +### Open Source + +Jan is a startup with an open source business model. We believe in the need for an open source AI ecosystem, and are committed to building it. + +- [Jan Framework](https://github.com/janhq/jan) (AGPLv3) +- [Jan Desktop Client & Local server](https://jan.ai) (AGPLv3, built on Jan Framework) +- [Nitro: run Local AI](https://github.com/janhq/nitro) (AGPLv3) + +### Build in Public + +We use GitHub to build in public and welcome anyone to join in. + +- [Jan's Kanban](https://github.com/orgs/janhq/projects/5) +- [Jan's Roadmap](https://github.com/orgs/janhq/projects/5/views/29) +- `coming soon` [Jan's Newsletter](https://newsletter.jan.ai) + +### Remote Team + +Jan has a fully-remote team. We are mainly based in the APAC timezone. We use [Discord](https://discord.gg/af6SaTdzpx) and [Github](https://github.com/janhq) to work. diff --git a/docs/docs/how-we-work/analytics/analytics.md b/docs/docs/how-we-work/analytics/analytics.md new file mode 100644 index 000000000..79e107a83 --- /dev/null +++ b/docs/docs/how-we-work/analytics/analytics.md @@ -0,0 +1,12 @@ +--- +title: Analytics +--- + +Adhering to Jan's privacy preserving philosophy, our analytics philosophy is to get "barely-enough-to-function'. + +#### What is tracked + +1. By default, Github tracks downloads and device metadata for all public GitHub repositories. This helps us troubleshoot & ensure cross-platform support. +2. We use [Umami](https://umami.is/) to collect, analyze, and understand application data while maintaining visitor privacy and data ownership. We are using the Umami Cloud in Europe to ensure GDPR compliance. Please see [Umami Privacy Policy](https://umami.is/privacy) for more details. +3. We use Umami to track a single `app.opened` event without additional user metadata, in order to understand retention. In addition, we track `app.version` to understand app version usage. +4. Additionally, we plan to enable a `Settings` feature for users to turn off all tracking. diff --git a/docs/docs/about/03-engineering/assets/01-get-help.png b/docs/docs/how-we-work/engineering/assets/01-get-help.png similarity index 100% rename from docs/docs/about/03-engineering/assets/01-get-help.png rename to docs/docs/how-we-work/engineering/assets/01-get-help.png diff --git a/docs/docs/about/03-engineering/01-ci-cd.md b/docs/docs/how-we-work/engineering/ci-cd.md similarity index 100% rename from docs/docs/about/03-engineering/01-ci-cd.md rename to docs/docs/how-we-work/engineering/ci-cd.md diff --git a/docs/docs/about/03-engineering/README.md b/docs/docs/how-we-work/engineering/engineering.md similarity index 100% rename from docs/docs/about/03-engineering/README.md rename to docs/docs/how-we-work/engineering/engineering.md diff --git a/docs/docs/about/03-engineering/02-qa.mdx b/docs/docs/how-we-work/engineering/qa.mdx similarity index 100% rename from docs/docs/about/03-engineering/02-qa.mdx rename to docs/docs/how-we-work/engineering/qa.mdx diff --git a/docs/docs/how-we-work/product-design/product-design.md b/docs/docs/how-we-work/product-design/product-design.md new file mode 100644 index 000000000..a2016b6b8 --- /dev/null +++ b/docs/docs/how-we-work/product-design/product-design.md @@ -0,0 +1,11 @@ +--- +title: Product & Design +--- + +## Roadmap + +- Conversations over Tickets + - Discord's #roadmap channel + - Work with the community to turn conversations into Product Specs +- Future System? + - Use Canny? \ No newline at end of file diff --git a/docs/docs/about/04-product/01-management.md b/docs/docs/how-we-work/project-management/project-management.md similarity index 90% rename from docs/docs/about/04-product/01-management.md rename to docs/docs/how-we-work/project-management/project-management.md index c231801c0..58af4a0d3 100644 --- a/docs/docs/about/04-product/01-management.md +++ b/docs/docs/how-we-work/project-management/project-management.md @@ -1,6 +1,5 @@ --- title: Project Management -slug: /product/management --- We use the [Jan Monorepo Project](https://github.com/orgs/janhq/projects/5) in Github to manage our roadmap and sprint Kanbans. @@ -58,3 +57,10 @@ We aim to always sprint on `tasks` that are a part of the [current roadmap](http - `Urgent bugs`: assign to an owner (or @engineers if you are not sure) && tag the current `sprint` & `milestone` - `All else`: assign the correct roadmap `label(s)` and owner (if any) + + +#### Request for help + +As a result, our feature prioritization can feel a bit black box at times. + +We'd appreciate high quality insights and volunteers for user interviews through [Discord](https://discord.gg/af6SaTdzpx) and [Github](https://github.com/janhq). diff --git a/docs/docs/how-we-work/strategy/strategy.md b/docs/docs/how-we-work/strategy/strategy.md new file mode 100644 index 000000000..09d9b9fb4 --- /dev/null +++ b/docs/docs/how-we-work/strategy/strategy.md @@ -0,0 +1,53 @@ +--- +title: Strategy +--- + +We only have 2 planning parameters: +- 10 year vision +- 2 week sprint +- Quarterly OKRs + +### Ideal Customer + +Our ideal customer is an AI enthusiast or business who has experienced some limitations with current AI solutions and is keen to find open source alternatives. + +### Problems + +Our ideal customer would use Jan to solve one of these problems. + +_Control_ + +- Control (e.g. preventing vendor lock-in) +- Stability (e.g. runs predictably every time) +- Local-use (e.g. for speed, or for airgapped environments) + +_Privacy_ + +- Data protection (e.g. personal data or company data) +- Privacy (e.g. nsfw) + +_Customisability_ + +- Tinkerability (e.g. ability to change model, experiment) +- Niche Models (e.g. fine-tuned, domain-specific models that outperform OpenAI) + +Sources: [^1] [^2] [^3] [^4] + +[^1]: [What are you guys doing that can't be done with ChatGPT?](https://www.reddit.com/r/LocalLLaMA/comments/17mghqr/comment/k7ksti6/?utm_source=share&utm_medium=web2x&context=3) +[^2]: [What's your main interest in running a local LLM instead of an existing API?](https://www.reddit.com/r/LocalLLaMA/comments/1718a9o/whats_your_main_interest_in_running_a_local_llm/) +[^3]: [Ask HN: What's the best self-hosted/local alternative to GPT-4?](https://news.ycombinator.com/item?id=36138224) +[^4]: [LoRAs](https://www.reddit.com/r/LocalLLaMA/comments/17mghqr/comment/k7mdz1i/?utm_source=share&utm_medium=web2x&context=3) + +### Solution + +Jan is a seamless user experience that runs on your personal computer, that glues the different pieces of the open source AI ecosystem to provide an alternative to OpenAI's closed platform. + +- We build a comprehensive, seamless platform that takes care of the technical chores across the stack required to run open source AI +- We run on top of a local folder of non-proprietary files, that anyone can tinker with (yes, even other apps!) +- We provide open formats for packaging and distributing AI to run reproducibly across devices + + +## Prerequisites + +- [Figma](https://figma.com) +- [ScreenStudio](https://www.screen.studio/) diff --git a/docs/docs/how-we-work/website-docs/website-docs.md b/docs/docs/how-we-work/website-docs/website-docs.md new file mode 100644 index 000000000..a152c6dc9 --- /dev/null +++ b/docs/docs/how-we-work/website-docs/website-docs.md @@ -0,0 +1,3 @@ +--- +title: Website & Docs +--- \ No newline at end of file diff --git a/docs/docs/integrations.md b/docs/docs/integrations.md new file mode 100644 index 000000000..0884d2242 --- /dev/null +++ b/docs/docs/integrations.md @@ -0,0 +1,3 @@ +--- +title: Integrations +--- \ No newline at end of file diff --git a/docs/docs/docs/02-integrations/langchain.md b/docs/docs/integrations/langchain.md similarity index 100% rename from docs/docs/docs/02-integrations/langchain.md rename to docs/docs/integrations/langchain.md diff --git a/docs/docs/docs/02-integrations/llamacpp.md b/docs/docs/integrations/llamacpp.md similarity index 94% rename from docs/docs/docs/02-integrations/llamacpp.md rename to docs/docs/integrations/llamacpp.md index 9d5004982..2764187c1 100644 --- a/docs/docs/docs/02-integrations/llamacpp.md +++ b/docs/docs/integrations/llamacpp.md @@ -1,5 +1,5 @@ --- -title: LlamaCPP +title: llama.cpp --- ## Quicklinks diff --git a/docs/docs/docs/02-integrations/ollama.md b/docs/docs/integrations/ollama.md similarity index 100% rename from docs/docs/docs/02-integrations/ollama.md rename to docs/docs/integrations/ollama.md diff --git a/docs/docs/docs/02-integrations/openai.md b/docs/docs/integrations/openai.md similarity index 100% rename from docs/docs/docs/02-integrations/openai.md rename to docs/docs/integrations/openai.md diff --git a/docs/docs/docs/02-integrations/openrouter.md b/docs/docs/integrations/openrouter.md similarity index 100% rename from docs/docs/docs/02-integrations/openrouter.md rename to docs/docs/integrations/openrouter.md diff --git a/docs/docs/docs/02-integrations/tensorrt.md b/docs/docs/integrations/tensorrt.md similarity index 100% rename from docs/docs/docs/02-integrations/tensorrt.md rename to docs/docs/integrations/tensorrt.md diff --git a/docs/docs/partners/become-a-partner.md b/docs/docs/partners/become-a-partner.md new file mode 100644 index 000000000..dce93e87a --- /dev/null +++ b/docs/docs/partners/become-a-partner.md @@ -0,0 +1,3 @@ +--- +title: Become a Partner +--- \ No newline at end of file diff --git a/docs/docs/partners/partners.md b/docs/docs/partners/partners.md new file mode 100644 index 000000000..1c5eaa0f2 --- /dev/null +++ b/docs/docs/partners/partners.md @@ -0,0 +1,3 @@ +--- +title: Partners +--- \ No newline at end of file diff --git a/docs/docs/platforms/desktop.md b/docs/docs/platforms/desktop.md new file mode 100644 index 000000000..fb4ea8389 --- /dev/null +++ b/docs/docs/platforms/desktop.md @@ -0,0 +1,84 @@ +--- +title: Jan Desktop +slug: /desktop +description: Turn your computer into an AI PC +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + ] +--- + +# Turn any computer into an AI computer + +![Alt text](image.png) + + + +### Designed for Everyone + +- **Installs with 1 click.** Jan is easy to use, beautiful, and the core features are free forever. +- **Runs on consumer laptops.** Tested by 250k+ users across `Windows, Mac, and Linux`, Jan even works on `CPU-only mode`. +- **Automatic GPU acceleration**. Models responds faster across `NVIDIA, AMD, Apple, and Intel` when benchmarked against industry alternatives. + +:::tip + +“Normies” can run LLMs like an AI engineer on the Desktop App - no programming experience needed. + +::: + +### Private & Offline + +- **Runs 100% locally**. AI models run directly on your laptop without an internet connection. `You own your AI`. +- **Private conversations.** Data is saved on your own filesystem in a transparent non-proprietary data format. `You own your data.` +- **Open source security**. Jan is open source, so you can scruntinize every line in our codebase. So when we say your data is not our product, we mean it. See [the code](https://github.com/janhq/jan) and our [data policy](/how-we-work/analytics). + +### Customizable AI + +- **Use AI without limitations.** Take control of censorship levels. Jan is usable from the classroom to the boardroom (and few other rooms if that’s your jam). +- **Use any models**. Download open source models from HuggingFace or upload your own custom models. [link] + +:::tip + +Download Jan Desktop Client [here](https://github.com/janhq/jan?tab=readme-ov-file#download). + +::: + +## Jan Desktop is + +### For Developers + +**You can easily integrate a locally running LLM into your own projects.** + +- Turn on `Local API Server` mode to start building on an `OpenAI compatible API`. +- Jan Desktop comes with a Developer Console out of the box. +- The UI makes it easy to see logs, configure models and more. + +![Alt text](image-1.png) + + + +### For Your Home Server + + + +- Use Jan as a UI only, pointing to a different backend. [See Docs](/guides/using-models/integrate-with-remote-server) +- Use Jan as a backend only, pointing to a different frontend. [See Docs](/guides/using-models/integrate-with-remote-server) +- Run Jan in team-mode on production-grade GPUs. [See Server Suite](/server-suite) + + + +### For People who Tinker + +- Customize the app’s look and feel though Themes. +- Customize assistants, models and other features with **no code**. +- Customize the entire Application through Extensions. Inspired by VSCode extensions, the desktop app can be entirely customized. + +[See the default extensions](https://github.com/janhq/jan/tree/main/extensions) bundled with every Desktop install. +![Alt text](image-4.png) diff --git a/docs/docs/platforms/hub.md b/docs/docs/platforms/hub.md new file mode 100644 index 000000000..9d8167618 --- /dev/null +++ b/docs/docs/platforms/hub.md @@ -0,0 +1,3 @@ +--- +title: Jan Hub +--- \ No newline at end of file diff --git a/docs/docs/platforms/image-1.png b/docs/docs/platforms/image-1.png new file mode 100644 index 000000000..cd94f34e9 Binary files /dev/null and b/docs/docs/platforms/image-1.png differ diff --git a/docs/docs/platforms/image-2.png b/docs/docs/platforms/image-2.png new file mode 100644 index 000000000..cc75d39ce Binary files /dev/null and b/docs/docs/platforms/image-2.png differ diff --git a/docs/docs/platforms/image-3.png b/docs/docs/platforms/image-3.png new file mode 100644 index 000000000..20828beb9 Binary files /dev/null and b/docs/docs/platforms/image-3.png differ diff --git a/docs/docs/platforms/image-4.png b/docs/docs/platforms/image-4.png new file mode 100644 index 000000000..38b75c1da Binary files /dev/null and b/docs/docs/platforms/image-4.png differ diff --git a/docs/docs/platforms/image.png b/docs/docs/platforms/image.png new file mode 100644 index 000000000..0237898c7 Binary files /dev/null and b/docs/docs/platforms/image.png differ diff --git a/docs/docs/platforms/mobile.md b/docs/docs/platforms/mobile.md new file mode 100644 index 000000000..827544201 --- /dev/null +++ b/docs/docs/platforms/mobile.md @@ -0,0 +1,16 @@ +--- +title: Jan Mobile +slug: /mobile +description: Jan Mobile allows you to bring your AI on the go +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + ] +--- \ No newline at end of file diff --git a/docs/docs/pricing/pricing.md b/docs/docs/pricing/pricing.md new file mode 100644 index 000000000..233610468 --- /dev/null +++ b/docs/docs/pricing/pricing.md @@ -0,0 +1,10 @@ +--- +title: Pricing +slug: /pricing +--- + +| $0 | $1 | Enterprise | +| ---------------- | ---------------- | ----------------------- | +| Free | Premium | TBA | +| ✅ Core features | ✅ Discord badge | ✅ Enterprise-level SLA | +| ✅ free forever | | | diff --git a/docs/docs/server-suite/admin-console.md b/docs/docs/server-suite/admin-console.md new file mode 100644 index 000000000..e2da5c096 --- /dev/null +++ b/docs/docs/server-suite/admin-console.md @@ -0,0 +1,3 @@ +--- +title: Admin Console +--- \ No newline at end of file diff --git a/docs/docs/server-suite/audit-compliance.md b/docs/docs/server-suite/audit-compliance.md new file mode 100644 index 000000000..af4684899 --- /dev/null +++ b/docs/docs/server-suite/audit-compliance.md @@ -0,0 +1,3 @@ +--- +title: Audit & Compliance +--- \ No newline at end of file diff --git a/docs/docs/server-suite/enterprise.md b/docs/docs/server-suite/enterprise.md new file mode 100644 index 000000000..565c14fde --- /dev/null +++ b/docs/docs/server-suite/enterprise.md @@ -0,0 +1,100 @@ +--- +title: Jan Enterprise +slug: /enterprise +description: Built for Enterprise Deployments +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + ] +--- + +# Customize and run AI across your organization + +Jan can professional backend to create, customize and run AIs at scale, for production-grade data centers. + +:::warning + +The server suite is actively under development and lacking documentation. +You can find the source code [here](https://github.com/janhq/jan/tree/dev/server) and [here](https://github.com/janhq/jan/blob/dev/docker-compose.yml). + +It is free to use. Your feedback is appreciated 🙏. + +::: + +## Own your AI. Own your data. Own your IP. + +Over time, we expect more teams and organizations to turn to running their own AIs on-prem. + +**Why?** + +- Prevent shadow data +- Avoid vendor lock-in +- Keep your IP in house +- Uptime and support predictability +- Eliminate monthly API bills - use your existing hardware +- Full control over your AI - you can open it up and see what's going on + +## Why Jan Enterprise + +### Fast deployment + +- **1 click deployment**. Immediately serve, customize, and scale models and assistants across your org. Scale your AI team so they can focus on the IP instead of fixing plumbing across every computer. +- **Scale across infrastructures**: on premise, with cloud providers, or as a hybrid deployment. Run Jan in completely air-gapped environments. +- **Optimized for datacenter-grade GPUs**: Can run on Nvidia, AMD Hardware, or even normal CPUs. Use TensorRT-LLM for more speedups on A6000s and above. + +### Full customization + +- Runs custom models or popular LLMs like Llama2, Mistral at production scale +- API that is fully OpenAI-compatible, i.e. can be a drop-in migration +- Powerful Agent framework to customize LLMs using RAG or Enterprise Data integrations. + +:::tip + +Not a Jan fan but convinced about local AI? No worries, here's a list of [awesome local ai](https://github.com/janhq/awesome-local-ai) alternatives that you can use in your team. + +::: + +## Supported Extensions + +The SDK and current implemention accomodate the following potential extensions. + +### Admin console + +Integrate SAML, OAUTH, OIDC + + + +### Identity access management + +Grant roles, groups and general ACL + + + +### Audit compliance + +Plug in Guardrails, LLMGuard, your custom rules engine and more + + + +### Observability + +Plug in Langfuse, Langsmith, Openllmetry and more + + + +## Enterprise support SLA + +Our core team and AI solutions partners are to help. + +Email us at: `inquiries@jan.ai` for: + +- Priority case routing +- Proactive case monitoring +- 24-hour support response diff --git a/docs/docs/server-suite/home-server.md b/docs/docs/server-suite/home-server.md new file mode 100644 index 000000000..97f3afbc7 --- /dev/null +++ b/docs/docs/server-suite/home-server.md @@ -0,0 +1,50 @@ +--- +title: Jan Home Server +slug: /home-server +description: Built for Home Servers +keywords: + [ + Jan AI, + Jan, + ChatGPT alternative, + local AI, + private AI, + conversational AI, + no-subscription fee, + large language model, + ] +--- + +# Customize and run AI across all of your devices + +Self-host and access your AI from anywhere with Jan server suite. + +:::warning + +Jan's server suite is actively under development and lacking documentation. +You can find the source code [here](https://github.com/janhq/jan/tree/dev/server) and [here](https://github.com/janhq/jan/blob/dev/docker-compose.yml). + +It is free to use. Your feedback is appreciated 🙏. + +::: + +## Why Home Servers + +We built [Jan Desktop](/desktop) for our personal use. We're now building Server Suite, for our team & community use. + +Our goal is to help teams, like ours, move past cobbling together demo apps to use AI at work. We should be able to customize and collaborate with AIs that are usable on a daily basis. + +**Check out [Server Suite](https://github.com/janhq/jan/tree/dev/server) if you need to:** + +- Self-host Jan, with multi client sync +- Customize it with Personal Data Connectors +- Simple Authentication (username / pw) +- Scales across Consumer-grade Hardware, including GPUs +- Everyone has admin level visibility and can see all conversations +- Create assistants that has access to the same knowledge base + +:::tip + +Not a Jan fan but convinced about running AI locally? No worries, here's a list of [awesome local ai](https://github.com/janhq/awesome-local-ai) alternatives that you can use in your home server. + +::: diff --git a/docs/docs/server-suite/identity-access-management.md b/docs/docs/server-suite/identity-access-management.md new file mode 100644 index 000000000..bdeafa9bd --- /dev/null +++ b/docs/docs/server-suite/identity-access-management.md @@ -0,0 +1,3 @@ +--- +title: Identity & Access Control +--- \ No newline at end of file diff --git a/docs/docs/server-suite/observability.md b/docs/docs/server-suite/observability.md new file mode 100644 index 000000000..6371a6287 --- /dev/null +++ b/docs/docs/server-suite/observability.md @@ -0,0 +1,3 @@ +--- +title: Observability +--- \ No newline at end of file diff --git a/docs/docs/solutions/ai-employees.md b/docs/docs/solutions/ai-employees.md new file mode 100644 index 000000000..53f425ab5 --- /dev/null +++ b/docs/docs/solutions/ai-employees.md @@ -0,0 +1,4 @@ +--- +title: AI Employees +tags: [use-cases] +--- \ No newline at end of file diff --git a/docs/docs/solutions/ai-pc.md b/docs/docs/solutions/ai-pc.md new file mode 100644 index 000000000..415dadef9 --- /dev/null +++ b/docs/docs/solutions/ai-pc.md @@ -0,0 +1,4 @@ +--- +title: AI PC +tags: [use-cases] +--- \ No newline at end of file diff --git a/docs/docs/solutions/chatgpt-alternative.md b/docs/docs/solutions/chatgpt-alternative.md new file mode 100644 index 000000000..84ec2a59b --- /dev/null +++ b/docs/docs/solutions/chatgpt-alternative.md @@ -0,0 +1,4 @@ +--- +title: Self-hosted alternative to OpenAI's Platform +tags: [use-cases] +--- \ No newline at end of file diff --git a/docs/docs/solutions/consultants.md b/docs/docs/solutions/consultants.md new file mode 100644 index 000000000..085a84954 --- /dev/null +++ b/docs/docs/solutions/consultants.md @@ -0,0 +1,4 @@ +--- +title: Software Consultants +tags: [audiences] +--- \ No newline at end of file diff --git a/docs/docs/solutions/developers.md b/docs/docs/solutions/developers.md new file mode 100644 index 000000000..e912365bf --- /dev/null +++ b/docs/docs/solutions/developers.md @@ -0,0 +1,4 @@ +--- +title: Developers +tags: [audiences] +--- \ No newline at end of file diff --git a/docs/docs/solutions/enterprises.md b/docs/docs/solutions/enterprises.md new file mode 100644 index 000000000..0a219ee74 --- /dev/null +++ b/docs/docs/solutions/enterprises.md @@ -0,0 +1,4 @@ +--- +title: Enterprises +tags: [audiences] +--- \ No newline at end of file diff --git a/docs/docs/solutions/finance.md b/docs/docs/solutions/finance.md new file mode 100644 index 000000000..dde66d618 --- /dev/null +++ b/docs/docs/solutions/finance.md @@ -0,0 +1,4 @@ +--- +title: Finance +tags: [industries] +--- \ No newline at end of file diff --git a/docs/docs/solutions/government.md b/docs/docs/solutions/government.md new file mode 100644 index 000000000..cb52f5d71 --- /dev/null +++ b/docs/docs/solutions/government.md @@ -0,0 +1,4 @@ +--- +title: Government +tags: [industries] +--- \ No newline at end of file diff --git a/docs/docs/solutions/healthcare.md b/docs/docs/solutions/healthcare.md new file mode 100644 index 000000000..1cd85463b --- /dev/null +++ b/docs/docs/solutions/healthcare.md @@ -0,0 +1,4 @@ +--- +title: Healthcare +tags: [industries] +--- \ No newline at end of file diff --git a/docs/docs/solutions/legal.md b/docs/docs/solutions/legal.md new file mode 100644 index 000000000..6cc88372c --- /dev/null +++ b/docs/docs/solutions/legal.md @@ -0,0 +1,4 @@ +--- +title: Legal +tags: [industries] +--- \ No newline at end of file diff --git a/docs/docs/solutions/startups.md b/docs/docs/solutions/startups.md new file mode 100644 index 000000000..23cdb5bbe --- /dev/null +++ b/docs/docs/solutions/startups.md @@ -0,0 +1,4 @@ +--- +title: Startups +tags: [audiences] +--- \ No newline at end of file diff --git a/docs/docs/studio/studio.md b/docs/docs/studio/studio.md new file mode 100644 index 000000000..e69de29bb diff --git a/docs/docs/team/contributor-program.md b/docs/docs/team/contributor-program.md new file mode 100644 index 000000000..cacdfa047 --- /dev/null +++ b/docs/docs/team/contributor-program.md @@ -0,0 +1,3 @@ +--- +title: Contributor Program +--- \ No newline at end of file diff --git a/docs/docs/team/join-us.md b/docs/docs/team/join-us.md new file mode 100644 index 000000000..84b8c263c --- /dev/null +++ b/docs/docs/team/join-us.md @@ -0,0 +1,12 @@ +--- +title: Join us +--- + +- [ ] Explain Core Team, Contributors and Open Source approach + + +[Careers on Bamboo](https://janai.bamboohr.com/careers) + +### Careers + +Jan has a culture of ownership, independent thought, and lightning fast execution. If you'd like to join us, we have open positions on our [careers page](https://janai.bamboohr.com/careers). \ No newline at end of file diff --git a/docs/docs/team/team.md b/docs/docs/team/team.md new file mode 100644 index 000000000..7d5e07cfb --- /dev/null +++ b/docs/docs/team/team.md @@ -0,0 +1,28 @@ +--- +title: Who we are +--- + +What's Jan the company about? +We aim to build the cognitive framework for future robots + +### Open Source + +Jan is a startup with an open source business model. We believe in the need for an open source AI ecosystem, and are committed to building it. + +- [Jan Framework](https://github.com/janhq/jan) (AGPLv3) +- [Jan Desktop Client & Local server](https://jan.ai) (AGPLv3, built on Jan Framework) +- [Nitro: run Local AI](https://github.com/janhq/nitro) (AGPLv3) + + +### Bootstrapped + +Jan is currently a bootstrapped startup. + +We balance technical invention with the search for a sustainable business model. Thus, we appreciate any business inquiries that can balance growth with cashflow. + +**We invite you to join us on our journey to find PMF**. Join our [Discord here](https://discord.gg/BnHRr3Q7Ms) + +## Our Team + +- Contributors +- Core Team \ No newline at end of file diff --git a/docs/docs/wall-of-love.md b/docs/docs/wall-of-love.md new file mode 100644 index 000000000..f196c90e9 --- /dev/null +++ b/docs/docs/wall-of-love.md @@ -0,0 +1,3 @@ +--- +title: Wall of Love ❤️ +--- \ No newline at end of file diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 4ed7f0bea..42ed55529 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -224,7 +224,7 @@ const config = { description: "Jan runs 100% offline on your computer, utilizes open-source AI models, prioritizes privacy, and is highly customizable.", keywords: - "Jan AI, Jan, ChatGPT alternative, local AI, private AI, conversational AI, no-subscription fee, large language model ", + "Jan AI, Jan, ChatGPT alternative, local AI, private AI, conversational AI, no-subscription fee, large language model ", applicationCategory: "BusinessApplication", operatingSystem: "Multiple", url: "https://jan.ai/", @@ -239,41 +239,84 @@ const config = { }, items: [ // Navbar Left + // { + // type: "docSidebar", + // sidebarId: "aboutSidebar", + // position: "left", + // label: "About", + // }, { - type: "docSidebar", - sidebarId: "guidesSidebar", + type: "dropdown", + label: "About", position: "left", - label: "Guides", + items: [ + { + type: "doc", + label: "What is Jan?", + docId: "about/about", + }, + { + type: "doc", + label: "Who we are", + docId: "team/team", + }, + { + type: "doc", + label: "Wall of love", + docId: "wall-of-love", + }, + ], }, { type: "docSidebar", - sidebarId: "developerSidebar", - position: "left", - label: "Developer", - }, - { - position: "left", - to: "/api-reference", - label: "API Reference", + sidebarId: "productSidebar", + positionL: "left", + label: "Product", }, { type: "docSidebar", + sidebarId: "ecosystemSidebar", position: "left", - sidebarId: "docsSidebar", - label: "Framework", + label: "Ecosystem", }, + // { + // type: "docSidebar", + // sidebarId: "pricingSidebar", + // positionL: "left", + // label: "Pricing", + // }, // Navbar right + { + type: "dropdown", + label: "Docs", + position: "right", + items: [ + { + type: "docSidebar", + sidebarId: "guidesSidebar", + label: "User Guide", + }, + { + type: "docSidebar", + sidebarId: "developerSidebar", + label: "Developer", + }, + { + to: "/api-reference", + label: "API Reference", + }, + { + type: "docSidebar", + sidebarId: "docsSidebar", + label: "Framework", + }, + ], + }, { to: "blog", label: "Blog", position: "right", }, - { - type: "docSidebar", - sidebarId: "aboutSidebar", - position: "right", - label: "About", - }, ], }, prism: { diff --git a/docs/sidebars.js b/docs/sidebars.js index 00eb8d40d..02ea7589f 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -13,33 +13,160 @@ /** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ const sidebars = { + aboutSidebar: [ + { + type: "category", + label: "What is Jan?", + link: { type: "doc", id: "about/about" }, + items: [ + //"about/roadmap", + "community/community", + ], + }, + { + type: "category", + label: "Who we are", + link: { type: "doc", id: "team/team" }, + items: ["team/join-us", "team/contributor-program"], + }, + "wall-of-love", + { + type: "category", + label: "How We Work", + link: { type: "doc", id: "how-we-work" }, + items: [ + "how-we-work/strategy/strategy", + "how-we-work/project-management/project-management", + { + type: "category", + label: "Engineering", + link: { type: "doc", id: "how-we-work/engineering/engineering" }, + items: [ + "how-we-work/engineering/ci-cd", + "how-we-work/engineering/qa", + ], + }, + "how-we-work/product-design/product-design", + "how-we-work/analytics/analytics", + "how-we-work/website-docs/website-docs", + ], + }, + ], + productSidebar: [ + { + type: "category", + label: "Platforms", + collapsible: false, + items: [ + "platforms/desktop", + "server-suite/home-server", + // "server-suite/enterprise", + // "platforms/mobile", + // "platforms/hub", + ], + }, + { + type: "category", + collapsible: true, + collapsed: false, + label: "Features", + link: { type: "doc", id: "features/features" }, + items: [ + "features/local", + "features/remote", + "features/api-server", + "features/extensions-framework", + "features/agents-framework", + "features/data-security", + ], + }, + // NOTE: Jan Server Suite will be torn out into it's own section in the future + // { + // type: "category", + // label: "Jan Server Suite", + // link: { type: "doc", id: "server-suite/server-suite" }, + // items: [ + // "server-suite/admin-console", + // "server-suite/identity-access-management", + // "server-suite/audit-compliance", + // "server-suite/observability", + // ], + // }, + ], + solutionSidebar: [ + { + type: "category", + label: "Use Cases", + collapsed: true, + collapsible: true, + items: ["solutions/ai-pc", "solutions/chatgpt-alternative"], + }, + { + type: "category", + label: "Sectors", + collapsed: true, + collapsible: true, + items: [ + "solutions/finance", + "solutions/healthcare", + "solutions/legal", + "solutions/government", + ], + }, + { + type: "category", + label: "Organization Type", + collapsed: true, + collapsible: true, + items: [ + "solutions/developers", + "solutions/consultants", + "solutions/startups", + "solutions/enterprises", + ], + }, + ], + + pricingSidebar: ["pricing/pricing"], + ecosystemSidebar: [ + "ecosystem/ecosystem", + { + type: "category", + label: "Partners", + link: { type: "doc", id: "partners/partners" }, + collapsible: true, + items: ["partners/become-a-partner"], + }, + { + type: "category", + label: "Integrations", + link: { type: "doc", id: "integrations" }, + items: [ + { + type: "autogenerated", + dirName: "integrations", + }, + ], + }, + ], guidesSidebar: [ { type: "autogenerated", dirName: "guides", }, ], - developerSidebar: [ { type: "autogenerated", dirName: "developer", }, ], - docsSidebar: [ { type: "autogenerated", dirName: "docs", }, ], - - aboutSidebar: [ - { - type: "autogenerated", - dirName: "about", - }, - ], }; module.exports = sidebars; diff --git a/docs/src/containers/Footer/index.js b/docs/src/containers/Footer/index.js index 80b5a2d4b..7cd648149 100644 --- a/docs/src/containers/Footer/index.js +++ b/docs/src/containers/Footer/index.js @@ -1,7 +1,7 @@ import React from "react"; import { AiOutlineGithub, AiOutlineTwitter } from "react-icons/ai"; -import { BiLogoDiscordAlt } from "react-icons/bi"; +import { BiLogoDiscordAlt, BiLogoLinkedin } from "react-icons/bi"; const socials = [ { @@ -16,6 +16,10 @@ const socials = [ icon: , href: "https://github.com/janhq/jan", }, + { + icon: , + href: "https://www.linkedin.com/company/janframework/", + } ]; const menus = [ @@ -59,6 +63,11 @@ const menus = [ path: "https://twitter.com/janframework", external: true, }, + { + menu: "LinkedIn", + path: "https://www.linkedin.com/company/janframework/", + external: true, + } ], }, { diff --git a/docs/static/img/bladerunner.png b/docs/static/img/bladerunner.png new file mode 100644 index 000000000..be3c6c634 Binary files /dev/null and b/docs/static/img/bladerunner.png differ diff --git a/docs/static/img/c3po-anakin.jpeg b/docs/static/img/c3po-anakin.jpeg new file mode 100644 index 000000000..f64917411 Binary files /dev/null and b/docs/static/img/c3po-anakin.jpeg differ diff --git a/docs/static/img/star-wars-droids.png b/docs/static/img/star-wars-droids.png new file mode 100644 index 000000000..41819af98 Binary files /dev/null and b/docs/static/img/star-wars-droids.png differ diff --git a/electron/.prettierrc b/electron/.prettierrc deleted file mode 100644 index 46f1abcb0..000000000 --- a/electron/.prettierrc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "semi": false, - "singleQuote": true, - "quoteProps": "consistent", - "trailingComma": "es5", - "endOfLine": "auto", - "plugins": ["prettier-plugin-tailwindcss"] -} diff --git a/electron/handlers/app.ts b/electron/handlers/app.ts deleted file mode 100644 index c1f431ef3..000000000 --- a/electron/handlers/app.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { app, ipcMain, dialog, shell } from 'electron' -import { join, basename, relative as getRelative, isAbsolute } from 'path' -import { WindowManager } from './../managers/window' -import { getResourcePath } from './../utils/path' -import { AppRoute, AppConfiguration } from '@janhq/core' -import { ServerConfig, startServer, stopServer } from '@janhq/server' -import { - ModuleManager, - getJanDataFolderPath, - getJanExtensionsPath, - init, - log, - logServer, - getAppConfigurations, - updateAppConfiguration, -} from '@janhq/core/node' - -export function handleAppIPCs() { - /** - * Handles the "openAppDirectory" IPC message by opening the app's user data directory. - * The `shell.openPath` method is used to open the directory in the user's default file explorer. - * @param _event - The IPC event object. - */ - ipcMain.handle(AppRoute.openAppDirectory, async (_event) => { - shell.openPath(getJanDataFolderPath()) - }) - - /** - * Opens a URL in the user's default browser. - * @param _event - The IPC event object. - * @param url - The URL to open. - */ - ipcMain.handle(AppRoute.openExternalUrl, async (_event, url) => { - shell.openExternal(url) - }) - - /** - * Opens a URL in the user's default browser. - * @param _event - The IPC event object. - * @param url - The URL to open. - */ - ipcMain.handle(AppRoute.openFileExplore, async (_event, url) => { - shell.openPath(url) - }) - - /** - * Joins multiple paths together, respect to the current OS. - */ - ipcMain.handle(AppRoute.joinPath, async (_event, paths: string[]) => - join(...paths) - ) - - /** - * Checks if the given path is a subdirectory of the given directory. - * - * @param _event - The IPC event object. - * @param from - The path to check. - * @param to - The directory to check against. - * - * @returns {Promise} - A promise that resolves with the result. - */ - ipcMain.handle( - AppRoute.isSubdirectory, - async (_event, from: string, to: string) => { - const relative = getRelative(from, to) - const isSubdir = - relative && !relative.startsWith('..') && !isAbsolute(relative) - - if (isSubdir === '') return false - else return isSubdir - } - ) - - /** - * Retrieve basename from given path, respect to the current OS. - */ - ipcMain.handle(AppRoute.baseName, async (_event, path: string) => - basename(path) - ) - - /** - * Start Jan API Server. - */ - ipcMain.handle(AppRoute.startServer, async (_event, configs?: ServerConfig) => - startServer({ - host: configs?.host, - port: configs?.port, - isCorsEnabled: configs?.isCorsEnabled, - isVerboseEnabled: configs?.isVerboseEnabled, - schemaPath: app.isPackaged - ? join(getResourcePath(), 'docs', 'openapi', 'jan.yaml') - : undefined, - baseDir: app.isPackaged - ? join(getResourcePath(), 'docs', 'openapi') - : undefined, - }) - ) - - /** - * Stop Jan API Server. - */ - ipcMain.handle(AppRoute.stopServer, stopServer) - - /** - * Relaunches the app in production - reload window in development. - * @param _event - The IPC event object. - * @param url - The URL to reload. - */ - ipcMain.handle(AppRoute.relaunch, async (_event) => { - ModuleManager.instance.clearImportedModules() - - if (app.isPackaged) { - app.relaunch() - app.exit() - } else { - for (const modulePath in ModuleManager.instance.requiredModules) { - delete require.cache[ - require.resolve(join(getJanExtensionsPath(), modulePath)) - ] - } - init({ - // Function to check from the main process that user wants to install a extension - confirmInstall: async (_extensions: string[]) => { - return true - }, - // Path to install extension to - extensionsPath: getJanExtensionsPath(), - }) - WindowManager.instance.currentWindow?.reload() - } - }) - - /** - * Log message to log file. - */ - ipcMain.handle(AppRoute.log, async (_event, message) => log(message)) - - /** - * Log message to log file. - */ - ipcMain.handle(AppRoute.logServer, async (_event, message) => - logServer(message) - ) - - ipcMain.handle(AppRoute.selectDirectory, async () => { - const mainWindow = WindowManager.instance.currentWindow - if (!mainWindow) { - console.error('No main window found') - return - } - const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, { - title: 'Select a folder', - buttonLabel: 'Select Folder', - properties: ['openDirectory', 'createDirectory'], - }) - if (canceled) { - return - } else { - return filePaths[0] - } - }) - - ipcMain.handle(AppRoute.getAppConfigurations, async () => - getAppConfigurations() - ) - - ipcMain.handle( - AppRoute.updateAppConfiguration, - async (_event, appConfiguration: AppConfiguration) => { - await updateAppConfiguration(appConfiguration) - } - ) -} diff --git a/electron/handlers/common.ts b/electron/handlers/common.ts new file mode 100644 index 000000000..5a54a92bd --- /dev/null +++ b/electron/handlers/common.ts @@ -0,0 +1,25 @@ +import { Handler, RequestHandler } from '@janhq/core/node' +import { ipcMain } from 'electron' +import { WindowManager } from '../managers/window' + +export function injectHandler() { + const ipcWrapper: Handler = ( + route: string, + listener: (...args: any[]) => any + ) => { + return ipcMain.handle(route, async (event, ...args: any[]) => { + return listener(...args) + }) + } + + const handler = new RequestHandler( + ipcWrapper, + (channel: string, args: any) => { + return WindowManager.instance.currentWindow?.webContents.send( + channel, + args + ) + } + ) + handler.handle() +} diff --git a/electron/handlers/download.ts b/electron/handlers/download.ts deleted file mode 100644 index f63e56f6b..000000000 --- a/electron/handlers/download.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { ipcMain } from 'electron' -import { resolve } from 'path' -import { WindowManager } from './../managers/window' -import request from 'request' -import { createWriteStream, renameSync } from 'fs' -import { DownloadEvent, DownloadRoute } from '@janhq/core' -const progress = require('request-progress') -import { DownloadManager, getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' - -export function handleDownloaderIPCs() { - /** - * Handles the "pauseDownload" IPC message by pausing the download associated with the provided fileName. - * @param _event - The IPC event object. - * @param fileName - The name of the file being downloaded. - */ - ipcMain.handle(DownloadRoute.pauseDownload, async (_event, fileName) => { - DownloadManager.instance.networkRequests[fileName]?.pause() - }) - - /** - * Handles the "resumeDownload" IPC message by resuming the download associated with the provided fileName. - * @param _event - The IPC event object. - * @param fileName - The name of the file being downloaded. - */ - ipcMain.handle(DownloadRoute.resumeDownload, async (_event, fileName) => { - DownloadManager.instance.networkRequests[fileName]?.resume() - }) - - /** - * Handles the "abortDownload" IPC message by aborting the download associated with the provided fileName. - * The network request associated with the fileName is then removed from the networkRequests object. - * @param _event - The IPC event object. - * @param fileName - The name of the file being downloaded. - */ - ipcMain.handle(DownloadRoute.abortDownload, async (_event, fileName) => { - const rq = DownloadManager.instance.networkRequests[fileName] - if (rq) { - DownloadManager.instance.networkRequests[fileName] = undefined - rq?.abort() - } else { - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadError, - { - fileName, - err: { message: 'aborted' }, - } - ) - } - }) - - /** - * Downloads a file from a given URL. - * @param _event - The IPC event object. - * @param url - The URL to download the file from. - * @param fileName - The name to give the downloaded file. - */ - ipcMain.handle( - DownloadRoute.downloadFile, - async (_event, url, fileName, network) => { - const strictSSL = !network?.ignoreSSL - const proxy = network?.proxy?.startsWith('http') - ? network.proxy - : undefined - - if (typeof fileName === 'string') { - fileName = normalizeFilePath(fileName) - } - const destination = resolve(getJanDataFolderPath(), fileName) - const rq = request({ url, strictSSL, proxy }) - - // Put request to download manager instance - DownloadManager.instance.setRequest(fileName, rq) - - // Downloading file to a temp file first - const downloadingTempFile = `${destination}.download` - - progress(rq, {}) - .on('progress', function (state: any) { - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadUpdate, - { - ...state, - fileName, - } - ) - }) - .on('error', function (err: Error) { - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadError, - { - fileName, - err, - } - ) - }) - .on('end', function () { - if (DownloadManager.instance.networkRequests[fileName]) { - // Finished downloading, rename temp file to actual file - renameSync(downloadingTempFile, destination) - - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadSuccess, - { - fileName, - } - ) - DownloadManager.instance.setRequest(fileName, undefined) - } else { - WindowManager?.instance.currentWindow?.webContents.send( - DownloadEvent.onFileDownloadError, - { - fileName, - err: { message: 'aborted' }, - } - ) - } - }) - .pipe(createWriteStream(downloadingTempFile)) - } - ) -} diff --git a/electron/handlers/extension.ts b/electron/handlers/extension.ts deleted file mode 100644 index 763c4cdec..000000000 --- a/electron/handlers/extension.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { ipcMain, webContents } from 'electron' -import { readdirSync } from 'fs' -import { join, extname } from 'path' - -import { - installExtensions, - getExtension, - removeExtension, - getActiveExtensions, - ModuleManager, - getJanExtensionsPath, -} from '@janhq/core/node' - -import { getResourcePath } from './../utils/path' -import { ExtensionRoute } from '@janhq/core' - -export function handleExtensionIPCs() { - /**MARK: General handlers */ - /** - * Invokes a function from a extension module in main node process. - * @param _event - The IPC event object. - * @param modulePath - The path to the extension module. - * @param method - The name of the function to invoke. - * @param args - The arguments to pass to the function. - * @returns The result of the invoked function. - */ - ipcMain.handle( - ExtensionRoute.invokeExtensionFunc, - async (_event, modulePath, method, ...args) => { - const module = require( - /* webpackIgnore: true */ join(getJanExtensionsPath(), modulePath) - ) - ModuleManager.instance.setModule(modulePath, module) - - if (typeof module[method] === 'function') { - return module[method](...args) - } else { - console.debug(module[method]) - console.error(`Function "${method}" does not exist in the module.`) - } - } - ) - - /** - * Returns the paths of the base extensions. - * @param _event - The IPC event object. - * @returns An array of paths to the base extensions. - */ - ipcMain.handle(ExtensionRoute.baseExtensions, async (_event) => { - const baseExtensionPath = join(getResourcePath(), 'pre-install') - return readdirSync(baseExtensionPath) - .filter((file) => extname(file) === '.tgz') - .map((file) => join(baseExtensionPath, file)) - }) - - /**MARK: Extension Manager handlers */ - ipcMain.handle(ExtensionRoute.installExtension, async (e, extensions) => { - // Install and activate all provided extensions - const installed = await installExtensions(extensions) - return JSON.parse(JSON.stringify(installed)) - }) - - // Register IPC route to uninstall a extension - ipcMain.handle( - ExtensionRoute.uninstallExtension, - async (e, extensions, reload) => { - // Uninstall all provided extensions - for (const ext of extensions) { - const extension = getExtension(ext) - await extension.uninstall() - if (extension.name) removeExtension(extension.name) - } - - // Reload all renderer pages if needed - reload && webContents.getAllWebContents().forEach((wc) => wc.reload()) - return true - } - ) - - // Register IPC route to update a extension - ipcMain.handle( - ExtensionRoute.updateExtension, - async (e, extensions, reload) => { - // Update all provided extensions - const updated: any[] = [] - for (const ext of extensions) { - const extension = getExtension(ext) - const res = await extension.update() - if (res) updated.push(extension) - } - - // Reload all renderer pages if needed - if (updated.length && reload) - webContents.getAllWebContents().forEach((wc) => wc.reload()) - - return JSON.parse(JSON.stringify(updated)) - } - ) - - // Register IPC route to get the list of active extensions - ipcMain.handle(ExtensionRoute.getActiveExtensions, () => { - return JSON.parse(JSON.stringify(getActiveExtensions())) - }) -} diff --git a/electron/handlers/fileManager.ts b/electron/handlers/fileManager.ts deleted file mode 100644 index e328cb53b..000000000 --- a/electron/handlers/fileManager.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { ipcMain, app } from 'electron' -// @ts-ignore -import reflect from '@alumna/reflect' - -import { FileManagerRoute, FileStat } from '@janhq/core' -import { getResourcePath } from './../utils/path' -import fs from 'fs' -import { join } from 'path' -import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' - -/** - * Handles file system extensions operations. - */ -export function handleFileMangerIPCs() { - // Handles the 'syncFile' IPC event. This event is triggered to synchronize a file from a source path to a destination path. - ipcMain.handle( - FileManagerRoute.syncFile, - async (_event, src: string, dest: string) => { - return reflect({ - src, - dest, - recursive: true, - delete: false, - overwrite: true, - errorOnExist: false, - }) - } - ) - - // Handles the 'getJanDataFolderPath' IPC event. This event is triggered to get the user space path. - ipcMain.handle( - FileManagerRoute.getJanDataFolderPath, - (): Promise => Promise.resolve(getJanDataFolderPath()) - ) - - // Handles the 'getResourcePath' IPC event. This event is triggered to get the resource path. - ipcMain.handle(FileManagerRoute.getResourcePath, async (_event) => - getResourcePath() - ) - - ipcMain.handle(FileManagerRoute.getUserHomePath, async (_event) => - app.getPath('home') - ) - - // handle fs is directory here - ipcMain.handle( - FileManagerRoute.fileStat, - async (_event, path: string): Promise => { - const normalizedPath = normalizeFilePath(path) - - const fullPath = join(getJanDataFolderPath(), normalizedPath) - const isExist = fs.existsSync(fullPath) - if (!isExist) return undefined - - const isDirectory = fs.lstatSync(fullPath).isDirectory() - const size = fs.statSync(fullPath).size - - const fileStat: FileStat = { - isDirectory, - size, - } - - return fileStat - } - ) - - ipcMain.handle( - FileManagerRoute.writeBlob, - async (_event, path: string, data: string): Promise => { - try { - const normalizedPath = normalizeFilePath(path) - const dataBuffer = Buffer.from(data, 'base64') - fs.writeFileSync( - join(getJanDataFolderPath(), normalizedPath), - dataBuffer - ) - } catch (err) { - console.error(`writeFile ${path} result: ${err}`) - } - } - ) -} diff --git a/electron/handlers/fs.ts b/electron/handlers/fs.ts deleted file mode 100644 index 34026b940..000000000 --- a/electron/handlers/fs.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { ipcMain } from 'electron' - -import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' -import fs from 'fs' -import { FileManagerRoute, FileSystemRoute } from '@janhq/core' -import { join } from 'path' -/** - * Handles file system operations. - */ -export function handleFsIPCs() { - const moduleName = 'fs' - Object.values(FileSystemRoute).forEach((route) => { - ipcMain.handle(route, async (event, ...args) => { - return import(moduleName).then((mdl) => - mdl[route]( - ...args.map((arg) => - typeof arg === 'string' && - (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) - ? join(getJanDataFolderPath(), normalizeFilePath(arg)) - : arg - ) - ) - ) - }) - }) -} diff --git a/electron/handlers/native.ts b/electron/handlers/native.ts new file mode 100644 index 000000000..14ead07bd --- /dev/null +++ b/electron/handlers/native.ts @@ -0,0 +1,86 @@ +import { app, ipcMain, dialog, shell } from 'electron' +import { join } from 'path' +import { WindowManager } from '../managers/window' +import { + ModuleManager, + getJanDataFolderPath, + getJanExtensionsPath, + init, +} from '@janhq/core/node' +import { NativeRoute } from '@janhq/core' + +export function handleAppIPCs() { + /** + * Handles the "openAppDirectory" IPC message by opening the app's user data directory. + * The `shell.openPath` method is used to open the directory in the user's default file explorer. + * @param _event - The IPC event object. + */ + ipcMain.handle(NativeRoute.openAppDirectory, async (_event) => { + shell.openPath(getJanDataFolderPath()) + }) + + /** + * Opens a URL in the user's default browser. + * @param _event - The IPC event object. + * @param url - The URL to open. + */ + ipcMain.handle(NativeRoute.openExternalUrl, async (_event, url) => { + shell.openExternal(url) + }) + + /** + * Opens a URL in the user's default browser. + * @param _event - The IPC event object. + * @param url - The URL to open. + */ + ipcMain.handle(NativeRoute.openFileExplore, async (_event, url) => { + shell.openPath(url) + }) + + /** + * Relaunches the app in production - reload window in development. + * @param _event - The IPC event object. + * @param url - The URL to reload. + */ + ipcMain.handle(NativeRoute.relaunch, async (_event) => { + ModuleManager.instance.clearImportedModules() + + if (app.isPackaged) { + app.relaunch() + app.exit() + } else { + for (const modulePath in ModuleManager.instance.requiredModules) { + delete require.cache[ + require.resolve(join(getJanExtensionsPath(), modulePath)) + ] + } + init({ + // Function to check from the main process that user wants to install a extension + confirmInstall: async (_extensions: string[]) => { + return true + }, + // Path to install extension to + extensionsPath: getJanExtensionsPath(), + }) + WindowManager.instance.currentWindow?.reload() + } + }) + + ipcMain.handle(NativeRoute.selectDirectory, async () => { + const mainWindow = WindowManager.instance.currentWindow + if (!mainWindow) { + console.error('No main window found') + return + } + const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, { + title: 'Select a folder', + buttonLabel: 'Select Folder', + properties: ['openDirectory', 'createDirectory'], + }) + if (canceled) { + return + } else { + return filePaths[0] + } + }) +} diff --git a/electron/handlers/update.ts b/electron/handlers/update.ts index cbb34c22b..0d8cc4cc0 100644 --- a/electron/handlers/update.ts +++ b/electron/handlers/update.ts @@ -11,7 +11,8 @@ export function handleAppUpdates() { /* New Update Available */ autoUpdater.on('update-available', async (_info: any) => { const action = await dialog.showMessageBox({ - message: `Update available. Do you want to download the latest update?`, + title: 'Update Available', + message: 'Would you like to download and install it now?', buttons: ['Download', 'Later'], }) if (action.response === 0) await autoUpdater.downloadUpdate() @@ -36,7 +37,7 @@ export function handleAppUpdates() { autoUpdater.on('error', (info: any) => { WindowManager.instance.currentWindow?.webContents.send( AppEvent.onAppUpdateDownloadError, - {} + info ) }) diff --git a/electron/main.ts b/electron/main.ts index 5d7e59c0f..de18b8f9d 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,4 +1,4 @@ -import { app, BrowserWindow } from 'electron' +import { app, BrowserWindow, shell } from 'electron' import { join } from 'path' /** * Managers @@ -9,12 +9,9 @@ import { log } from '@janhq/core/node' /** * IPC Handlers **/ -import { handleDownloaderIPCs } from './handlers/download' -import { handleExtensionIPCs } from './handlers/extension' -import { handleFileMangerIPCs } from './handlers/fileManager' -import { handleAppIPCs } from './handlers/app' +import { injectHandler } from './handlers/common' import { handleAppUpdates } from './handlers/update' -import { handleFsIPCs } from './handlers/fs' +import { handleAppIPCs } from './handlers/native' /** * Utils @@ -25,25 +22,12 @@ import { migrateExtensions } from './utils/migration' import { cleanUpAndQuit } from './utils/clean' import { setupExtensions } from './utils/extension' import { setupCore } from './utils/setup' +import { setupReactDevTool } from './utils/dev' +import { cleanLogs } from './utils/log' app .whenReady() - .then(async () => { - if (!app.isPackaged) { - // Which means you're running from source code - const { default: installExtension, REACT_DEVELOPER_TOOLS } = await import( - 'electron-devtools-installer' - ) // Don't use import on top level, since the installer package is dev-only - try { - const name = installExtension(REACT_DEVELOPER_TOOLS) - console.log(`Added Extension: ${name}`) - } catch (err) { - console.log('An error occurred while installing devtools:') - console.error(err) - // Only log the error and don't throw it because it's not critical - } - } - }) + .then(setupReactDevTool) .then(setupCore) .then(createUserSpace) .then(migrateExtensions) @@ -59,6 +43,7 @@ app } }) }) + .then(() => cleanLogs()) app.once('window-all-closed', () => { cleanUpAndQuit() @@ -92,23 +77,24 @@ function createMainWindow() { /* Open external links in the default browser */ mainWindow.webContents.setWindowOpenHandler(({ url }) => { - require('electron').shell.openExternal(url) + shell.openExternal(url) return { action: 'deny' } }) /* Enable dev tools for development */ if (!app.isPackaged) mainWindow.webContents.openDevTools() + log(`Version: ${app.getVersion()}`) } /** * Handles various IPC messages from the renderer process. */ function handleIPCs() { - handleFsIPCs() - handleDownloaderIPCs() - handleExtensionIPCs() + // Inject core handlers for IPCs + injectHandler() + + // Handle native IPCs handleAppIPCs() - handleFileMangerIPCs() } /* diff --git a/electron/merge-latest-ymls.js b/electron/merge-latest-ymls.js index 8172a3176..ee8caf825 100644 --- a/electron/merge-latest-ymls.js +++ b/electron/merge-latest-ymls.js @@ -9,7 +9,9 @@ const file3 = args[2] // check that all arguments are present and throw error instead if (!file1 || !file2 || !file3) { - throw new Error('Please provide 3 file paths as arguments: path to file1, to file2 and destination path') + throw new Error( + 'Please provide 3 file paths as arguments: path to file1, to file2 and destination path' + ) } const doc1 = yaml.load(fs.readFileSync(file1, 'utf8')) diff --git a/electron/package.json b/electron/package.json index 08f15b262..a89803077 100644 --- a/electron/package.json +++ b/electron/package.json @@ -4,6 +4,7 @@ "main": "./build/main.js", "author": "Jan ", "license": "MIT", + "productName": "Jan", "homepage": "https://github.com/janhq/jan/tree/main/electron", "description": "Use offline LLMs with your own data. Run open source models like Llama2 or Falcon on your internal computers/servers.", "build": { @@ -11,7 +12,6 @@ "productName": "Jan", "files": [ "renderer/**/*", - "build/*.{js,map}", "build/**/*.{js,map}", "pre-install", "models/**/*", @@ -57,16 +57,17 @@ "scripts": { "lint": "eslint . --ext \".js,.jsx,.ts,.tsx\"", "test:e2e": "playwright test --workers=1", - "dev": "tsc -p . && electron .", - "build": "run-script-os", - "build:test": "run-script-os", + "copy:assets": "rimraf --glob \"./pre-install/*.tgz\" && cpx \"../pre-install/*.tgz\" \"./pre-install\"", + "dev": "yarn copy:assets && tsc -p . && electron .", + "build": "yarn copy:assets && run-script-os", + "build:test": "yarn copy:assets && run-script-os", "build:test:darwin": "tsc -p . && electron-builder -p never -m --dir", "build:test:win32": "tsc -p . && electron-builder -p never -w --dir", "build:test:linux": "tsc -p . && electron-builder -p never -l --dir", "build:darwin": "tsc -p . && electron-builder -p never -m", "build:win32": "tsc -p . && electron-builder -p never -w", "build:linux": "tsc -p . && electron-builder -p never -l deb -l AppImage", - "build:publish": "run-script-os", + "build:publish": "yarn copy:assets && run-script-os", "build:publish:darwin": "tsc -p . && electron-builder -p always -m", "build:publish:win32": "tsc -p . && electron-builder -p always -w", "build:publish:linux": "tsc -p . && electron-builder -p always -l deb -l AppImage" @@ -76,7 +77,6 @@ "@janhq/core": "link:./core", "@janhq/server": "link:./server", "@npmcli/arborist": "^7.1.0", - "@types/request": "^2.48.12", "@uiball/loaders": "^1.3.0", "electron-store": "^8.1.0", "electron-updater": "^6.1.7", @@ -85,8 +85,6 @@ "pacote": "^17.0.4", "request": "^2.88.2", "request-progress": "^3.0.0", - "rimraf": "^5.0.5", - "typescript": "^5.2.2", "ulid": "^2.3.0", "use-debounce": "^9.0.4" }, @@ -95,6 +93,7 @@ "@playwright/test": "^1.38.1", "@types/npmcli__arborist": "^5.6.4", "@types/pacote": "^11.1.7", + "@types/request": "^2.48.12", "@typescript-eslint/eslint-plugin": "^6.7.3", "@typescript-eslint/parser": "^6.7.3", "electron": "28.0.0", @@ -102,7 +101,9 @@ "electron-devtools-installer": "^3.2.0", "electron-playwright-helpers": "^1.6.0", "eslint-plugin-react": "^7.33.2", - "run-script-os": "^1.1.6" + "rimraf": "^5.0.5", + "run-script-os": "^1.1.6", + "typescript": "^5.2.2" }, "installConfig": { "hoistingLimits": "workspaces" diff --git a/electron/playwright.config.ts b/electron/playwright.config.ts index 8047b7513..d3dff40c6 100644 --- a/electron/playwright.config.ts +++ b/electron/playwright.config.ts @@ -3,14 +3,12 @@ import { PlaywrightTestConfig } from '@playwright/test' const config: PlaywrightTestConfig = { testDir: './tests/e2e', retries: 0, - globalTimeout: 300000, + globalTimeout: 350000, use: { screenshot: 'only-on-failure', video: 'retain-on-failure', trace: 'retain-on-failure', }, - reporter: [['html', { outputFolder: './playwright-report' }]], } - export default config diff --git a/electron/sign.js b/electron/sign.js index 6e973eb6e..73afedc4e 100644 --- a/electron/sign.js +++ b/electron/sign.js @@ -1,44 +1,48 @@ -const { exec } = require('child_process'); +const { exec } = require('child_process') +function sign({ + path, + name, + certUrl, + clientId, + tenantId, + clientSecret, + certName, + timestampServer, + version, +}) { + return new Promise((resolve, reject) => { + const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"` -function sign({ path, name, certUrl, clientId, tenantId, clientSecret, certName, timestampServer, version }) { - return new Promise((resolve, reject) => { - - const command = `azuresigntool.exe sign -kvu "${certUrl}" -kvi "${clientId}" -kvt "${tenantId}" -kvs "${clientSecret}" -kvc "${certName}" -tr "${timestampServer}" -v "${path}"`; - - - exec(command, (error, stdout, stderr) => { - if (error) { - console.error(`Error: ${error}`); - return reject(error); - } - console.log(`stdout: ${stdout}`); - console.error(`stderr: ${stderr}`); - resolve(); - }); - }); + exec(command, (error, stdout, stderr) => { + if (error) { + console.error(`Error: ${error}`) + return reject(error) + } + console.log(`stdout: ${stdout}`) + console.error(`stderr: ${stderr}`) + resolve() + }) + }) } +exports.default = async function (options) { + const certUrl = process.env.AZURE_KEY_VAULT_URI + const clientId = process.env.AZURE_CLIENT_ID + const tenantId = process.env.AZURE_TENANT_ID + const clientSecret = process.env.AZURE_CLIENT_SECRET + const certName = process.env.AZURE_CERT_NAME + const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1' -exports.default = async function(options) { - - const certUrl = process.env.AZURE_KEY_VAULT_URI; - const clientId = process.env.AZURE_CLIENT_ID; - const tenantId = process.env.AZURE_TENANT_ID; - const clientSecret = process.env.AZURE_CLIENT_SECRET; - const certName = process.env.AZURE_CERT_NAME; - const timestampServer = 'http://timestamp.globalsign.com/tsa/r6advanced1'; - - - await sign({ - path: options.path, - name: "jan-win-x64", - certUrl, - clientId, - tenantId, - clientSecret, - certName, - timestampServer, - version: options.version - }); -}; + await sign({ + path: options.path, + name: 'jan-win-x64', + certUrl, + clientId, + tenantId, + clientSecret, + certName, + timestampServer, + version: options.version, + }) +} diff --git a/electron/tests/config/constants.ts b/electron/tests/config/constants.ts new file mode 100644 index 000000000..7039ad58c --- /dev/null +++ b/electron/tests/config/constants.ts @@ -0,0 +1,4 @@ +export const Constants = { + VIDEO_DIR: './playwright-video', + TIMEOUT: '300000', +} diff --git a/electron/tests/config/fixtures.ts b/electron/tests/config/fixtures.ts new file mode 100644 index 000000000..680b09785 --- /dev/null +++ b/electron/tests/config/fixtures.ts @@ -0,0 +1,119 @@ +import { + _electron as electron, + BrowserContext, + ElectronApplication, + expect, + Page, + test as base, +} from '@playwright/test' +import { + ElectronAppInfo, + findLatestBuild, + parseElectronApp, + stubDialog, +} from 'electron-playwright-helpers' +import { Constants } from './constants' +import { HubPage } from '../pages/hubPage' +import { CommonActions } from '../pages/commonActions' + +export let electronApp: ElectronApplication +export let page: Page +export let appInfo: ElectronAppInfo +export const TIMEOUT = parseInt(process.env.TEST_TIMEOUT || Constants.TIMEOUT) + +export async function setupElectron() { + process.env.CI = 'e2e' + + const latestBuild = findLatestBuild('dist') + expect(latestBuild).toBeTruthy() + + // parse the packaged Electron app and find paths and other info + appInfo = parseElectronApp(latestBuild) + expect(appInfo).toBeTruthy() + + electronApp = await electron.launch({ + args: [appInfo.main], // main file from package.json + executablePath: appInfo.executable, // path to the Electron executable + // recordVideo: { dir: Constants.VIDEO_DIR }, // Specify the directory for video recordings + }) + await stubDialog(electronApp, 'showMessageBox', { response: 1 }) + + page = await electronApp.firstWindow({ + timeout: TIMEOUT, + }) +} + +export async function teardownElectron() { + await page.close() + await electronApp.close() +} + +/** + * this fixture is needed to record and attach videos / screenshot on failed tests when + * tests are run in serial mode (i.e. browser is not closed between tests) + */ +export const test = base.extend< + { + commonActions: CommonActions + hubPage: HubPage + attachVideoPage: Page + attachScreenshotsToReport: void + }, + { createVideoContext: BrowserContext } +>({ + commonActions: async ({ request }, use, testInfo) => { + await use(new CommonActions(page, testInfo)) + }, + hubPage: async ({ commonActions }, use) => { + await use(new HubPage(page, commonActions)) + }, + createVideoContext: [ + async ({ playwright }, use) => { + const context = electronApp.context() + await use(context) + }, + { scope: 'worker' }, + ], + + attachVideoPage: [ + async ({ createVideoContext }, use, testInfo) => { + await use(page) + + if (testInfo.status !== testInfo.expectedStatus) { + const path = await createVideoContext.pages()[0].video()?.path() + await createVideoContext.close() + await testInfo.attach('video', { + path: path, + }) + } + }, + { scope: 'test', auto: true }, + ], + + attachScreenshotsToReport: [ + async ({ commonActions }, use, testInfo) => { + await use() + + // After the test, we can check whether the test passed or failed. + if (testInfo.status !== testInfo.expectedStatus) { + await commonActions.takeScreenshot('') + } + }, + { auto: true }, + ], +}) + +test.setTimeout(TIMEOUT) + +test.beforeAll(async () => { + await setupElectron() + await page.waitForSelector('img[alt="Jan - Logo"]', { + state: 'visible', + timeout: TIMEOUT, + }) +}) + +test.afterAll(async () => { + // temporally disabling this due to the config for parallel testing WIP + // teardownElectron() +}) diff --git a/electron/tests/e2e/hub.e2e.spec.ts b/electron/tests/e2e/hub.e2e.spec.ts index 68632058e..d968e7641 100644 --- a/electron/tests/e2e/hub.e2e.spec.ts +++ b/electron/tests/e2e/hub.e2e.spec.ts @@ -1,34 +1,19 @@ -import { - page, - test, - setupElectron, - teardownElectron, - TIMEOUT, -} from '../pages/basePage' +import { test, appInfo } from '../config/fixtures' import { expect } from '@playwright/test' test.beforeAll(async () => { - const appInfo = await setupElectron() - expect(appInfo.asar).toBe(true) - expect(appInfo.executable).toBeTruthy() - expect(appInfo.main).toBeTruthy() - expect(appInfo.name).toBe('jan') - expect(appInfo.packageJson).toBeTruthy() - expect(appInfo.packageJson.name).toBe('jan') - expect(appInfo.platform).toBeTruthy() - expect(appInfo.platform).toBe(process.platform) - expect(appInfo.resourcesDir).toBeTruthy() -}) - -test.afterAll(async () => { - await teardownElectron() -}) - -test('explores hub', async () => { - await page.getByTestId('Hub').first().click({ - timeout: TIMEOUT, - }) - await page.getByTestId('hub-container-test-id').isVisible({ - timeout: TIMEOUT, + expect(appInfo).toMatchObject({ + asar: true, + executable: expect.anything(), + main: expect.anything(), + name: 'jan', + packageJson: expect.objectContaining({ name: 'jan' }), + platform: process.platform, + resourcesDir: expect.anything(), }) }) + +test('explores hub', async ({ hubPage }) => { + await hubPage.navigateByMenu() + await hubPage.verifyContainerVisible() +}) diff --git a/electron/tests/e2e/navigation.e2e.spec.ts b/electron/tests/e2e/navigation.e2e.spec.ts index 2da59953c..b599a951c 100644 --- a/electron/tests/e2e/navigation.e2e.spec.ts +++ b/electron/tests/e2e/navigation.e2e.spec.ts @@ -1,32 +1,12 @@ import { expect } from '@playwright/test' -import { - page, - setupElectron, - TIMEOUT, - test, - teardownElectron, -} from '../pages/basePage' - -test.beforeAll(async () => { - await setupElectron() -}) - -test.afterAll(async () => { - await teardownElectron() -}) +import { page, test, TIMEOUT } from '../config/fixtures' test('renders left navigation panel', async () => { - const systemMonitorBtn = await page - .getByTestId('System Monitor') - .first() - .isEnabled({ - timeout: TIMEOUT, - }) const settingsBtn = await page .getByTestId('Thread') .first() .isEnabled({ timeout: TIMEOUT }) - expect([systemMonitorBtn, settingsBtn].filter((e) => !e).length).toBe(0) + expect([settingsBtn].filter((e) => !e).length).toBe(0) // Chat section should be there await page.getByTestId('Local API Server').first().click({ timeout: TIMEOUT, diff --git a/electron/tests/e2e/settings.e2e.spec.ts b/electron/tests/e2e/settings.e2e.spec.ts index 54215d9b1..06b4d1acc 100644 --- a/electron/tests/e2e/settings.e2e.spec.ts +++ b/electron/tests/e2e/settings.e2e.spec.ts @@ -1,23 +1,11 @@ import { expect } from '@playwright/test' -import { - setupElectron, - teardownElectron, - test, - page, - TIMEOUT, -} from '../pages/basePage' - -test.beforeAll(async () => { - await setupElectron() -}) - -test.afterAll(async () => { - await teardownElectron() -}) +import { test, page, TIMEOUT } from '../config/fixtures' test('shows settings', async () => { - await page.getByTestId('Settings').first().click({ timeout: TIMEOUT }) + await page.getByTestId('Settings').first().click({ + timeout: TIMEOUT, + }) const settingDescription = page.getByTestId('testid-setting-description') await expect(settingDescription).toBeVisible({ timeout: TIMEOUT }) }) diff --git a/electron/tests/pages/basePage.ts b/electron/tests/pages/basePage.ts index 5f1a6fca1..4e16a3c23 100644 --- a/electron/tests/pages/basePage.ts +++ b/electron/tests/pages/basePage.ts @@ -1,67 +1,49 @@ -import { - expect, - test as base, - _electron as electron, - ElectronApplication, - Page, -} from '@playwright/test' -import { - findLatestBuild, - parseElectronApp, - stubDialog, -} from 'electron-playwright-helpers' +import { Page, expect } from '@playwright/test' +import { CommonActions } from './commonActions' +import { TIMEOUT } from '../config/fixtures' -export const TIMEOUT: number = parseInt(process.env.TEST_TIMEOUT || '300000') +export class BasePage { + menuId: string -export let electronApp: ElectronApplication -export let page: Page + constructor( + protected readonly page: Page, + readonly action: CommonActions, + protected containerId: string + ) {} -export async function setupElectron() { - process.env.CI = 'e2e' + public getValue(key: string) { + return this.action.getValue(key) + } - const latestBuild = findLatestBuild('dist') - expect(latestBuild).toBeTruthy() + public setValue(key: string, value: string) { + this.action.setValue(key, value) + } - // parse the packaged Electron app and find paths and other info - const appInfo = parseElectronApp(latestBuild) - expect(appInfo).toBeTruthy() + async takeScreenshot(name: string = '') { + await this.action.takeScreenshot(name) + } - electronApp = await electron.launch({ - args: [appInfo.main], // main file from package.json - executablePath: appInfo.executable, // path to the Electron executable - }) - await stubDialog(electronApp, 'showMessageBox', { response: 1 }) + async navigateByMenu() { + await this.page.getByTestId(this.menuId).first().click() + } - page = await electronApp.firstWindow({ - timeout: TIMEOUT, - }) - // Return appInfo for future use - return appInfo + async verifyContainerVisible() { + const container = this.page.getByTestId(this.containerId) + expect(container.isVisible()).toBeTruthy() + } + + async waitUpdateLoader() { + await this.isElementVisible('img[alt="Jan - Logo"]') + } + + //wait and find a specific element with it's selector and return Visible + async isElementVisible(selector: any) { + let isVisible = true + await this.page + .waitForSelector(selector, { state: 'visible', timeout: TIMEOUT }) + .catch(() => { + isVisible = false + }) + return isVisible + } } - -export async function teardownElectron() { - await page.close() - await electronApp.close() -} - -export const test = base.extend<{ - attachScreenshotsToReport: void -}>({ - attachScreenshotsToReport: [ - async ({ request }, use, testInfo) => { - await use() - - // After the test, we can check whether the test passed or failed. - if (testInfo.status !== testInfo.expectedStatus) { - const screenshot = await page.screenshot() - await testInfo.attach('screenshot', { - body: screenshot, - contentType: 'image/png', - }) - } - }, - { auto: true }, - ], -}) - -test.setTimeout(TIMEOUT) diff --git a/electron/tests/pages/commonActions.ts b/electron/tests/pages/commonActions.ts new file mode 100644 index 000000000..08ea15f92 --- /dev/null +++ b/electron/tests/pages/commonActions.ts @@ -0,0 +1,34 @@ +import { Page, TestInfo } from '@playwright/test' +import { page } from '../config/fixtures' + +export class CommonActions { + private testData = new Map() + + constructor( + public page: Page, + public testInfo: TestInfo + ) {} + + async takeScreenshot(name: string) { + const screenshot = await page.screenshot({ + fullPage: true, + }) + const attachmentName = `${this.testInfo.title}_${name || new Date().toISOString().slice(5, 19).replace(/[-:]/g, '').replace('T', '_')}` + await this.testInfo.attach(attachmentName.replace(/\s+/g, ''), { + body: screenshot, + contentType: 'image/png', + }) + } + + async hooks() { + console.log('hook from the scenario page') + } + + setValue(key: string, value: string) { + this.testData.set(key, value) + } + + getValue(key: string) { + return this.testData.get(key) + } +} diff --git a/electron/tests/pages/hubPage.ts b/electron/tests/pages/hubPage.ts new file mode 100644 index 000000000..0299ab15d --- /dev/null +++ b/electron/tests/pages/hubPage.ts @@ -0,0 +1,15 @@ +import { Page } from '@playwright/test' +import { BasePage } from './basePage' +import { CommonActions } from './commonActions' + +export class HubPage extends BasePage { + readonly menuId: string = 'Hub' + static readonly containerId: string = 'hub-container-test-id' + + constructor( + public page: Page, + readonly action: CommonActions + ) { + super(page, action, HubPage.containerId) + } +} diff --git a/electron/utils/dev.ts b/electron/utils/dev.ts new file mode 100644 index 000000000..b2a492886 --- /dev/null +++ b/electron/utils/dev.ts @@ -0,0 +1,18 @@ +import { app } from 'electron' + +export const setupReactDevTool = async () => { + if (!app.isPackaged) { + // Which means you're running from source code + const { default: installExtension, REACT_DEVELOPER_TOOLS } = await import( + 'electron-devtools-installer' + ) // Don't use import on top level, since the installer package is dev-only + try { + const name = await installExtension(REACT_DEVELOPER_TOOLS) + console.log(`Added Extension: ${name}`) + } catch (err) { + console.log('An error occurred while installing devtools:') + console.error(err) + // Only log the error and don't throw it because it's not critical + } + } +} diff --git a/electron/utils/disposable.ts b/electron/utils/disposable.ts index 462f7e3e5..59018a775 100644 --- a/electron/utils/disposable.ts +++ b/electron/utils/disposable.ts @@ -1,8 +1,8 @@ export function dispose(requiredModules: Record) { for (const key in requiredModules) { - const module = requiredModules[key]; - if (typeof module["dispose"] === "function") { - module["dispose"](); + const module = requiredModules[key] + if (typeof module['dispose'] === 'function') { + module['dispose']() } } } diff --git a/electron/utils/log.ts b/electron/utils/log.ts new file mode 100644 index 000000000..84c185d75 --- /dev/null +++ b/electron/utils/log.ts @@ -0,0 +1,67 @@ +import { getJanDataFolderPath } from '@janhq/core/node' +import * as fs from 'fs' +import * as path from 'path' + +export function cleanLogs( + maxFileSizeBytes?: number | undefined, + daysToKeep?: number | undefined, + delayMs?: number | undefined +): void { + const size = maxFileSizeBytes ?? 1 * 1024 * 1024 // 1 MB + const days = daysToKeep ?? 7 // 7 days + const delays = delayMs ?? 10000 // 10 seconds + const logDirectory = path.join(getJanDataFolderPath(), 'logs') + + // Perform log cleaning + const currentDate = new Date() + fs.readdir(logDirectory, (err, files) => { + if (err) { + console.error('Error reading log directory:', err) + return + } + + files.forEach((file) => { + const filePath = path.join(logDirectory, file) + fs.stat(filePath, (err, stats) => { + if (err) { + console.error('Error getting file stats:', err) + return + } + + // Check size + if (stats.size > size) { + fs.unlink(filePath, (err) => { + if (err) { + console.error('Error deleting log file:', err) + return + } + console.log( + `Deleted log file due to exceeding size limit: ${filePath}` + ) + }) + } else { + // Check age + const creationDate = new Date(stats.ctime) + const daysDifference = Math.floor( + (currentDate.getTime() - creationDate.getTime()) / + (1000 * 3600 * 24) + ) + if (daysDifference > days) { + fs.unlink(filePath, (err) => { + if (err) { + console.error('Error deleting log file:', err) + return + } + console.log(`Deleted old log file: ${filePath}`) + }) + } + } + }) + }) + }) + + // Schedule the next execution with doubled delays + setTimeout(() => { + cleanLogs(maxFileSizeBytes, daysToKeep, delays * 2) + }, delays) +} diff --git a/electron/utils/menu.ts b/electron/utils/menu.ts index 7721b7c78..893907c48 100644 --- a/electron/utils/menu.ts +++ b/electron/utils/menu.ts @@ -1,19 +1,41 @@ // @ts-nocheck -import { app, Menu, dialog, shell } from 'electron' -const isMac = process.platform === 'darwin' +import { app, Menu, shell, dialog } from 'electron' import { autoUpdater } from 'electron-updater' -import { compareSemanticVersions } from './versionDiff' +import { log } from '@janhq/core/node' +const isMac = process.platform === 'darwin' const template: (Electron.MenuItemConstructorOptions | Electron.MenuItem)[] = [ { label: app.name, submenu: [ - { role: 'about' }, + { + label: `About ${app.name}`, + click: () => + dialog.showMessageBox({ + title: `Jan`, + message: `Jan Version v${app.getVersion()}\n\nCopyright © 2024 Jan`, + }), + }, { label: 'Check for Updates...', click: () => // Check for updates and notify user if there are any - autoUpdater.checkForUpdatesAndNotify(), + autoUpdater + .checkForUpdatesAndNotify() + .then((updateCheckResult) => { + if ( + !updateCheckResult?.updateInfo || + updateCheckResult?.updateInfo.version === app.getVersion() + ) { + dialog.showMessageBox({ + message: `No updates available.`, + }) + return + } + }) + .catch((error) => { + log('Error checking for updates:' + JSON.stringify(error)) + }), }, { type: 'separator' }, { role: 'services' }, diff --git a/electron/utils/path.ts b/electron/utils/path.ts index 4e47cc312..4438156bc 100644 --- a/electron/utils/path.ts +++ b/electron/utils/path.ts @@ -1,5 +1,3 @@ -import { join } from 'path' -import { app } from 'electron' import { mkdir } from 'fs-extra' import { existsSync } from 'fs' import { getJanDataFolderPath } from '@janhq/core/node' @@ -16,13 +14,3 @@ export async function createUserSpace(): Promise { } } } - -export function getResourcePath() { - let appPath = join(app.getAppPath(), '..', 'app.asar.unpacked') - - if (!app.isPackaged) { - // for development mode - appPath = join(__dirname, '..', '..') - } - return appPath -} diff --git a/electron/utils/setup.ts b/electron/utils/setup.ts index 887c3c2b7..01b0b31da 100644 --- a/electron/utils/setup.ts +++ b/electron/utils/setup.ts @@ -1,9 +1,9 @@ import { app } from 'electron' export const setupCore = async () => { - // Setup core api for main process - global.core = { - // Define appPath function for app to retrieve app path globaly - appPath: () => app.getPath('userData') - } -} \ No newline at end of file + // Setup core api for main process + global.core = { + // Define appPath function for app to retrieve app path globaly + appPath: () => app.getPath('userData'), + } +} diff --git a/electron/utils/versionDiff.ts b/electron/utils/versionDiff.ts deleted file mode 100644 index 25934e87f..000000000 --- a/electron/utils/versionDiff.ts +++ /dev/null @@ -1,21 +0,0 @@ -export const compareSemanticVersions = (a: string, b: string) => { - - // 1. Split the strings into their parts. - const a1 = a.split('.'); - const b1 = b.split('.'); - // 2. Contingency in case there's a 4th or 5th version - const len = Math.min(a1.length, b1.length); - // 3. Look through each version number and compare. - for (let i = 0; i < len; i++) { - const a2 = +a1[ i ] || 0; - const b2 = +b1[ i ] || 0; - - if (a2 !== b2) { - return a2 > b2 ? 1 : -1; - } - } - - // 4. We hit this if the all checked versions so far are equal - // - return b1.length - a1.length; -}; \ No newline at end of file diff --git a/extensions/assistant-extension/package.json b/extensions/assistant-extension/package.json index 84bcdf47e..baa858655 100644 --- a/extensions/assistant-extension/package.json +++ b/extensions/assistant-extension/package.json @@ -1,16 +1,17 @@ { "name": "@janhq/assistant-extension", - "version": "1.0.0", + "version": "1.0.1", "description": "This extension enables assistants, including Jan, a default assistant that can call all downloaded models", "main": "dist/index.js", "node": "dist/node/index.js", "author": "Jan ", "license": "AGPL-3.0", "scripts": { - "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install", - "build:publish:darwin": "rimraf *.tgz --glob && npm run build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../electron/pre-install", - "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install", + "clean:modules": "rimraf node_modules/pdf-parse/test && cd node_modules/pdf-parse/lib/pdf.js && rimraf v1.9.426 v1.10.88 v2.0.550", + "build": "yarn clean:modules && tsc --module commonjs && rollup -c rollup.config.ts", + "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install", + "build:publish:darwin": "rimraf *.tgz --glob && npm run build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../pre-install", + "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install", "build:publish": "run-script-os" }, "devDependencies": { @@ -25,7 +26,7 @@ "rollup-plugin-define": "^1.0.1", "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", - "typescript": "^5.3.3", + "typescript": "^5.2.2", "run-script-os": "^1.1.6" }, "dependencies": { @@ -44,9 +45,6 @@ ], "bundleDependencies": [ "@janhq/core", - "@langchain/community", - "hnswlib-node", - "langchain", - "pdf-parse" + "hnswlib-node" ] } diff --git a/extensions/assistant-extension/rollup.config.ts b/extensions/assistant-extension/rollup.config.ts index 7916ef9c8..d3c39cab2 100644 --- a/extensions/assistant-extension/rollup.config.ts +++ b/extensions/assistant-extension/rollup.config.ts @@ -1,22 +1,22 @@ -import resolve from "@rollup/plugin-node-resolve"; -import commonjs from "@rollup/plugin-commonjs"; -import sourceMaps from "rollup-plugin-sourcemaps"; -import typescript from "rollup-plugin-typescript2"; -import json from "@rollup/plugin-json"; -import replace from "@rollup/plugin-replace"; +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' -const packageJson = require("./package.json"); +const packageJson = require('./package.json') -const pkg = require("./package.json"); +const pkg = require('./package.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: "es", sourcemap: true }], + output: [{ file: pkg.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { - include: "src/**", + include: 'src/**', }, plugins: [ replace({ @@ -35,7 +35,7 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".js", ".ts", ".svelte"], + extensions: ['.js', '.ts', '.svelte'], }), // Resolve source maps to the original source @@ -44,18 +44,11 @@ export default [ }, { input: `src/node/index.ts`, - output: [{ dir: "dist/node", format: "cjs", sourcemap: false }], + output: [{ dir: 'dist/node', format: 'cjs', sourcemap: false }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: [ - "@janhq/core/node", - "@langchain/community", - "langchain", - "langsmith", - "path", - "hnswlib-node", - ], + external: ['@janhq/core/node', 'path', 'hnswlib-node'], watch: { - include: "src/node/**", + include: 'src/node/**', }, // inlineDynamicImports: true, plugins: [ @@ -71,11 +64,11 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".ts", ".js", ".json"], + extensions: ['.ts', '.js', '.json'], }), // Resolve source maps to the original source // sourceMaps(), ], }, -]; +] diff --git a/extensions/assistant-extension/src/@types/global.d.ts b/extensions/assistant-extension/src/@types/global.d.ts index dc11709a4..bc97157cd 100644 --- a/extensions/assistant-extension/src/@types/global.d.ts +++ b/extensions/assistant-extension/src/@types/global.d.ts @@ -1,3 +1,3 @@ -declare const NODE: string; -declare const EXTENSION_NAME: string; -declare const VERSION: string; +declare const NODE: string +declare const EXTENSION_NAME: string +declare const VERSION: string diff --git a/extensions/assistant-extension/src/index.ts b/extensions/assistant-extension/src/index.ts index 6495ea786..0a5319c8a 100644 --- a/extensions/assistant-extension/src/index.ts +++ b/extensions/assistant-extension/src/index.ts @@ -9,143 +9,169 @@ import { joinPath, executeOnMain, AssistantExtension, -} from "@janhq/core"; + AssistantEvent, +} from '@janhq/core' export default class JanAssistantExtension extends AssistantExtension { - private static readonly _homeDir = "file://assistants"; + private static readonly _homeDir = 'file://assistants' + private static readonly _threadDir = 'file://threads' - controller = new AbortController(); - isCancelled = false; - retrievalThreadId: string | undefined = undefined; + controller = new AbortController() + isCancelled = false + retrievalThreadId: string | undefined = undefined async onLoad() { // making the assistant directory const assistantDirExist = await fs.existsSync( - JanAssistantExtension._homeDir, - ); + JanAssistantExtension._homeDir + ) if ( localStorage.getItem(`${EXTENSION_NAME}-version`) !== VERSION || !assistantDirExist ) { - if (!assistantDirExist) - await fs.mkdirSync(JanAssistantExtension._homeDir); + if (!assistantDirExist) await fs.mkdirSync(JanAssistantExtension._homeDir) // Write assistant metadata - this.createJanAssistant(); + await this.createJanAssistant() // Finished migration - localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION); + localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION) + // Update the assistant list + events.emit(AssistantEvent.OnAssistantsUpdate, {}) } // Events subscription events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => - JanAssistantExtension.handleMessageRequest(data, this), - ); + JanAssistantExtension.handleMessageRequest(data, this) + ) events.on(InferenceEvent.OnInferenceStopped, () => { - JanAssistantExtension.handleInferenceStopped(this); - }); + JanAssistantExtension.handleInferenceStopped(this) + }) } private static async handleInferenceStopped(instance: JanAssistantExtension) { - instance.isCancelled = true; - instance.controller?.abort(); + instance.isCancelled = true + instance.controller?.abort() } private static async handleMessageRequest( data: MessageRequest, - instance: JanAssistantExtension, + instance: JanAssistantExtension ) { - instance.isCancelled = false; - instance.controller = new AbortController(); + instance.isCancelled = false + instance.controller = new AbortController() if ( data.model?.engine !== InferenceEngine.tool_retrieval_enabled || !data.messages || + // TODO: Since the engine is defined, its unsafe to assume that assistant tools are defined + // That could lead to an issue where thread stuck at generating response !data.thread?.assistants[0]?.tools ) { - return; + return } - const latestMessage = data.messages[data.messages.length - 1]; + const latestMessage = data.messages[data.messages.length - 1] - // Ingest the document if needed + // 1. Ingest the document if needed if ( latestMessage && latestMessage.content && - typeof latestMessage.content !== "string" + typeof latestMessage.content !== 'string' && + latestMessage.content.length > 1 ) { - const docFile = latestMessage.content[1]?.doc_url?.url; + const docFile = latestMessage.content[1]?.doc_url?.url if (docFile) { await executeOnMain( NODE, - "toolRetrievalIngestNewDocument", + 'toolRetrievalIngestNewDocument', docFile, - data.model?.proxyEngine, - ); + data.model?.proxyEngine + ) } + } else if ( + // Check whether we need to ingest document or not + // Otherwise wrong context will be sent + !(await fs.existsSync( + await joinPath([ + JanAssistantExtension._threadDir, + data.threadId, + 'memory', + ]) + )) + ) { + // No document ingested, reroute the result to inference engine + const output = { + ...data, + model: { + ...data.model, + engine: data.model.proxyEngine, + }, + } + events.emit(MessageEvent.OnMessageSent, output) + return } - - // Load agent on thread changed + // 2. Load agent on thread changed if (instance.retrievalThreadId !== data.threadId) { - await executeOnMain(NODE, "toolRetrievalLoadThreadMemory", data.threadId); + await executeOnMain(NODE, 'toolRetrievalLoadThreadMemory', data.threadId) - instance.retrievalThreadId = data.threadId; + instance.retrievalThreadId = data.threadId // Update the text splitter await executeOnMain( NODE, - "toolRetrievalUpdateTextSplitter", + 'toolRetrievalUpdateTextSplitter', data.thread.assistants[0].tools[0]?.settings?.chunk_size ?? 4000, - data.thread.assistants[0].tools[0]?.settings?.chunk_overlap ?? 200, - ); + data.thread.assistants[0].tools[0]?.settings?.chunk_overlap ?? 200 + ) } + // 3. Using the retrieval template with the result and query if (latestMessage.content) { const prompt = - typeof latestMessage.content === "string" + typeof latestMessage.content === 'string' ? latestMessage.content - : latestMessage.content[0].text; + : latestMessage.content[0].text // Retrieve the result - console.debug("toolRetrievalQuery", latestMessage.content); const retrievalResult = await executeOnMain( NODE, - "toolRetrievalQueryResult", - prompt, - ); + 'toolRetrievalQueryResult', + prompt + ) + console.debug('toolRetrievalQueryResult', retrievalResult) - // Update the message content - // Using the retrieval template with the result and query - if (data.thread?.assistants[0].tools) + // Update message content + if (data.thread?.assistants[0]?.tools && retrievalResult) data.messages[data.messages.length - 1].content = data.thread.assistants[0].tools[0].settings?.retrieval_template - ?.replace("{CONTEXT}", retrievalResult) - .replace("{QUESTION}", prompt); + ?.replace('{CONTEXT}', retrievalResult) + .replace('{QUESTION}', prompt) } // Filter out all the messages that are not text data.messages = data.messages.map((message) => { if ( message.content && - typeof message.content !== "string" && + typeof message.content !== 'string' && (message.content.length ?? 0) > 0 ) { return { ...message, content: [message.content[0]], - }; + } } - return message; - }); + return message + }) - // Reroute the result to inference engine + // 4. Reroute the result to inference engine const output = { ...data, model: { ...data.model, engine: data.model.proxyEngine, }, - }; - events.emit(MessageEvent.OnMessageSent, output); + } + events.emit(MessageEvent.OnMessageSent, output) } /** @@ -157,107 +183,107 @@ export default class JanAssistantExtension extends AssistantExtension { const assistantDir = await joinPath([ JanAssistantExtension._homeDir, assistant.id, - ]); - if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir); + ]) + if (!(await fs.existsSync(assistantDir))) await fs.mkdirSync(assistantDir) // store the assistant metadata json const assistantMetadataPath = await joinPath([ assistantDir, - "assistant.json", - ]); + 'assistant.json', + ]) try { await fs.writeFileSync( assistantMetadataPath, - JSON.stringify(assistant, null, 2), - ); + JSON.stringify(assistant, null, 2) + ) } catch (err) { - console.error(err); + console.error(err) } } async getAssistants(): Promise { // get all the assistant directories // get all the assistant metadata json - const results: Assistant[] = []; + const results: Assistant[] = [] const allFileName: string[] = await fs.readdirSync( - JanAssistantExtension._homeDir, - ); + JanAssistantExtension._homeDir + ) for (const fileName of allFileName) { const filePath = await joinPath([ JanAssistantExtension._homeDir, fileName, - ]); + ]) - if (filePath.includes(".DS_Store")) continue; + if (filePath.includes('.DS_Store')) continue const jsonFiles: string[] = (await fs.readdirSync(filePath)).filter( - (file: string) => file === "assistant.json", - ); + (file: string) => file === 'assistant.json' + ) if (jsonFiles.length !== 1) { // has more than one assistant file -> ignore - continue; + continue } const content = await fs.readFileSync( await joinPath([filePath, jsonFiles[0]]), - "utf-8", - ); + 'utf-8' + ) const assistant: Assistant = - typeof content === "object" ? content : JSON.parse(content); + typeof content === 'object' ? content : JSON.parse(content) - results.push(assistant); + results.push(assistant) } - return results; + return results } async deleteAssistant(assistant: Assistant): Promise { - if (assistant.id === "jan") { - return Promise.reject("Cannot delete Jan Assistant"); + if (assistant.id === 'jan') { + return Promise.reject('Cannot delete Jan Assistant') } // remove the directory const assistantDir = await joinPath([ JanAssistantExtension._homeDir, assistant.id, - ]); - await fs.rmdirSync(assistantDir); - return Promise.resolve(); + ]) + await fs.rmdirSync(assistantDir) + return Promise.resolve() } private async createJanAssistant(): Promise { const janAssistant: Assistant = { - avatar: "", + avatar: '', thread_location: undefined, - id: "jan", - object: "assistant", + id: 'jan', + object: 'assistant', created_at: Date.now(), - name: "Jan", - description: "A default assistant that can use all downloaded models", - model: "*", - instructions: "", + name: 'Jan', + description: 'A default assistant that can use all downloaded models', + model: '*', + instructions: '', tools: [ { - type: "retrieval", + type: 'retrieval', enabled: false, settings: { top_k: 2, chunk_size: 1024, chunk_overlap: 64, retrieval_template: `Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. - ---------------- - CONTEXT: {CONTEXT} - ---------------- - QUESTION: {QUESTION} - ---------------- - Helpful Answer:`, +---------------- +CONTEXT: {CONTEXT} +---------------- +QUESTION: {QUESTION} +---------------- +Helpful Answer:`, }, }, ], file_ids: [], metadata: undefined, - }; + } - await this.createAssistant(janAssistant); + await this.createAssistant(janAssistant) } } diff --git a/extensions/assistant-extension/src/node/engine.ts b/extensions/assistant-extension/src/node/engine.ts index 54b2a6ba1..70d02af1f 100644 --- a/extensions/assistant-extension/src/node/engine.ts +++ b/extensions/assistant-extension/src/node/engine.ts @@ -1,13 +1,13 @@ -import fs from "fs"; -import path from "path"; -import { getJanDataFolderPath } from "@janhq/core/node"; +import fs from 'fs' +import path from 'path' +import { getJanDataFolderPath } from '@janhq/core/node' // Sec: Do not send engine settings over requests // Read it manually instead export const readEmbeddingEngine = (engineName: string) => { const engineSettings = fs.readFileSync( - path.join(getJanDataFolderPath(), "engines", `${engineName}.json`), - "utf-8", - ); - return JSON.parse(engineSettings); -}; + path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), + 'utf-8' + ) + return JSON.parse(engineSettings) +} diff --git a/extensions/assistant-extension/src/node/index.ts b/extensions/assistant-extension/src/node/index.ts index 95a7243a4..d52a4b23e 100644 --- a/extensions/assistant-extension/src/node/index.ts +++ b/extensions/assistant-extension/src/node/index.ts @@ -1,39 +1,39 @@ -import { getJanDataFolderPath, normalizeFilePath } from "@janhq/core/node"; -import { Retrieval } from "./tools/retrieval"; -import path from "path"; +import { getJanDataFolderPath, normalizeFilePath } from '@janhq/core/node' +import { retrieval } from './tools/retrieval' +import path from 'path' -const retrieval = new Retrieval(); - -export async function toolRetrievalUpdateTextSplitter( +export function toolRetrievalUpdateTextSplitter( chunkSize: number, - chunkOverlap: number, + chunkOverlap: number ) { - retrieval.updateTextSplitter(chunkSize, chunkOverlap); - return Promise.resolve(); + retrieval.updateTextSplitter(chunkSize, chunkOverlap) } export async function toolRetrievalIngestNewDocument( file: string, - engine: string, + engine: string ) { - const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file)); - const threadPath = path.dirname(filePath.replace("files", "")); - retrieval.updateEmbeddingEngine(engine); - await retrieval.ingestAgentKnowledge(filePath, `${threadPath}/memory`); - return Promise.resolve(); + const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file)) + const threadPath = path.dirname(filePath.replace('files', '')) + retrieval.updateEmbeddingEngine(engine) + return retrieval + .ingestAgentKnowledge(filePath, `${threadPath}/memory`) + .catch((err) => { + console.error(err) + }) } export async function toolRetrievalLoadThreadMemory(threadId: string) { - try { - await retrieval.loadRetrievalAgent( - path.join(getJanDataFolderPath(), "threads", threadId, "memory"), - ); - return Promise.resolve(); - } catch (err) { - console.debug(err); - } + return retrieval + .loadRetrievalAgent( + path.join(getJanDataFolderPath(), 'threads', threadId, 'memory') + ) + .catch((err) => { + console.error(err) + }) } export async function toolRetrievalQueryResult(query: string) { - const res = await retrieval.generateResult(query); - return Promise.resolve(res); + return retrieval.generateResult(query).catch((err) => { + console.error(err) + }) } diff --git a/extensions/assistant-extension/src/node/tools/retrieval/index.ts b/extensions/assistant-extension/src/node/tools/retrieval/index.ts index 8c7a6aa2b..e58ec0c46 100644 --- a/extensions/assistant-extension/src/node/tools/retrieval/index.ts +++ b/extensions/assistant-extension/src/node/tools/retrieval/index.ts @@ -1,77 +1,80 @@ -import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; -import { formatDocumentsAsString } from "langchain/util/document"; -import { PDFLoader } from "langchain/document_loaders/fs/pdf"; +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter' +import { formatDocumentsAsString } from 'langchain/util/document' +import { PDFLoader } from 'langchain/document_loaders/fs/pdf' -import { HNSWLib } from "langchain/vectorstores/hnswlib"; +import { HNSWLib } from 'langchain/vectorstores/hnswlib' -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { readEmbeddingEngine } from "../../engine"; +import { OpenAIEmbeddings } from 'langchain/embeddings/openai' +import { readEmbeddingEngine } from '../../engine' export class Retrieval { - public chunkSize: number = 100; - public chunkOverlap?: number = 0; - private retriever: any; + public chunkSize: number = 100 + public chunkOverlap?: number = 0 + private retriever: any - private embeddingModel?: OpenAIEmbeddings = undefined; - private textSplitter?: RecursiveCharacterTextSplitter; + private embeddingModel?: OpenAIEmbeddings = undefined + private textSplitter?: RecursiveCharacterTextSplitter constructor(chunkSize: number = 4000, chunkOverlap: number = 200) { - this.updateTextSplitter(chunkSize, chunkOverlap); + this.updateTextSplitter(chunkSize, chunkOverlap) } public updateTextSplitter(chunkSize: number, chunkOverlap: number): void { - this.chunkSize = chunkSize; - this.chunkOverlap = chunkOverlap; + this.chunkSize = chunkSize + this.chunkOverlap = chunkOverlap this.textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: chunkSize, chunkOverlap: chunkOverlap, - }); + }) } public updateEmbeddingEngine(engine: string): void { // Engine settings are not compatible with the current embedding model params // Switch case manually for now - const settings = readEmbeddingEngine(engine); - if (engine === "nitro") { + const settings = readEmbeddingEngine(engine) + if (engine === 'nitro') { this.embeddingModel = new OpenAIEmbeddings( - { openAIApiKey: "nitro-embedding" }, - { basePath: "http://127.0.0.1:3928/v1" }, - ); + { openAIApiKey: 'nitro-embedding' }, + // TODO: Raw settings + { basePath: 'http://127.0.0.1:3928/v1' } + ) } else { // Fallback to OpenAI Settings this.embeddingModel = new OpenAIEmbeddings({ openAIApiKey: settings.api_key, - }); + }) } } public ingestAgentKnowledge = async ( filePath: string, - memoryPath: string, + memoryPath: string ): Promise => { const loader = new PDFLoader(filePath, { splitPages: true, - }); - if (!this.embeddingModel) return Promise.reject(); - const doc = await loader.load(); - const docs = await this.textSplitter!.splitDocuments(doc); - const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel); - return vectorStore.save(memoryPath); - }; + }) + if (!this.embeddingModel) return Promise.reject() + const doc = await loader.load() + const docs = await this.textSplitter!.splitDocuments(doc) + const vectorStore = await HNSWLib.fromDocuments(docs, this.embeddingModel) + return vectorStore.save(memoryPath) + } public loadRetrievalAgent = async (memoryPath: string): Promise => { - if (!this.embeddingModel) return Promise.reject(); - const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel); - this.retriever = vectorStore.asRetriever(2); - return Promise.resolve(); - }; + if (!this.embeddingModel) return Promise.reject() + const vectorStore = await HNSWLib.load(memoryPath, this.embeddingModel) + this.retriever = vectorStore.asRetriever(2) + return Promise.resolve() + } public generateResult = async (query: string): Promise => { if (!this.retriever) { - return Promise.resolve(" "); + return Promise.resolve(' ') } - const relevantDocs = await this.retriever.getRelevantDocuments(query); - const serializedDoc = formatDocumentsAsString(relevantDocs); - return Promise.resolve(serializedDoc); - }; + const relevantDocs = await this.retriever.getRelevantDocuments(query) + const serializedDoc = formatDocumentsAsString(relevantDocs) + return Promise.resolve(serializedDoc) + } } + +export const retrieval = new Retrieval() diff --git a/extensions/assistant-extension/tsconfig.json b/extensions/assistant-extension/tsconfig.json index d3794cace..e425358c3 100644 --- a/extensions/assistant-extension/tsconfig.json +++ b/extensions/assistant-extension/tsconfig.json @@ -14,7 +14,7 @@ "outDir": "dist", "importHelpers": true, "typeRoots": ["node_modules/@types"], - "skipLibCheck": true, + "skipLibCheck": true }, - "include": ["src"], + "include": ["src"] } diff --git a/extensions/conversational-extension/.prettierrc b/extensions/conversational-extension/.prettierrc deleted file mode 100644 index 46f1abcb0..000000000 --- a/extensions/conversational-extension/.prettierrc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "semi": false, - "singleQuote": true, - "quoteProps": "consistent", - "trailingComma": "es5", - "endOfLine": "auto", - "plugins": ["prettier-plugin-tailwindcss"] -} diff --git a/extensions/conversational-extension/package.json b/extensions/conversational-extension/package.json index a60c12339..8a6da14e5 100644 --- a/extensions/conversational-extension/package.json +++ b/extensions/conversational-extension/package.json @@ -7,7 +7,7 @@ "license": "MIT", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { ".": "./dist/index.js", @@ -17,12 +17,12 @@ "cpx": "^1.5.0", "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "dependencies": { "@janhq/core": "file:../../core", - "path-browserify": "^1.0.1", - "ts-loader": "^9.5.0" + "path-browserify": "^1.0.1" }, "engines": { "node": ">=18.0.0" diff --git a/extensions/conversational-extension/webpack.config.js b/extensions/conversational-extension/webpack.config.js index 36e338295..a3eb873d7 100644 --- a/extensions/conversational-extension/webpack.config.js +++ b/extensions/conversational-extension/webpack.config.js @@ -1,27 +1,27 @@ -const path = require("path"); -const webpack = require("webpack"); +const path = require('path') +const webpack = require('webpack') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], }, output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, plugins: [new webpack.DefinePlugin({})], resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { path: require.resolve('path-browserify'), }, @@ -31,4 +31,4 @@ module.exports = { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/inference-nitro-extension/README.md b/extensions/inference-nitro-extension/README.md index 455783efb..f499e0b9c 100644 --- a/extensions/inference-nitro-extension/README.md +++ b/extensions/inference-nitro-extension/README.md @@ -64,10 +64,10 @@ There are a few things to keep in mind when writing your plugin code: In `index.ts`, you will see that the extension function will return a `Promise`. ```typescript - import { core } from "@janhq/core"; + import { core } from '@janhq/core' function onStart(): Promise { - return core.invokePluginFunc(MODULE_PATH, "run", 0); + return core.invokePluginFunc(MODULE_PATH, 'run', 0) } ``` @@ -75,4 +75,3 @@ There are a few things to keep in mind when writing your plugin code: [documentation](https://github.com/janhq/jan/blob/main/core/README.md). So, what are you waiting for? Go ahead and start customizing your plugin! - diff --git a/extensions/inference-nitro-extension/bin/version.txt b/extensions/inference-nitro-extension/bin/version.txt index c2c0004f0..0b9c01996 100644 --- a/extensions/inference-nitro-extension/bin/version.txt +++ b/extensions/inference-nitro-extension/bin/version.txt @@ -1 +1 @@ -0.3.5 +0.3.12 diff --git a/extensions/inference-nitro-extension/download.bat b/extensions/inference-nitro-extension/download.bat index 22e1c85b3..2ef3165c1 100644 --- a/extensions/inference-nitro-extension/download.bat +++ b/extensions/inference-nitro-extension/download.bat @@ -1,3 +1,3 @@ @echo off set /p NITRO_VERSION=<./bin/version.txt -.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu +.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan && .\node_modules\.bin\download https://delta.jan.ai/vulkaninfoSDK.exe -o ./bin diff --git a/extensions/inference-nitro-extension/package.json b/extensions/inference-nitro-extension/package.json index 8ad516ad9..ba6b473eb 100644 --- a/extensions/inference-nitro-extension/package.json +++ b/extensions/inference-nitro-extension/package.json @@ -8,13 +8,13 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc --module commonjs && rollup -c rollup.config.ts", - "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro", + "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro && download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo", "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro", "downloadnitro:win32": "download.bat", "downloadnitro": "run-script-os", - "build:publish:darwin": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../electron/pre-install", - "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../electron/pre-install", - "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../electron/pre-install", + "build:publish:darwin": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && ../../.github/scripts/auto-sign.sh && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish:win32": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", + "build:publish:linux": "rimraf *.tgz --glob && npm run build && npm run downloadnitro && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install", "build:publish": "run-script-os" }, "exports": { @@ -35,12 +35,12 @@ "rollup-plugin-sourcemaps": "^0.6.3", "rollup-plugin-typescript2": "^0.36.0", "run-script-os": "^1.1.6", - "typescript": "^5.2.2" + "typescript": "^5.2.2", + "@types/os-utils": "^0.0.4", + "@rollup/plugin-replace": "^5.0.5" }, "dependencies": { "@janhq/core": "file:../../core", - "@rollup/plugin-replace": "^5.0.5", - "@types/os-utils": "^0.0.4", "fetch-retry": "^5.0.6", "path-browserify": "^1.0.1", "rxjs": "^7.8.1", diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts index 77a9fb208..ec8943f9c 100644 --- a/extensions/inference-nitro-extension/rollup.config.ts +++ b/extensions/inference-nitro-extension/rollup.config.ts @@ -1,34 +1,34 @@ -import resolve from "@rollup/plugin-node-resolve"; -import commonjs from "@rollup/plugin-commonjs"; -import sourceMaps from "rollup-plugin-sourcemaps"; -import typescript from "rollup-plugin-typescript2"; -import json from "@rollup/plugin-json"; -import replace from "@rollup/plugin-replace"; -const packageJson = require("./package.json"); +import resolve from '@rollup/plugin-node-resolve' +import commonjs from '@rollup/plugin-commonjs' +import sourceMaps from 'rollup-plugin-sourcemaps' +import typescript from 'rollup-plugin-typescript2' +import json from '@rollup/plugin-json' +import replace from '@rollup/plugin-replace' +const packageJson = require('./package.json') -const pkg = require("./package.json"); +const pkg = require('./package.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: "es", sourcemap: true }], + output: [{ file: pkg.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { - include: "src/**", + include: 'src/**', }, plugins: [ replace({ NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), INFERENCE_URL: JSON.stringify( process.env.INFERENCE_URL || - "http://127.0.0.1:3928/inferences/llamacpp/chat_completion" + 'http://127.0.0.1:3928/inferences/llamacpp/chat_completion' ), TROUBLESHOOTING_URL: JSON.stringify( - "https://jan.ai/guides/troubleshooting" + 'https://jan.ai/guides/troubleshooting' ), JAN_SERVER_INFERENCE_URL: JSON.stringify( - "http://localhost:1337/v1/chat/completions" + 'http://localhost:1337/v1/chat/completions' ), }), // Allow json resolution @@ -42,7 +42,7 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".js", ".ts", ".svelte"], + extensions: ['.js', '.ts', '.svelte'], }), // Resolve source maps to the original source @@ -52,12 +52,12 @@ export default [ { input: `src/node/index.ts`, output: [ - { file: "dist/node/index.cjs.js", format: "cjs", sourcemap: true }, + { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true }, ], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') - external: ["@janhq/core/node"], + external: ['@janhq/core/node'], watch: { - include: "src/node/**", + include: 'src/node/**', }, plugins: [ // Allow json resolution @@ -70,11 +70,11 @@ export default [ // which external modules to include in the bundle // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ - extensions: [".ts", ".js", ".json"], + extensions: ['.ts', '.js', '.json'], }), // Resolve source maps to the original source sourceMaps(), ], }, -]; +] diff --git a/extensions/inference-nitro-extension/src/@types/global.d.ts b/extensions/inference-nitro-extension/src/@types/global.d.ts index 7a4fb4805..3a3d2aa32 100644 --- a/extensions/inference-nitro-extension/src/@types/global.d.ts +++ b/extensions/inference-nitro-extension/src/@types/global.d.ts @@ -1,13 +1,13 @@ -declare const NODE: string; -declare const INFERENCE_URL: string; -declare const TROUBLESHOOTING_URL: string; -declare const JAN_SERVER_INFERENCE_URL: string; +declare const NODE: string +declare const INFERENCE_URL: string +declare const TROUBLESHOOTING_URL: string +declare const JAN_SERVER_INFERENCE_URL: string /** * The response from the initModel function. * @property error - An error message if the model fails to load. */ interface ModelOperationResponse { - error?: any; - modelFile?: string; + error?: any + modelFile?: string } diff --git a/extensions/inference-nitro-extension/src/helpers/sse.ts b/extensions/inference-nitro-extension/src/helpers/sse.ts index aab260828..06176c9b9 100644 --- a/extensions/inference-nitro-extension/src/helpers/sse.ts +++ b/extensions/inference-nitro-extension/src/helpers/sse.ts @@ -1,5 +1,5 @@ -import { Model } from "@janhq/core"; -import { Observable } from "rxjs"; +import { Model } from '@janhq/core' +import { Observable } from 'rxjs' /** * Sends a request to the inference server to generate a response based on the recent messages. * @param recentMessages - An array of recent messages to use as context for the inference. @@ -17,50 +17,50 @@ export function requestInference( model: model.id, stream: true, ...model.parameters, - }); + }) fetch(inferenceUrl, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - Accept: model.parameters.stream - ? "text/event-stream" - : "application/json", + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Accept': model.parameters.stream + ? 'text/event-stream' + : 'application/json', }, body: requestBody, signal: controller?.signal, }) .then(async (response) => { if (model.parameters.stream === false) { - const data = await response.json(); - subscriber.next(data.choices[0]?.message?.content ?? ""); + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') } else { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - if (content.startsWith("assistant: ")) { - content = content.replace("assistant: ", ""); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') } - subscriber.next(content); + subscriber.next(content) } } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index 9e96ad93f..979b4cfac 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -10,6 +10,7 @@ import { ChatCompletionRole, ContentType, MessageRequest, + MessageRequestType, MessageStatus, ThreadContent, ThreadMessage, @@ -25,9 +26,10 @@ import { ModelEvent, InferenceEvent, ModelSettingParams, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; + getJanDataFolderPath, +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -35,16 +37,16 @@ import { ulid } from "ulid"; * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceNitroExtension extends InferenceExtension { - private static readonly _homeDir = "file://engines"; - private static readonly _settingsDir = "file://settings"; - private static readonly _engineMetadataFileName = "nitro.json"; + private static readonly _homeDir = 'file://engines' + private static readonly _settingsDir = 'file://settings' + private static readonly _engineMetadataFileName = 'nitro.json' /** * Checking the health for Nitro's process each 5 secs. */ - private static readonly _intervalHealthCheck = 5 * 1000; + private static readonly _intervalHealthCheck = 5 * 1000 - private _currentModel: Model | undefined; + private _currentModel: Model | undefined private _engineSettings: ModelSettingParams = { ctx_len: 2048, @@ -52,23 +54,22 @@ export default class JanInferenceNitroExtension extends InferenceExtension { cpu_threads: 1, cont_batching: false, embedding: true, - }; + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * The interval id for the health check. Used to stop the health check. */ - private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = - undefined; + private getNitroProcesHealthIntervalId: NodeJS.Timeout | undefined = undefined /** * Tracking the current state of nitro process. */ - private nitroProcessInfo: any = undefined; + private nitroProcessInfo: any = undefined - private inferenceUrl = ""; + private inferenceUrl = '' /** * Subscribes to events emitted by the @janhq/core package. @@ -76,44 +77,40 @@ export default class JanInferenceNitroExtension extends InferenceExtension { async onLoad() { if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) { try { - await fs.mkdirSync(JanInferenceNitroExtension._homeDir); + await fs.mkdirSync(JanInferenceNitroExtension._homeDir) } catch (e) { - console.debug(e); + console.debug(e) } } // init inference url // @ts-ignore - const electronApi = window?.electronAPI; - this.inferenceUrl = INFERENCE_URL; + const electronApi = window?.electronAPI + this.inferenceUrl = INFERENCE_URL if (!electronApi) { - this.inferenceUrl = JAN_SERVER_INFERENCE_URL; + this.inferenceUrl = `${window.core?.api?.baseApiUrl}/v1/chat/completions` } - console.debug("Inference url: ", this.inferenceUrl); + console.debug('Inference url: ', this.inferenceUrl) if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir))) - await fs.mkdirSync(JanInferenceNitroExtension._settingsDir); - this.writeDefaultEngineSettings(); + await fs.mkdirSync(JanInferenceNitroExtension._settingsDir) + this.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data: MessageRequest) => this.onMessageRequest(data) - ); + ) - events.on(ModelEvent.OnModelInit, (model: Model) => - this.onModelInit(model) - ); + events.on(ModelEvent.OnModelInit, (model: Model) => this.onModelInit(model)) - events.on(ModelEvent.OnModelStop, (model: Model) => - this.onModelStop(model) - ); + events.on(ModelEvent.OnModelStop, (model: Model) => this.onModelStop(model)) events.on(InferenceEvent.OnInferenceStopped, () => this.onInferenceStopped() - ); + ) // Attempt to fetch nvidia info - await executeOnMain(NODE, "updateNvidiaInfo", {}); + await executeOnMain(NODE, 'updateNvidiaInfo', {}) } /** @@ -126,56 +123,62 @@ export default class JanInferenceNitroExtension extends InferenceExtension { const engineFile = await joinPath([ JanInferenceNitroExtension._homeDir, JanInferenceNitroExtension._engineMetadataFileName, - ]); + ]) if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, "utf-8"); + const engine = await fs.readFileSync(engineFile, 'utf-8') this._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engineFile, JSON.stringify(this._engineSettings, null, 2) - ); + ) } } catch (err) { - console.error(err); + console.error(err) } } private async onModelInit(model: Model) { - if (model.engine !== InferenceEngine.nitro) return; + if (model.engine !== InferenceEngine.nitro) return - const modelFullPath = await joinPath(["models", model.id]); - - this._currentModel = model; - const nitroInitResult = await executeOnMain(NODE, "runModel", { - modelFullPath, + const modelFolder = await joinPath([ + await getJanDataFolderPath(), + 'models', + model.id, + ]) + this._currentModel = model + const nitroInitResult = await executeOnMain(NODE, 'runModel', { + modelFolder, model, - }); + }) if (nitroInitResult?.error) { - events.emit(ModelEvent.OnModelFail, model); - return; + events.emit(ModelEvent.OnModelFail, { + ...model, + error: nitroInitResult.error, + }) + return } - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) this.getNitroProcesHealthIntervalId = setInterval( () => this.periodicallyGetNitroHealth(), JanInferenceNitroExtension._intervalHealthCheck - ); + ) } private async onModelStop(model: Model) { - if (model.engine !== "nitro") return; + if (model.engine !== 'nitro') return - await executeOnMain(NODE, "stopModel"); - events.emit(ModelEvent.OnModelStopped, {}); + await executeOnMain(NODE, 'stopModel') + events.emit(ModelEvent.OnModelStopped, {}) // stop the periocally health check if (this.getNitroProcesHealthIntervalId) { - clearInterval(this.getNitroProcesHealthIntervalId); - this.getNitroProcesHealthIntervalId = undefined; + clearInterval(this.getNitroProcesHealthIntervalId) + this.getNitroProcesHealthIntervalId = undefined } } @@ -183,19 +186,19 @@ export default class JanInferenceNitroExtension extends InferenceExtension { * Periodically check for nitro process's health. */ private async periodicallyGetNitroHealth(): Promise { - const health = await executeOnMain(NODE, "getCurrentNitroProcessInfo"); + const health = await executeOnMain(NODE, 'getCurrentNitroProcessInfo') - const isRunning = this.nitroProcessInfo?.isRunning ?? false; + const isRunning = this.nitroProcessInfo?.isRunning ?? false if (isRunning && health.isRunning === false) { - console.debug("Nitro process is stopped"); - events.emit(ModelEvent.OnModelStopped, {}); + console.debug('Nitro process is stopped') + events.emit(ModelEvent.OnModelStopped, {}) } - this.nitroProcessInfo = health; + this.nitroProcessInfo = health } private async onInferenceStopped() { - this.isCancelled = true; - this.controller?.abort(); + this.isCancelled = true + this.controller?.abort() } /** @@ -204,20 +207,20 @@ export default class JanInferenceNitroExtension extends InferenceExtension { * @returns {Promise} A promise that resolves with the inference response. */ async inference(data: MessageRequest): Promise { - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { thread_id: data.threadId, created: timestamp, updated: timestamp, status: MessageStatus.Ready, - id: "", + id: '', role: ChatCompletionRole.Assistant, - object: "thread.message", + object: 'thread.message', content: [], - }; + } return new Promise(async (resolve, reject) => { - if (!this._currentModel) return Promise.reject("No model loaded"); + if (!this._currentModel) return Promise.reject('No model loaded') requestInference( this.inferenceUrl, @@ -226,13 +229,13 @@ export default class JanInferenceNitroExtension extends InferenceExtension { ).subscribe({ next: (_content: any) => {}, complete: async () => { - resolve(message); + resolve(message) }, error: async (err: any) => { - reject(err); + reject(err) }, - }); - }); + }) + }) } /** @@ -243,31 +246,35 @@ export default class JanInferenceNitroExtension extends InferenceExtension { */ private async onMessageRequest(data: MessageRequest) { if (data.model?.engine !== InferenceEngine.nitro || !this._currentModel) { - return; + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, + type: data.type, assistant_id: data.assistantId, role: ChatCompletionRole.Assistant, content: [], status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; - events.emit(MessageEvent.OnMessageResponse, message); + object: 'thread.message', + } - this.isCancelled = false; - this.controller = new AbortController(); + if (data.type !== MessageRequestType.Summary) { + events.emit(MessageEvent.OnMessageResponse, message) + } + + this.isCancelled = false + this.controller = new AbortController() // @ts-ignore const model: Model = { ...(this._currentModel || {}), ...(data.model || {}), - }; + } requestInference( this.inferenceUrl, data.messages ?? [], @@ -281,26 +288,26 @@ export default class JanInferenceNitroExtension extends InferenceExtension { value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err: any) => { if (this.isCancelled || message.content.length) { - message.status = MessageStatus.Stopped; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return } - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); - log(`[APP]::Error: ${err.message}`); + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + log(`[APP]::Error: ${err.message}`) }, - }); + }) } } diff --git a/extensions/inference-nitro-extension/src/node/accelerator.ts b/extensions/inference-nitro-extension/src/node/accelerator.ts new file mode 100644 index 000000000..972f88681 --- /dev/null +++ b/extensions/inference-nitro-extension/src/node/accelerator.ts @@ -0,0 +1,240 @@ +import { writeFileSync, existsSync, readFileSync } from 'fs' +import { exec, spawn } from 'child_process' +import path from 'path' +import { getJanDataFolderPath, log } from '@janhq/core/node' + +/** + * Default GPU settings + * TODO: This needs to be refactored to support multiple accelerators + **/ +const DEFALT_SETTINGS = { + notify: true, + run_mode: 'cpu', + nvidia_driver: { + exist: false, + version: '', + }, + cuda: { + exist: false, + version: '', + }, + gpus: [], + gpu_highest_vram: '', + gpus_in_use: [], + is_initial: true, + // TODO: This needs to be set based on user toggle in settings + vulkan: { + enabled: true, + gpu_in_use: '1', + }, +} + +/** + * Path to the settings file + **/ +export const GPU_INFO_FILE = path.join( + getJanDataFolderPath(), + 'settings', + 'settings.json' +) + +/** + * Current nitro process + */ +let nitroProcessInfo: NitroProcessInfo | undefined = undefined + +/** + * Nitro process info + */ +export interface NitroProcessInfo { + isRunning: boolean +} + +/** + * This will retrive GPU informations and persist settings.json + * Will be called when the extension is loaded to turn on GPU acceleration if supported + */ +export async function updateNvidiaInfo() { + if (process.platform !== 'darwin') { + let data + try { + data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + } catch (error) { + data = DEFALT_SETTINGS + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + } + updateNvidiaDriverInfo() + updateGpuInfo() + } +} + +/** + * Retrieve current nitro process + */ +export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { + nitroProcessInfo = { + isRunning: subprocess != null, + } + return nitroProcessInfo +} + +/** + * Validate nvidia and cuda for linux and windows + */ +export async function updateNvidiaDriverInfo(): Promise { + exec( + 'nvidia-smi --query-gpu=driver_version --format=csv,noheader', + (error, stdout) => { + let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + + if (!error) { + const firstLine = stdout.split('\n')[0].trim() + data['nvidia_driver'].exist = true + data['nvidia_driver'].version = firstLine + } else { + data['nvidia_driver'].exist = false + } + + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + Promise.resolve() + } + ) +} + +/** + * Check if file exists in paths + */ +export function checkFileExistenceInPaths( + file: string, + paths: string[] +): boolean { + return paths.some((p) => existsSync(path.join(p, file))) +} + +/** + * Validate cuda for linux and windows + */ +export function updateCudaExistence( + data: Record = DEFALT_SETTINGS +): Record { + let filesCuda12: string[] + let filesCuda11: string[] + let paths: string[] + let cudaVersion: string = '' + + if (process.platform === 'win32') { + filesCuda12 = ['cublas64_12.dll', 'cudart64_12.dll', 'cublasLt64_12.dll'] + filesCuda11 = ['cublas64_11.dll', 'cudart64_11.dll', 'cublasLt64_11.dll'] + paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : [] + } else { + filesCuda12 = ['libcudart.so.12', 'libcublas.so.12', 'libcublasLt.so.12'] + filesCuda11 = ['libcudart.so.11.0', 'libcublas.so.11', 'libcublasLt.so.11'] + paths = process.env.LD_LIBRARY_PATH + ? process.env.LD_LIBRARY_PATH.split(path.delimiter) + : [] + paths.push('/usr/lib/x86_64-linux-gnu/') + } + + let cudaExists = filesCuda12.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ) + + if (!cudaExists) { + cudaExists = filesCuda11.every( + (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) + ) + if (cudaExists) { + cudaVersion = '11' + } + } else { + cudaVersion = '12' + } + + data['cuda'].exist = cudaExists + data['cuda'].version = cudaVersion + console.log(data['is_initial'], data['gpus_in_use']) + if (cudaExists && data['is_initial'] && data['gpus_in_use'].length > 0) { + data.run_mode = 'gpu' + } + data.is_initial = false + return data +} + +/** + * Get GPU information + */ +export async function updateGpuInfo(): Promise { + let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + + // Cuda + if (data['vulkan'] === true) { + // Vulkan + exec( + process.platform === 'win32' + ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary` + : `${__dirname}/../bin/vulkaninfo --summary`, + (error, stdout) => { + if (!error) { + const output = stdout.toString() + log(output) + const gpuRegex = /GPU(\d+):(?:[\s\S]*?)deviceName\s*=\s*(.*)/g + + let gpus = [] + let match + while ((match = gpuRegex.exec(output)) !== null) { + const id = match[1] + const name = match[2] + gpus.push({ id, vram: 0, name }) + } + data.gpus = gpus + + if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { + data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0'] + } + + data = updateCudaExistence(data) + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + } + Promise.resolve() + } + ) + } else { + exec( + 'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits', + (error, stdout) => { + if (!error) { + log(stdout) + // Get GPU info and gpu has higher memory first + let highestVram = 0 + let highestVramId = '0' + let gpus = stdout + .trim() + .split('\n') + .map((line) => { + let [id, vram, name] = line.split(', ') + vram = vram.replace(/\r/g, '') + if (parseFloat(vram) > highestVram) { + highestVram = parseFloat(vram) + highestVramId = id + } + return { id, vram, name } + }) + + data.gpus = gpus + data.gpu_highest_vram = highestVramId + } else { + data.gpus = [] + data.gpu_highest_vram = '' + } + + if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) { + data.gpus_in_use = [data['gpu_highest_vram']] + } + + data = updateCudaExistence(data) + writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2)) + Promise.resolve() + } + ) + } +} diff --git a/extensions/inference-nitro-extension/src/node/execute.ts b/extensions/inference-nitro-extension/src/node/execute.ts index 83b5226d4..08baba0d5 100644 --- a/extensions/inference-nitro-extension/src/node/execute.ts +++ b/extensions/inference-nitro-extension/src/node/execute.ts @@ -1,65 +1,79 @@ -import { readFileSync } from "fs"; -import * as path from "path"; -import { NVIDIA_INFO_FILE } from "./nvidia"; +import { readFileSync } from 'fs' +import * as path from 'path' +import { GPU_INFO_FILE } from './accelerator' export interface NitroExecutableOptions { - executablePath: string; - cudaVisibleDevices: string; + executablePath: string + cudaVisibleDevices: string + vkVisibleDevices: string } /** * Find which executable file to run based on the current platform. * @returns The name of the executable file to run. */ export const executableNitroFile = (): NitroExecutableOptions => { - let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default - let cudaVisibleDevices = ""; - let binaryName = "nitro"; + let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default + let cudaVisibleDevices = '' + let vkVisibleDevices = '' + let binaryName = 'nitro' /** * The binary folder is different for each platform. */ - if (process.platform === "win32") { + if (process.platform === 'win32') { /** - * For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0 + * For Windows: win-cpu, win-vulkan, win-cuda-11-7, win-cuda-12-0 */ - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { - binaryFolder = path.join(binaryFolder, "win-cpu"); + let gpuInfo = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + if (gpuInfo['run_mode'] === 'cpu') { + binaryFolder = path.join(binaryFolder, 'win-cpu') } else { - if (nvidiaInfo["cuda"].version === "11") { - binaryFolder = path.join(binaryFolder, "win-cuda-11-7"); + if (gpuInfo['cuda']?.version === '11') { + binaryFolder = path.join(binaryFolder, 'win-cuda-11-7') } else { - binaryFolder = path.join(binaryFolder, "win-cuda-12-0"); + binaryFolder = path.join(binaryFolder, 'win-cuda-12-0') } - cudaVisibleDevices = nvidiaInfo["gpus_in_use"].join(","); + cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',') } - binaryName = "nitro.exe"; - } else if (process.platform === "darwin") { + if (gpuInfo['vulkan'] === true) { + binaryFolder = path.join(__dirname, '..', 'bin') + binaryFolder = path.join(binaryFolder, 'win-vulkan') + vkVisibleDevices = gpuInfo['gpus_in_use'].toString() + } + binaryName = 'nitro.exe' + } else if (process.platform === 'darwin') { /** * For MacOS: mac-arm64 (Silicon), mac-x64 (InteL) */ - if (process.arch === "arm64") { - binaryFolder = path.join(binaryFolder, "mac-arm64"); + if (process.arch === 'arm64') { + binaryFolder = path.join(binaryFolder, 'mac-arm64') } else { - binaryFolder = path.join(binaryFolder, "mac-x64"); + binaryFolder = path.join(binaryFolder, 'mac-x64') } } else { /** - * For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0 + * For Linux: linux-cpu, linux-vulkan, linux-cuda-11-7, linux-cuda-12-0 */ - let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - if (nvidiaInfo["run_mode"] === "cpu") { - binaryFolder = path.join(binaryFolder, "linux-cpu"); + let gpuInfo = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8')) + if (gpuInfo['run_mode'] === 'cpu') { + binaryFolder = path.join(binaryFolder, 'linux-cpu') } else { - if (nvidiaInfo["cuda"].version === "11") { - binaryFolder = path.join(binaryFolder, "linux-cuda-11-7"); + if (gpuInfo['cuda']?.version === '11') { + binaryFolder = path.join(binaryFolder, 'linux-cuda-11-7') } else { - binaryFolder = path.join(binaryFolder, "linux-cuda-12-0"); + binaryFolder = path.join(binaryFolder, 'linux-cuda-12-0') } - cudaVisibleDevices = nvidiaInfo["gpus_in_use"].join(","); + cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',') + } + + if (gpuInfo['vulkan'] === true) { + binaryFolder = path.join(__dirname, '..', 'bin') + binaryFolder = path.join(binaryFolder, 'win-vulkan') + vkVisibleDevices = gpuInfo['gpus_in_use'].toString() } } return { executablePath: path.join(binaryFolder, binaryName), cudaVisibleDevices, - }; -}; + vkVisibleDevices, + } +} diff --git a/extensions/inference-nitro-extension/src/node/index.ts b/extensions/inference-nitro-extension/src/node/index.ts index 7ba90b556..9b2684a6c 100644 --- a/extensions/inference-nitro-extension/src/node/index.ts +++ b/extensions/inference-nitro-extension/src/node/index.ts @@ -1,55 +1,50 @@ -import fs from "fs"; -import path from "path"; -import { ChildProcessWithoutNullStreams, spawn } from "child_process"; -import tcpPortUsed from "tcp-port-used"; -import fetchRT from "fetch-retry"; -import { - log, - getJanDataFolderPath, - getSystemResourceInfo, -} from "@janhq/core/node"; -import { getNitroProcessInfo, updateNvidiaInfo } from "./nvidia"; +import fs from 'fs' +import path from 'path' +import { ChildProcessWithoutNullStreams, spawn } from 'child_process' +import tcpPortUsed from 'tcp-port-used' +import fetchRT from 'fetch-retry' +import { log, getSystemResourceInfo } from '@janhq/core/node' +import { getNitroProcessInfo, updateNvidiaInfo } from './accelerator' import { Model, InferenceEngine, ModelSettingParams, PromptTemplate, -} from "@janhq/core"; -import { executableNitroFile } from "./execute"; +} from '@janhq/core' +import { executableNitroFile } from './execute' // Polyfill fetch with retry -const fetchRetry = fetchRT(fetch); +const fetchRetry = fetchRT(fetch) /** * The response object for model init operation. */ interface ModelInitOptions { - modelFullPath: string; - model: Model; + modelFolder: string + model: Model } // The PORT to use for the Nitro subprocess -const PORT = 3928; +const PORT = 3928 // The HOST address to use for the Nitro subprocess -const LOCAL_HOST = "127.0.0.1"; +const LOCAL_HOST = '127.0.0.1' // The URL for the Nitro subprocess -const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}`; +const NITRO_HTTP_SERVER_URL = `http://${LOCAL_HOST}:${PORT}` // The URL for the Nitro subprocess to load a model -const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel`; +const NITRO_HTTP_LOAD_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/loadmodel` // The URL for the Nitro subprocess to validate a model -const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus`; +const NITRO_HTTP_VALIDATE_MODEL_URL = `${NITRO_HTTP_SERVER_URL}/inferences/llamacpp/modelstatus` // The URL for the Nitro subprocess to kill itself -const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy`; +const NITRO_HTTP_KILL_URL = `${NITRO_HTTP_SERVER_URL}/processmanager/destroy` // The supported model format // TODO: Should be an array to support more models -const SUPPORTED_MODEL_FORMAT = ".gguf"; +const SUPPORTED_MODEL_FORMAT = '.gguf' // The subprocess instance for Nitro -let subprocess: ChildProcessWithoutNullStreams | undefined = undefined; -// The current model file url -let currentModelFile: string = ""; +let subprocess: ChildProcessWithoutNullStreams | undefined = undefined + // The current model settings -let currentSettings: ModelSettingParams | undefined = undefined; +let currentSettings: ModelSettingParams | undefined = undefined /** * Stops a Nitro subprocess. @@ -57,7 +52,7 @@ let currentSettings: ModelSettingParams | undefined = undefined; * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. */ function stopModel(): Promise { - return killSubprocess(); + return killSubprocess() } /** @@ -67,62 +62,79 @@ function stopModel(): Promise { * TODO: Should pass absolute of the model file instead of just the name - So we can modurize the module.ts to npm package */ async function runModel( - wrapper: ModelInitOptions, + wrapper: ModelInitOptions ): Promise { if (wrapper.model.engine !== InferenceEngine.nitro) { // Not a nitro model - return Promise.resolve(); + return Promise.resolve() } - currentModelFile = wrapper.modelFullPath; - const janRoot = await getJanDataFolderPath(); - if (!currentModelFile.includes(janRoot)) { - currentModelFile = path.join(janRoot, currentModelFile); - } - const files: string[] = fs.readdirSync(currentModelFile); - - // Look for GGUF model file - const ggufBinFile = files.find( - (file) => - file === path.basename(currentModelFile) || - file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT), - ); - - if (!ggufBinFile) return Promise.reject("No GGUF model file found"); - - currentModelFile = path.join(currentModelFile, ggufBinFile); - if (wrapper.model.engine !== InferenceEngine.nitro) { - return Promise.reject("Not a nitro model"); + return Promise.reject('Not a nitro model') } else { - const nitroResourceProbe = await getSystemResourceInfo(); + const nitroResourceProbe = await getSystemResourceInfo() // Convert settings.prompt_template to system_prompt, user_prompt, ai_prompt if (wrapper.model.settings.prompt_template) { - const promptTemplate = wrapper.model.settings.prompt_template; - const prompt = promptTemplateConverter(promptTemplate); + const promptTemplate = wrapper.model.settings.prompt_template + const prompt = promptTemplateConverter(promptTemplate) if (prompt?.error) { - return Promise.reject(prompt.error); + return Promise.reject(prompt.error) } - wrapper.model.settings.system_prompt = prompt.system_prompt; - wrapper.model.settings.user_prompt = prompt.user_prompt; - wrapper.model.settings.ai_prompt = prompt.ai_prompt; + wrapper.model.settings.system_prompt = prompt.system_prompt + wrapper.model.settings.user_prompt = prompt.user_prompt + wrapper.model.settings.ai_prompt = prompt.ai_prompt } - const modelFolderPath = path.join(janRoot, "models", wrapper.model.id); - const modelPath = wrapper.model.settings.llama_model_path - ? path.join(modelFolderPath, wrapper.model.settings.llama_model_path) - : currentModelFile; + // modelFolder is the absolute path to the running model folder + // e.g. ~/jan/models/llama-2 + let modelFolder = wrapper.modelFolder + + let llama_model_path = wrapper.model.settings.llama_model_path + + // Absolute model path support + if ( + wrapper.model?.sources.length && + wrapper.model.sources.every((e) => fs.existsSync(e.url)) + ) { + llama_model_path = + wrapper.model.sources.length === 1 + ? wrapper.model.sources[0].url + : wrapper.model.sources.find((e) => + e.url.includes(llama_model_path ?? wrapper.model.id) + )?.url + } + + if (!llama_model_path || !path.isAbsolute(llama_model_path)) { + // Look for GGUF model file + const modelFiles: string[] = fs.readdirSync(modelFolder) + const ggufBinFile = modelFiles.find( + (file) => + // 1. Prioritize llama_model_path (predefined) + (llama_model_path && file === llama_model_path) || + // 2. Prioritize GGUF File (manual import) + file.toLowerCase().includes(SUPPORTED_MODEL_FORMAT) || + // 3. Fallback Model ID (for backward compatibility) + file === wrapper.model.id + ) + if (ggufBinFile) llama_model_path = path.join(modelFolder, ggufBinFile) + } + + // Look for absolute source path for single model + + if (!llama_model_path) return Promise.reject('No GGUF model file found') currentSettings = { ...wrapper.model.settings, - llama_model_path: modelPath, + llama_model_path, // This is critical and requires real CPU physical core count (or performance core) cpu_threads: Math.max(1, nitroResourceProbe.numCpuPhysicalCore), ...(wrapper.model.settings.mmproj && { - mmproj: path.join(modelFolderPath, wrapper.model.settings.mmproj), + mmproj: path.isAbsolute(wrapper.model.settings.mmproj) + ? wrapper.model.settings.mmproj + : path.join(modelFolder, wrapper.model.settings.mmproj), }), - }; - return runNitroAndLoadModel(); + } + return runNitroAndLoadModel() } } @@ -142,10 +154,10 @@ async function runNitroAndLoadModel() { * Should wait for awhile to make sure the port is free and subprocess is killed * The tested threshold is 500ms **/ - if (process.platform === "win32") { - return new Promise((resolve) => setTimeout(resolve, 500)); + if (process.platform === 'win32') { + return new Promise((resolve) => setTimeout(resolve, 500)) } else { - return Promise.resolve(); + return Promise.resolve() } }) .then(spawnNitroProcess) @@ -153,9 +165,9 @@ async function runNitroAndLoadModel() { .then(validateModelStatus) .catch((err) => { // TODO: Broadcast error so app could display proper error message - log(`[NITRO]::Error: ${err}`); - return { error: err }; - }); + log(`[NITRO]::Error: ${err}`) + return { error: err } + }) } /** @@ -165,43 +177,43 @@ async function runNitroAndLoadModel() { */ function promptTemplateConverter(promptTemplate: string): PromptTemplate { // Split the string using the markers - const systemMarker = "{system_message}"; - const promptMarker = "{prompt}"; + const systemMarker = '{system_message}' + const promptMarker = '{prompt}' if ( promptTemplate.includes(systemMarker) && promptTemplate.includes(promptMarker) ) { // Find the indices of the markers - const systemIndex = promptTemplate.indexOf(systemMarker); - const promptIndex = promptTemplate.indexOf(promptMarker); + const systemIndex = promptTemplate.indexOf(systemMarker) + const promptIndex = promptTemplate.indexOf(promptMarker) // Extract the parts of the string - const system_prompt = promptTemplate.substring(0, systemIndex); + const system_prompt = promptTemplate.substring(0, systemIndex) const user_prompt = promptTemplate.substring( systemIndex + systemMarker.length, - promptIndex, - ); + promptIndex + ) const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length, - ); + promptIndex + promptMarker.length + ) // Return the split parts - return { system_prompt, user_prompt, ai_prompt }; + return { system_prompt, user_prompt, ai_prompt } } else if (promptTemplate.includes(promptMarker)) { // Extract the parts of the string for the case where only promptMarker is present - const promptIndex = promptTemplate.indexOf(promptMarker); - const user_prompt = promptTemplate.substring(0, promptIndex); + const promptIndex = promptTemplate.indexOf(promptMarker) + const user_prompt = promptTemplate.substring(0, promptIndex) const ai_prompt = promptTemplate.substring( - promptIndex + promptMarker.length, - ); + promptIndex + promptMarker.length + ) // Return the split parts - return { user_prompt, ai_prompt }; + return { user_prompt, ai_prompt } } // Return an error if none of the conditions are met - return { error: "Cannot split prompt template" }; + return { error: 'Cannot split prompt template' } } /** @@ -210,13 +222,13 @@ function promptTemplateConverter(promptTemplate: string): PromptTemplate { */ function loadLLMModel(settings: any): Promise { if (!settings?.ngl) { - settings.ngl = 100; + settings.ngl = 100 } - log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`); + log(`[NITRO]::Debug: Loading model with params ${JSON.stringify(settings)}`) return fetchRetry(NITRO_HTTP_LOAD_MODEL_URL, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", + 'Content-Type': 'application/json', }, body: JSON.stringify(settings), retries: 3, @@ -225,15 +237,15 @@ function loadLLMModel(settings: any): Promise { .then((res) => { log( `[NITRO]::Debug: Load model success with response ${JSON.stringify( - res, - )}`, - ); - return Promise.resolve(res); + res + )}` + ) + return Promise.resolve(res) }) .catch((err) => { - log(`[NITRO]::Error: Load model failed with error ${err}`); - return Promise.reject(err); - }); + log(`[NITRO]::Error: Load model failed with error ${err}`) + return Promise.reject(err) + }) } /** @@ -246,9 +258,9 @@ async function validateModelStatus(): Promise { // Send a GET request to the validation URL. // Retry the request up to 3 times if it fails, with a delay of 500 milliseconds between retries. return fetchRetry(NITRO_HTTP_VALIDATE_MODEL_URL, { - method: "GET", + method: 'GET', headers: { - "Content-Type": "application/json", + 'Content-Type': 'application/json', }, retries: 5, retryDelay: 500, @@ -257,10 +269,10 @@ async function validateModelStatus(): Promise { `[NITRO]::Debug: Validate model state with response ${JSON.stringify( res.status )}` - ); + ) // If the response is OK, check model_loaded status. if (res.ok) { - const body = await res.json(); + const body = await res.json() // If the model is loaded, return an empty object. // Otherwise, return an object with an error message. if (body.model_loaded) { @@ -268,17 +280,17 @@ async function validateModelStatus(): Promise { `[NITRO]::Debug: Validate model state success with response ${JSON.stringify( body )}` - ); - return Promise.resolve(); + ) + return Promise.resolve() } } log( `[NITRO]::Debug: Validate model state failed with response ${JSON.stringify( res.statusText )}` - ); - return Promise.reject("Validate model status failed"); - }); + ) + return Promise.reject('Validate model status failed') + }) } /** @@ -286,21 +298,27 @@ async function validateModelStatus(): Promise { * @returns A Promise that resolves when the subprocess is terminated successfully, or rejects with an error message if the subprocess fails to terminate. */ async function killSubprocess(): Promise { - const controller = new AbortController(); - setTimeout(() => controller.abort(), 5000); - log(`[NITRO]::Debug: Request to kill Nitro`); + const controller = new AbortController() + setTimeout(() => controller.abort(), 5000) + log(`[NITRO]::Debug: Request to kill Nitro`) return fetch(NITRO_HTTP_KILL_URL, { - method: "DELETE", + method: 'DELETE', signal: controller.signal, }) .then(() => { - subprocess?.kill(); - subprocess = undefined; + subprocess?.kill() + subprocess = undefined }) - .catch(() => {}) + .catch(() => {}) // Do nothing with this attempt .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000)) - .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)); + .then(() => log(`[NITRO]::Debug: Nitro process is terminated`)) + .catch((err) => { + log( + `[NITRO]::Debug: Could not kill running process on port ${PORT}. Might be another process running on the same port? ${err}` + ) + throw 'PORT_NOT_AVAILABLE' + }) } /** @@ -308,49 +326,53 @@ async function killSubprocess(): Promise { * @returns A promise that resolves when the Nitro subprocess is started. */ function spawnNitroProcess(): Promise { - log(`[NITRO]::Debug: Spawning Nitro subprocess...`); + log(`[NITRO]::Debug: Spawning Nitro subprocess...`) return new Promise(async (resolve, reject) => { - let binaryFolder = path.join(__dirname, "..", "bin"); // Current directory by default - let executableOptions = executableNitroFile(); + let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default + let executableOptions = executableNitroFile() - const args: string[] = ["1", LOCAL_HOST, PORT.toString()]; + const args: string[] = ['1', LOCAL_HOST, PORT.toString()] // Execute the binary log( - `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}`, - ); + `[NITRO]::Debug: Spawn nitro at path: ${executableOptions.executablePath}, and args: ${args}` + ) subprocess = spawn( executableOptions.executablePath, - ["1", LOCAL_HOST, PORT.toString()], + ['1', LOCAL_HOST, PORT.toString()], { cwd: binaryFolder, env: { ...process.env, CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, + // Vulkan - Support 1 device at a time for now + ...(executableOptions.vkVisibleDevices?.length > 0 && { + GGML_VULKAN_DEVICE: executableOptions.vkVisibleDevices[0], + }), }, - }, - ); + } + ) // Handle subprocess output - subprocess.stdout.on("data", (data: any) => { - log(`[NITRO]::Debug: ${data}`); - }); + subprocess.stdout.on('data', (data: any) => { + log(`[NITRO]::Debug: ${data}`) + }) - subprocess.stderr.on("data", (data: any) => { - log(`[NITRO]::Error: ${data}`); - }); + subprocess.stderr.on('data', (data: any) => { + log(`[NITRO]::Error: ${data}`) + }) - subprocess.on("close", (code: any) => { - log(`[NITRO]::Debug: Nitro exited with code: ${code}`); - subprocess = undefined; - reject(`child process exited with code ${code}`); - }); + subprocess.on('close', (code: any) => { + log(`[NITRO]::Debug: Nitro exited with code: ${code}`) + subprocess = undefined + reject(`child process exited with code ${code}`) + }) tcpPortUsed.waitUntilUsed(PORT, 300, 30000).then(() => { - log(`[NITRO]::Debug: Nitro is ready`); - resolve(); - }); - }); + log(`[NITRO]::Debug: Nitro is ready`) + resolve() + }) + }) } /** @@ -360,7 +382,7 @@ function spawnNitroProcess(): Promise { */ function dispose() { // clean other registered resources here - killSubprocess(); + killSubprocess() } export default { @@ -370,4 +392,4 @@ export default { dispose, updateNvidiaInfo, getCurrentNitroProcessInfo: () => getNitroProcessInfo(subprocess), -}; +} diff --git a/extensions/inference-nitro-extension/src/node/nvidia.ts b/extensions/inference-nitro-extension/src/node/nvidia.ts deleted file mode 100644 index bed2856a1..000000000 --- a/extensions/inference-nitro-extension/src/node/nvidia.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { writeFileSync, existsSync, readFileSync } from "fs"; -import { exec } from "child_process"; -import path from "path"; -import { getJanDataFolderPath } from "@janhq/core/node"; - -/** - * Default GPU settings - **/ -const DEFALT_SETTINGS = { - notify: true, - run_mode: "cpu", - nvidia_driver: { - exist: false, - version: "", - }, - cuda: { - exist: false, - version: "", - }, - gpus: [], - gpu_highest_vram: "", - gpus_in_use: [], - is_initial: true, -}; - -/** - * Path to the settings file - **/ -export const NVIDIA_INFO_FILE = path.join( - getJanDataFolderPath(), - "settings", - "settings.json" -); - -/** - * Current nitro process - */ -let nitroProcessInfo: NitroProcessInfo | undefined = undefined; - -/** - * Nitro process info - */ -export interface NitroProcessInfo { - isRunning: boolean; -} - -/** - * This will retrive GPU informations and persist settings.json - * Will be called when the extension is loaded to turn on GPU acceleration if supported - */ -export async function updateNvidiaInfo() { - if (process.platform !== "darwin") { - let data; - try { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - } catch (error) { - data = DEFALT_SETTINGS; - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - } - updateNvidiaDriverInfo(); - updateGpuInfo(); - } -} - -/** - * Retrieve current nitro process - */ -export const getNitroProcessInfo = (subprocess: any): NitroProcessInfo => { - nitroProcessInfo = { - isRunning: subprocess != null, - }; - return nitroProcessInfo; -}; - -/** - * Validate nvidia and cuda for linux and windows - */ -export async function updateNvidiaDriverInfo(): Promise { - exec( - "nvidia-smi --query-gpu=driver_version --format=csv,noheader", - (error, stdout) => { - let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - - if (!error) { - const firstLine = stdout.split("\n")[0].trim(); - data["nvidia_driver"].exist = true; - data["nvidia_driver"].version = firstLine; - } else { - data["nvidia_driver"].exist = false; - } - - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); - } - ); -} - -/** - * Check if file exists in paths - */ -export function checkFileExistenceInPaths( - file: string, - paths: string[] -): boolean { - return paths.some((p) => existsSync(path.join(p, file))); -} - -/** - * Validate cuda for linux and windows - */ -export function updateCudaExistence(data: Record = DEFALT_SETTINGS): Record { - let filesCuda12: string[]; - let filesCuda11: string[]; - let paths: string[]; - let cudaVersion: string = ""; - - if (process.platform === "win32") { - filesCuda12 = ["cublas64_12.dll", "cudart64_12.dll", "cublasLt64_12.dll"]; - filesCuda11 = ["cublas64_11.dll", "cudart64_11.dll", "cublasLt64_11.dll"]; - paths = process.env.PATH ? process.env.PATH.split(path.delimiter) : []; - } else { - filesCuda12 = ["libcudart.so.12", "libcublas.so.12", "libcublasLt.so.12"]; - filesCuda11 = ["libcudart.so.11.0", "libcublas.so.11", "libcublasLt.so.11"]; - paths = process.env.LD_LIBRARY_PATH - ? process.env.LD_LIBRARY_PATH.split(path.delimiter) - : []; - paths.push("/usr/lib/x86_64-linux-gnu/"); - } - - let cudaExists = filesCuda12.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ); - - if (!cudaExists) { - cudaExists = filesCuda11.every( - (file) => existsSync(file) || checkFileExistenceInPaths(file, paths) - ); - if (cudaExists) { - cudaVersion = "11"; - } - } else { - cudaVersion = "12"; - } - - data["cuda"].exist = cudaExists; - data["cuda"].version = cudaVersion; - console.log(data["is_initial"], data["gpus_in_use"]); - if (cudaExists && data["is_initial"] && data["gpus_in_use"].length > 0) { - data.run_mode = "gpu"; - } - data.is_initial = false; - return data; -} - -/** - * Get GPU information - */ -export async function updateGpuInfo(): Promise { - exec( - "nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits", - (error, stdout) => { - let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); - - if (!error) { - // Get GPU info and gpu has higher memory first - let highestVram = 0; - let highestVramId = "0"; - let gpus = stdout - .trim() - .split("\n") - .map((line) => { - let [id, vram, name] = line.split(", "); - vram = vram.replace(/\r/g, ""); - if (parseFloat(vram) > highestVram) { - highestVram = parseFloat(vram); - highestVramId = id; - } - return { id, vram, name }; - }); - - data.gpus = gpus; - data.gpu_highest_vram = highestVramId; - } else { - data.gpus = []; - data.gpu_highest_vram = ""; - } - - if (!data["gpus_in_use"] || data["gpus_in_use"].length === 0) { - data.gpus_in_use = [data["gpu_highest_vram"]]; - } - - data = updateCudaExistence(data); - writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2)); - Promise.resolve(); - } - ); -} diff --git a/extensions/inference-openai-extension/package.json b/extensions/inference-openai-extension/package.json index 5fa0ce974..5efdbf874 100644 --- a/extensions/inference-openai-extension/package.json +++ b/extensions/inference-openai-extension/package.json @@ -8,7 +8,7 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { ".": "./dist/index.js", @@ -18,13 +18,13 @@ "cpx": "^1.5.0", "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "dependencies": { "@janhq/core": "file:../../core", "fetch-retry": "^5.0.6", "path-browserify": "^1.0.1", - "ts-loader": "^9.5.0", "ulid": "^2.3.0" }, "engines": { diff --git a/extensions/inference-openai-extension/src/@types/global.d.ts b/extensions/inference-openai-extension/src/@types/global.d.ts index 84f86c145..a49bb5a2f 100644 --- a/extensions/inference-openai-extension/src/@types/global.d.ts +++ b/extensions/inference-openai-extension/src/@types/global.d.ts @@ -1,26 +1,26 @@ -declare const MODULE: string; -declare const OPENAI_DOMAIN: string; +declare const MODULE: string +declare const OPENAI_DOMAIN: string declare interface EngineSettings { - full_url?: string; - api_key?: string; + full_url?: string + api_key?: string } enum OpenAIChatCompletionModelName { - "gpt-3.5-turbo-instruct" = "gpt-3.5-turbo-instruct", - "gpt-3.5-turbo-instruct-0914" = "gpt-3.5-turbo-instruct-0914", - "gpt-4-1106-preview" = "gpt-4-1106-preview", - "gpt-3.5-turbo-0613" = "gpt-3.5-turbo-0613", - "gpt-3.5-turbo-0301" = "gpt-3.5-turbo-0301", - "gpt-3.5-turbo" = "gpt-3.5-turbo", - "gpt-3.5-turbo-16k-0613" = "gpt-3.5-turbo-16k-0613", - "gpt-3.5-turbo-1106" = "gpt-3.5-turbo-1106", - "gpt-4-vision-preview" = "gpt-4-vision-preview", - "gpt-4" = "gpt-4", - "gpt-4-0314" = "gpt-4-0314", - "gpt-4-0613" = "gpt-4-0613", + 'gpt-3.5-turbo-instruct' = 'gpt-3.5-turbo-instruct', + 'gpt-3.5-turbo-instruct-0914' = 'gpt-3.5-turbo-instruct-0914', + 'gpt-4-1106-preview' = 'gpt-4-1106-preview', + 'gpt-3.5-turbo-0613' = 'gpt-3.5-turbo-0613', + 'gpt-3.5-turbo-0301' = 'gpt-3.5-turbo-0301', + 'gpt-3.5-turbo' = 'gpt-3.5-turbo', + 'gpt-3.5-turbo-16k-0613' = 'gpt-3.5-turbo-16k-0613', + 'gpt-3.5-turbo-1106' = 'gpt-3.5-turbo-1106', + 'gpt-4-vision-preview' = 'gpt-4-vision-preview', + 'gpt-4' = 'gpt-4', + 'gpt-4-0314' = 'gpt-4-0314', + 'gpt-4-0613' = 'gpt-4-0613', } -declare type OpenAIModel = Omit & { - id: OpenAIChatCompletionModelName; -}; +declare type OpenAIModel = Omit & { + id: OpenAIChatCompletionModelName +} diff --git a/extensions/inference-openai-extension/src/helpers/sse.ts b/extensions/inference-openai-extension/src/helpers/sse.ts index fb75816e7..11db38282 100644 --- a/extensions/inference-openai-extension/src/helpers/sse.ts +++ b/extensions/inference-openai-extension/src/helpers/sse.ts @@ -1,4 +1,4 @@ -import { Observable } from "rxjs"; +import { Observable } from 'rxjs' /** * Sends a request to the inference server to generate a response based on the recent messages. @@ -14,26 +14,26 @@ export function requestInference( controller?: AbortController ): Observable { return new Observable((subscriber) => { - let model_id: string = model.id; + let model_id: string = model.id if (engine.full_url.includes(OPENAI_DOMAIN)) { - model_id = engine.full_url.split("/")[5]; + model_id = engine.full_url.split('/')[5] } const requestBody = JSON.stringify({ messages: recentMessages, stream: true, model: model_id, ...model.parameters, - }); + }) fetch(`${engine.full_url}`, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - Accept: model.parameters.stream - ? "text/event-stream" - : "application/json", - "Access-Control-Allow-Origin": "*", - Authorization: `Bearer ${engine.api_key}`, - "api-key": `${engine.api_key}`, + 'Content-Type': 'application/json', + 'Accept': model.parameters.stream + ? 'text/event-stream' + : 'application/json', + 'Access-Control-Allow-Origin': '*', + 'Authorization': `Bearer ${engine.api_key}`, + 'api-key': `${engine.api_key}`, }, body: requestBody, signal: controller?.signal, @@ -41,41 +41,41 @@ export function requestInference( .then(async (response) => { if (!response.ok) { subscriber.next( - (await response.json()).error?.message ?? "Error occured" - ); - subscriber.complete(); - return; + (await response.json()).error?.message ?? 'Error occurred.' + ) + subscriber.complete() + return } if (model.parameters.stream === false) { - const data = await response.json(); - subscriber.next(data.choices[0]?.message?.content ?? ""); + const data = await response.json() + subscriber.next(data.choices[0]?.message?.content ?? '') } else { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - if (content.startsWith("assistant: ")) { - content = content.replace("assistant: ", ""); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + if (content.startsWith('assistant: ')) { + content = content.replace('assistant: ', '') } - subscriber.next(content); + subscriber.next(content) } } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts index fd1230bc7..481171742 100644 --- a/extensions/inference-openai-extension/src/index.ts +++ b/extensions/inference-openai-extension/src/index.ts @@ -18,14 +18,15 @@ import { InferenceEngine, BaseExtension, MessageEvent, + MessageRequestType, ModelEvent, InferenceEvent, AppConfigurationEventName, joinPath, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; -import { join } from "path"; +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { join } from 'path' /** * A class that implements the InferenceExtension interface from the @janhq/core package. @@ -33,18 +34,18 @@ import { join } from "path"; * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceOpenAIExtension extends BaseExtension { - private static readonly _engineDir = "file://engines"; - private static readonly _engineMetadataFileName = "openai.json"; + private static readonly _engineDir = 'file://engines' + private static readonly _engineMetadataFileName = 'openai.json' - private static _currentModel: OpenAIModel; + private static _currentModel: OpenAIModel private static _engineSettings: EngineSettings = { - full_url: "https://api.openai.com/v1/chat/completions", - api_key: "sk-", - }; + full_url: 'https://api.openai.com/v1/chat/completions', + api_key: 'sk-', + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * Subscribes to events emitted by the @janhq/core package. @@ -53,40 +54,40 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) { await fs .mkdirSync(JanInferenceOpenAIExtension._engineDir) - .catch((err) => console.debug(err)); + .catch((err) => console.debug(err)) } - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); + JanInferenceOpenAIExtension.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data) => - JanInferenceOpenAIExtension.handleMessageRequest(data, this), - ); + JanInferenceOpenAIExtension.handleMessageRequest(data, this) + ) events.on(ModelEvent.OnModelInit, (model: OpenAIModel) => { - JanInferenceOpenAIExtension.handleModelInit(model); - }); + JanInferenceOpenAIExtension.handleModelInit(model) + }) events.on(ModelEvent.OnModelStop, (model: OpenAIModel) => { - JanInferenceOpenAIExtension.handleModelStop(model); - }); + JanInferenceOpenAIExtension.handleModelStop(model) + }) events.on(InferenceEvent.OnInferenceStopped, () => { - JanInferenceOpenAIExtension.handleInferenceStopped(this); - }); + JanInferenceOpenAIExtension.handleInferenceStopped(this) + }) const settingsFilePath = await joinPath([ JanInferenceOpenAIExtension._engineDir, JanInferenceOpenAIExtension._engineMetadataFileName, - ]); + ]) events.on( AppConfigurationEventName.OnConfigurationUpdate, (settingsKey: string) => { // Update settings on changes if (settingsKey === settingsFilePath) - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); - }, - ); + JanInferenceOpenAIExtension.writeDefaultEngineSettings() + } + ) } /** @@ -98,45 +99,45 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { try { const engineFile = join( JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName, - ); + JanInferenceOpenAIExtension._engineMetadataFileName + ) if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, "utf-8"); + const engine = await fs.readFileSync(engineFile, 'utf-8') JanInferenceOpenAIExtension._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engineFile, - JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2), - ); + JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2) + ) } } catch (err) { - console.error(err); + console.error(err) } } private static async handleModelInit(model: OpenAIModel) { if (model.engine !== InferenceEngine.openai) { - return; + return } else { - JanInferenceOpenAIExtension._currentModel = model; - JanInferenceOpenAIExtension.writeDefaultEngineSettings(); + JanInferenceOpenAIExtension._currentModel = model + JanInferenceOpenAIExtension.writeDefaultEngineSettings() // Todo: Check model list with API key - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) } } private static async handleModelStop(model: OpenAIModel) { - if (model.engine !== "openai") { - return; + if (model.engine !== 'openai') { + return } - events.emit(ModelEvent.OnModelStopped, model); + events.emit(ModelEvent.OnModelStopped, model) } private static async handleInferenceStopped( - instance: JanInferenceOpenAIExtension, + instance: JanInferenceOpenAIExtension ) { - instance.isCancelled = true; - instance.controller?.abort(); + instance.isCancelled = true + instance.controller?.abort() } /** @@ -147,28 +148,32 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { */ private static async handleMessageRequest( data: MessageRequest, - instance: JanInferenceOpenAIExtension, + instance: JanInferenceOpenAIExtension ) { - if (data.model.engine !== "openai") { - return; + if (data.model.engine !== 'openai') { + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, + type: data.type, assistant_id: data.assistantId, role: ChatCompletionRole.Assistant, content: [], status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; - events.emit(MessageEvent.OnMessageResponse, message); + object: 'thread.message', + } - instance.isCancelled = false; - instance.controller = new AbortController(); + if (data.type !== MessageRequestType.Summary) { + events.emit(MessageEvent.OnMessageResponse, message) + } + + instance.isCancelled = false + instance.controller = new AbortController() requestInference( data?.messages ?? [], @@ -177,7 +182,7 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { ...JanInferenceOpenAIExtension._currentModel, parameters: data.model.parameters, }, - instance.controller, + instance.controller ).subscribe({ next: (content) => { const messageContent: ThreadContent = { @@ -186,33 +191,33 @@ export default class JanInferenceOpenAIExtension extends BaseExtension { value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err) => { if (instance.isCancelled || message.content.length > 0) { - message.status = MessageStatus.Stopped; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Stopped + events.emit(MessageEvent.OnMessageUpdate, message) + return } const messageContent: ThreadContent = { type: ContentType.Text, text: { - value: "Error occurred: " + err.message, + value: 'Error occurred: ' + err.message, annotations: [], }, - }; - message.content = [messageContent]; - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, - }); + }) } } diff --git a/extensions/inference-openai-extension/tsconfig.json b/extensions/inference-openai-extension/tsconfig.json index 7bfdd9009..2477d58ce 100644 --- a/extensions/inference-openai-extension/tsconfig.json +++ b/extensions/inference-openai-extension/tsconfig.json @@ -8,7 +8,7 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src", + "rootDir": "./src" }, - "include": ["./src"], + "include": ["./src"] } diff --git a/extensions/inference-openai-extension/webpack.config.js b/extensions/inference-openai-extension/webpack.config.js index 72b7d90c1..ee2e3b624 100644 --- a/extensions/inference-openai-extension/webpack.config.js +++ b/extensions/inference-openai-extension/webpack.config.js @@ -1,16 +1,16 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], @@ -18,22 +18,22 @@ module.exports = { plugins: [ new webpack.DefinePlugin({ MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - OPENAI_DOMAIN: JSON.stringify("openai.azure.com"), + OPENAI_DOMAIN: JSON.stringify('openai.azure.com'), }), ], output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { - path: require.resolve("path-browserify"), + path: require.resolve('path-browserify'), }, }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/inference-triton-trtllm-extension/package.json b/extensions/inference-triton-trtllm-extension/package.json index 1d27f9f18..455f8030e 100644 --- a/extensions/inference-triton-trtllm-extension/package.json +++ b/extensions/inference-triton-trtllm-extension/package.json @@ -8,7 +8,7 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { ".": "./dist/index.js", @@ -18,13 +18,13 @@ "cpx": "^1.5.0", "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "dependencies": { "@janhq/core": "file:../../core", "fetch-retry": "^5.0.6", "path-browserify": "^1.0.1", - "ts-loader": "^9.5.0", "ulid": "^2.3.0", "rxjs": "^7.8.1" }, diff --git a/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts b/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts index 6224b8e68..c834feba0 100644 --- a/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts +++ b/extensions/inference-triton-trtllm-extension/src/@types/global.d.ts @@ -1,5 +1,5 @@ -import { Model } from "@janhq/core"; +import { Model } from '@janhq/core' declare interface EngineSettings { - base_url?: string; + base_url?: string } diff --git a/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts b/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts index da20fa32d..9aff61265 100644 --- a/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts +++ b/extensions/inference-triton-trtllm-extension/src/helpers/sse.ts @@ -1,6 +1,6 @@ -import { Observable } from "rxjs"; -import { EngineSettings } from "../@types/global"; -import { Model } from "@janhq/core"; +import { Observable } from 'rxjs' +import { EngineSettings } from '../@types/global' +import { Model } from '@janhq/core' /** * Sends a request to the inference server to generate a response based on the recent messages. @@ -16,48 +16,48 @@ export function requestInference( controller?: AbortController ): Observable { return new Observable((subscriber) => { - const text_input = recentMessages.map((message) => message.text).join("\n"); + const text_input = recentMessages.map((message) => message.text).join('\n') const requestBody = JSON.stringify({ text_input: text_input, max_tokens: 4096, temperature: 0, - bad_words: "", - stop_words: "[DONE]", - stream: true - }); + bad_words: '', + stop_words: '[DONE]', + stream: true, + }) fetch(`${engine.base_url}/v2/models/ensemble/generate_stream`, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - Accept: "text/event-stream", - "Access-Control-Allow-Origin": "*", + 'Content-Type': 'application/json', + 'Accept': 'text/event-stream', + 'Access-Control-Allow-Origin': '*', }, body: requestBody, signal: controller?.signal, }) .then(async (response) => { - const stream = response.body; - const decoder = new TextDecoder("utf-8"); - const reader = stream?.getReader(); - let content = ""; + const stream = response.body + const decoder = new TextDecoder('utf-8') + const reader = stream?.getReader() + let content = '' while (true && reader) { - const { done, value } = await reader.read(); + const { done, value } = await reader.read() if (done) { - break; + break } - const text = decoder.decode(value); - const lines = text.trim().split("\n"); + const text = decoder.decode(value) + const lines = text.trim().split('\n') for (const line of lines) { - if (line.startsWith("data: ") && !line.includes("data: [DONE]")) { - const data = JSON.parse(line.replace("data: ", "")); - content += data.choices[0]?.delta?.content ?? ""; - subscriber.next(content); + if (line.startsWith('data: ') && !line.includes('data: [DONE]')) { + const data = JSON.parse(line.replace('data: ', '')) + content += data.choices[0]?.delta?.content ?? '' + subscriber.next(content) } } } - subscriber.complete(); + subscriber.complete() }) - .catch((err) => subscriber.error(err)); - }); + .catch((err) => subscriber.error(err)) + }) } diff --git a/extensions/inference-triton-trtllm-extension/src/index.ts b/extensions/inference-triton-trtllm-extension/src/index.ts index 11ddf7893..f009a81e0 100644 --- a/extensions/inference-triton-trtllm-extension/src/index.ts +++ b/extensions/inference-triton-trtllm-extension/src/index.ts @@ -20,51 +20,49 @@ import { BaseExtension, MessageEvent, ModelEvent, -} from "@janhq/core"; -import { requestInference } from "./helpers/sse"; -import { ulid } from "ulid"; -import { join } from "path"; -import { EngineSettings } from "./@types/global"; +} from '@janhq/core' +import { requestInference } from './helpers/sse' +import { ulid } from 'ulid' +import { join } from 'path' +import { EngineSettings } from './@types/global' /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ -export default class JanInferenceTritonTrtLLMExtension - extends BaseExtension -{ - private static readonly _homeDir = "file://engines"; - private static readonly _engineMetadataFileName = "triton_trtllm.json"; +export default class JanInferenceTritonTrtLLMExtension extends BaseExtension { + private static readonly _homeDir = 'file://engines' + private static readonly _engineMetadataFileName = 'triton_trtllm.json' - static _currentModel: Model; + static _currentModel: Model static _engineSettings: EngineSettings = { - base_url: "", - }; + base_url: '', + } - controller = new AbortController(); - isCancelled = false; + controller = new AbortController() + isCancelled = false /** * Subscribes to events emitted by the @janhq/core package. */ async onLoad() { if (!(await fs.existsSync(JanInferenceTritonTrtLLMExtension._homeDir))) - JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings(); + JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings() // Events subscription events.on(MessageEvent.OnMessageSent, (data) => JanInferenceTritonTrtLLMExtension.handleMessageRequest(data, this) - ); + ) events.on(ModelEvent.OnModelInit, (model: Model) => { - JanInferenceTritonTrtLLMExtension.handleModelInit(model); - }); + JanInferenceTritonTrtLLMExtension.handleModelInit(model) + }) events.on(ModelEvent.OnModelStop, (model: Model) => { - JanInferenceTritonTrtLLMExtension.handleModelStop(model); - }); + JanInferenceTritonTrtLLMExtension.handleModelStop(model) + }) } /** @@ -81,7 +79,7 @@ export default class JanInferenceTritonTrtLLMExtension modelId: string, settings?: ModelSettingParams ): Promise { - return; + return } static async writeDefaultEngineSettings() { @@ -89,11 +87,11 @@ export default class JanInferenceTritonTrtLLMExtension const engine_json = join( JanInferenceTritonTrtLLMExtension._homeDir, JanInferenceTritonTrtLLMExtension._engineMetadataFileName - ); + ) if (await fs.existsSync(engine_json)) { - const engine = await fs.readFileSync(engine_json, "utf-8"); + const engine = await fs.readFileSync(engine_json, 'utf-8') JanInferenceTritonTrtLLMExtension._engineSettings = - typeof engine === "object" ? engine : JSON.parse(engine); + typeof engine === 'object' ? engine : JSON.parse(engine) } else { await fs.writeFileSync( engine_json, @@ -102,10 +100,10 @@ export default class JanInferenceTritonTrtLLMExtension null, 2 ) - ); + ) } } catch (err) { - console.error(err); + console.error(err) } } /** @@ -119,26 +117,26 @@ export default class JanInferenceTritonTrtLLMExtension * @returns {Promise} A promise that resolves when the streaming is stopped. */ async stopInference(): Promise { - this.isCancelled = true; - this.controller?.abort(); + this.isCancelled = true + this.controller?.abort() } private static async handleModelInit(model: Model) { - if (model.engine !== "triton_trtllm") { - return; + if (model.engine !== 'triton_trtllm') { + return } else { - JanInferenceTritonTrtLLMExtension._currentModel = model; - JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings(); + JanInferenceTritonTrtLLMExtension._currentModel = model + JanInferenceTritonTrtLLMExtension.writeDefaultEngineSettings() // Todo: Check model list with API key - events.emit(ModelEvent.OnModelReady, model); + events.emit(ModelEvent.OnModelReady, model) } } private static async handleModelStop(model: Model) { - if (model.engine !== "triton_trtllm") { - return; + if (model.engine !== 'triton_trtllm') { + return } - events.emit(ModelEvent.OnModelStopped, model); + events.emit(ModelEvent.OnModelStopped, model) } /** @@ -151,11 +149,11 @@ export default class JanInferenceTritonTrtLLMExtension data: MessageRequest, instance: JanInferenceTritonTrtLLMExtension ) { - if (data.model.engine !== "triton_trtllm") { - return; + if (data.model.engine !== 'triton_trtllm') { + return } - const timestamp = Date.now(); + const timestamp = Date.now() const message: ThreadMessage = { id: ulid(), thread_id: data.threadId, @@ -165,12 +163,12 @@ export default class JanInferenceTritonTrtLLMExtension status: MessageStatus.Pending, created: timestamp, updated: timestamp, - object: "thread.message", - }; - events.emit(MessageEvent.OnMessageResponse, message); + object: 'thread.message', + } + events.emit(MessageEvent.OnMessageResponse, message) - instance.isCancelled = false; - instance.controller = new AbortController(); + instance.isCancelled = false + instance.controller = new AbortController() requestInference( data?.messages ?? [], @@ -188,33 +186,33 @@ export default class JanInferenceTritonTrtLLMExtension value: content.trim(), annotations: [], }, - }; - message.content = [messageContent]; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + events.emit(MessageEvent.OnMessageUpdate, message) }, complete: async () => { message.status = message.content.length ? MessageStatus.Ready - : MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); + : MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) }, error: async (err) => { if (instance.isCancelled || message.content.length) { - message.status = MessageStatus.Error; - events.emit(MessageEvent.OnMessageUpdate, message); - return; + message.status = MessageStatus.Error + events.emit(MessageEvent.OnMessageUpdate, message) + return } const messageContent: ThreadContent = { type: ContentType.Text, text: { - value: "Error occurred: " + err.message, + value: 'Error occurred: ' + err.message, annotations: [], }, - }; - message.content = [messageContent]; - message.status = MessageStatus.Ready; - events.emit(MessageEvent.OnMessageUpdate, message); + } + message.content = [messageContent] + message.status = MessageStatus.Ready + events.emit(MessageEvent.OnMessageUpdate, message) }, - }); + }) } } diff --git a/extensions/inference-triton-trtllm-extension/tsconfig.json b/extensions/inference-triton-trtllm-extension/tsconfig.json index 7bfdd9009..2477d58ce 100644 --- a/extensions/inference-triton-trtllm-extension/tsconfig.json +++ b/extensions/inference-triton-trtllm-extension/tsconfig.json @@ -8,7 +8,7 @@ "forceConsistentCasingInFileNames": true, "strict": false, "skipLibCheck": true, - "rootDir": "./src", + "rootDir": "./src" }, - "include": ["./src"], + "include": ["./src"] } diff --git a/extensions/inference-triton-trtllm-extension/webpack.config.js b/extensions/inference-triton-trtllm-extension/webpack.config.js index 57a0adb0a..e83370a1a 100644 --- a/extensions/inference-triton-trtllm-extension/webpack.config.js +++ b/extensions/inference-triton-trtllm-extension/webpack.config.js @@ -1,16 +1,16 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], @@ -21,18 +21,18 @@ module.exports = { }), ], output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], fallback: { - path: require.resolve("path-browserify"), + path: require.resolve('path-browserify'), }, }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/extensions/model-extension/.prettierrc b/extensions/model-extension/.prettierrc deleted file mode 100644 index 46f1abcb0..000000000 --- a/extensions/model-extension/.prettierrc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "semi": false, - "singleQuote": true, - "quoteProps": "consistent", - "trailingComma": "es5", - "endOfLine": "auto", - "plugins": ["prettier-plugin-tailwindcss"] -} diff --git a/extensions/model-extension/package.json b/extensions/model-extension/package.json index 86f177d14..e99122bcf 100644 --- a/extensions/model-extension/package.json +++ b/extensions/model-extension/package.json @@ -1,6 +1,6 @@ { "name": "@janhq/model-extension", - "version": "1.0.23", + "version": "1.0.25", "description": "Model Management Extension provides model exploration and seamless downloads", "main": "dist/index.js", "module": "dist/module.js", @@ -8,13 +8,14 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "devDependencies": { "cpx": "^1.5.0", "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "files": [ "dist/*", @@ -23,7 +24,6 @@ ], "dependencies": { "@janhq/core": "file:../../core", - "path-browserify": "^1.0.1", - "ts-loader": "^9.5.0" + "path-browserify": "^1.0.1" } } diff --git a/extensions/model-extension/src/@types/global.d.ts b/extensions/model-extension/src/@types/global.d.ts index e998455f2..7a9202a62 100644 --- a/extensions/model-extension/src/@types/global.d.ts +++ b/extensions/model-extension/src/@types/global.d.ts @@ -1,3 +1,15 @@ -declare const EXTENSION_NAME: string -declare const MODULE_PATH: string -declare const VERSION: stringå +export {} +declare global { + declare const EXTENSION_NAME: string + declare const MODULE_PATH: string + declare const VERSION: string + + interface Core { + api: APIFunctions + events: EventEmitter + } + interface Window { + core?: Core | undefined + electronAPI?: any | undefined + } +} diff --git a/extensions/model-extension/src/helpers/path.ts b/extensions/model-extension/src/helpers/path.ts new file mode 100644 index 000000000..cbb151aa6 --- /dev/null +++ b/extensions/model-extension/src/helpers/path.ts @@ -0,0 +1,11 @@ +/** + * try to retrieve the download file name from the source url + */ + +export function extractFileName(url: string, fileExtension: string): string { + const extractedFileName = url.split('/').pop() + const fileName = extractedFileName.toLowerCase().endsWith(fileExtension) + ? extractedFileName + : extractedFileName + fileExtension + return fileName +} diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts index b9fa7731e..926e65ee5 100644 --- a/extensions/model-extension/src/index.ts +++ b/extensions/model-extension/src/index.ts @@ -8,8 +8,15 @@ import { ModelExtension, Model, getJanDataFolderPath, + events, + DownloadEvent, + DownloadRoute, + ModelEvent, + DownloadState, } from '@janhq/core' +import { extractFileName } from './helpers/path' + /** * A extension for models */ @@ -29,6 +36,8 @@ export default class JanModelExtension extends ModelExtension { */ async onLoad() { this.copyModelsToHomeDir() + // Handle Desktop Events + this.handleDesktopEvents() } /** @@ -61,6 +70,8 @@ export default class JanModelExtension extends ModelExtension { // Finished migration localStorage.setItem(`${EXTENSION_NAME}-version`, VERSION) + + events.emit(ModelEvent.OnModelsUpdate, {}) } catch (err) { console.error(err) } @@ -83,31 +94,66 @@ export default class JanModelExtension extends ModelExtension { if (model.sources.length > 1) { // path to model binaries for (const source of model.sources) { - let path = this.extractFileName(source.url) + let path = extractFileName( + source.url, + JanModelExtension._supportedModelFormat + ) if (source.filename) { path = await joinPath([modelDirPath, source.filename]) } downloadFile(source.url, path, network) } + // TODO: handle multiple binaries for web later } else { - const fileName = this.extractFileName(model.sources[0]?.url) + const fileName = extractFileName( + model.sources[0]?.url, + JanModelExtension._supportedModelFormat + ) const path = await joinPath([modelDirPath, fileName]) downloadFile(model.sources[0]?.url, path, network) + + if (window && window.core?.api && window.core.api.baseApiUrl) { + this.startPollingDownloadProgress(model.id) + } } } /** - * try to retrieve the download file name from the source url + * Specifically for Jan server. */ - private extractFileName(url: string): string { - const extractedFileName = url.split('/').pop() - const fileName = extractedFileName - .toLowerCase() - .endsWith(JanModelExtension._supportedModelFormat) - ? extractedFileName - : extractedFileName + JanModelExtension._supportedModelFormat - return fileName + private async startPollingDownloadProgress(modelId: string): Promise { + // wait for some seconds before polling + await new Promise((resolve) => setTimeout(resolve, 3000)) + + return new Promise((resolve) => { + const interval = setInterval(async () => { + fetch( + `${window.core.api.baseApiUrl}/v1/download/${DownloadRoute.getDownloadProgress}/${modelId}`, + { + method: 'GET', + headers: { contentType: 'application/json' }, + } + ).then(async (res) => { + const state: DownloadState = await res.json() + if (state.downloadState === 'end') { + events.emit(DownloadEvent.onFileDownloadSuccess, state) + clearInterval(interval) + resolve() + return + } + + if (state.downloadState === 'error') { + events.emit(DownloadEvent.onFileDownloadError, state) + clearInterval(interval) + resolve() + return + } + + events.emit(DownloadEvent.onFileDownloadUpdate, state) + }) + }, 1000) + }) } /** @@ -174,15 +220,20 @@ export default class JanModelExtension extends ModelExtension { async getDownloadedModels(): Promise { return await this.getModelsMetadata( async (modelDir: string, model: Model) => { - if (model.engine !== JanModelExtension._offlineInferenceEngine) { + if (model.engine !== JanModelExtension._offlineInferenceEngine) return true - } + + // model binaries (sources) are absolute path & exist + const existFiles = await Promise.all( + model.sources.map((source) => fs.existsSync(source.url)) + ) + if (existFiles.every((exist) => exist)) return true + return await fs .readdirSync(await joinPath([JanModelExtension._homeDir, modelDir])) .then((files: string[]) => { - // or model binary exists in the directory - // model binary name can match model ID or be a .gguf file and not be an incompleted model file - // TODO: Check diff between urls, filenames + // Model binary exists in the directory + // Model binary name can match model ID or be a .gguf file and not be an incompleted model file return ( files.includes(modelDir) || files.filter( @@ -228,8 +279,19 @@ export default class JanModelExtension extends ModelExtension { if (await fs.existsSync(jsonPath)) { // if we have the model.json file, read it let model = await this.readModelMetadata(jsonPath) + model = typeof model === 'object' ? model : JSON.parse(model) + // This to ensure backward compatibility with `model.json` with `source_url` + if (model['source_url'] != null) { + model['sources'] = [ + { + filename: model.id, + url: model['source_url'], + }, + ] + } + if (selector && !(await selector?.(dirName, model))) { return } @@ -243,31 +305,18 @@ export default class JanModelExtension extends ModelExtension { }) const results = await Promise.allSettled(readJsonPromises) const modelData = results.map((result) => { - if (result.status === 'fulfilled') { + if (result.status === 'fulfilled' && result.value) { try { - // This to ensure backward compatibility with `model.json` with `source_url` - const tmpModel = + const model = typeof result.value === 'object' ? result.value : JSON.parse(result.value) - if (tmpModel['source_url'] != null) { - tmpModel['source'] = [ - { - filename: tmpModel.id, - url: tmpModel['source_url'], - }, - ] - } - - return tmpModel as Model + return model as Model } catch { console.debug(`Unable to parse model metadata: ${result.value}`) - return undefined } - } else { - console.error(result.reason) - return undefined } + return undefined }) return modelData.filter((e) => !!e) @@ -318,7 +367,7 @@ export default class JanModelExtension extends ModelExtension { return } - const defaultModel = await this.getDefaultModel() as Model + const defaultModel = (await this.getDefaultModel()) as Model if (!defaultModel) { console.error('Unable to find default model') return @@ -382,4 +431,28 @@ export default class JanModelExtension extends ModelExtension { async getConfiguredModels(): Promise { return this.getModelsMetadata() } + + handleDesktopEvents() { + if (window && window.electronAPI) { + window.electronAPI.onFileDownloadUpdate( + async (_event: string, state: DownloadState | undefined) => { + if (!state) return + state.downloadState = 'downloading' + events.emit(DownloadEvent.onFileDownloadUpdate, state) + } + ) + window.electronAPI.onFileDownloadError( + async (_event: string, state: DownloadState) => { + state.downloadState = 'error' + events.emit(DownloadEvent.onFileDownloadError, state) + } + ) + window.electronAPI.onFileDownloadSuccess( + async (_event: string, state: DownloadState) => { + state.downloadState = 'end' + events.emit(DownloadEvent.onFileDownloadSuccess, state) + } + ) + } + } } diff --git a/extensions/monitoring-extension/package.json b/extensions/monitoring-extension/package.json index 20d3c485f..582f7cd7b 100644 --- a/extensions/monitoring-extension/package.json +++ b/extensions/monitoring-extension/package.json @@ -8,17 +8,17 @@ "license": "AGPL-3.0", "scripts": { "build": "tsc -b . && webpack --config webpack.config.js", - "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../electron/pre-install" + "build:publish": "rimraf *.tgz --glob && npm run build && npm pack && cpx *.tgz ../../pre-install" }, "devDependencies": { "rimraf": "^3.0.2", "webpack": "^5.88.2", - "webpack-cli": "^5.1.4" + "webpack-cli": "^5.1.4", + "ts-loader": "^9.5.0" }, "dependencies": { "@janhq/core": "file:../../core", - "node-os-utils": "^1.3.7", - "ts-loader": "^9.5.0" + "node-os-utils": "^1.3.7" }, "files": [ "dist/*", diff --git a/extensions/monitoring-extension/src/@types/global.d.ts b/extensions/monitoring-extension/src/@types/global.d.ts index 3b45ccc5a..8106353cf 100644 --- a/extensions/monitoring-extension/src/@types/global.d.ts +++ b/extensions/monitoring-extension/src/@types/global.d.ts @@ -1 +1 @@ -declare const MODULE: string; +declare const MODULE: string diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts index 9297a770f..ce9b2fc14 100644 --- a/extensions/monitoring-extension/src/index.ts +++ b/extensions/monitoring-extension/src/index.ts @@ -1,4 +1,4 @@ -import { MonitoringExtension, executeOnMain } from "@janhq/core"; +import { MonitoringExtension, executeOnMain } from '@janhq/core' /** * JanMonitoringExtension is a extension that provides system monitoring functionality. @@ -20,7 +20,7 @@ export default class JanMonitoringExtension extends MonitoringExtension { * @returns A Promise that resolves to an object containing information about the system resources. */ getResourcesInfo(): Promise { - return executeOnMain(MODULE, "getResourcesInfo"); + return executeOnMain(MODULE, 'getResourcesInfo') } /** @@ -28,6 +28,6 @@ export default class JanMonitoringExtension extends MonitoringExtension { * @returns A Promise that resolves to an object containing information about the current system load. */ getCurrentLoad(): Promise { - return executeOnMain(MODULE, "getCurrentLoad"); + return executeOnMain(MODULE, 'getCurrentLoad') } } diff --git a/extensions/monitoring-extension/src/module.ts b/extensions/monitoring-extension/src/module.ts index 2c1b14343..27781a5d6 100644 --- a/extensions/monitoring-extension/src/module.ts +++ b/extensions/monitoring-extension/src/module.ts @@ -1,73 +1,92 @@ -const nodeOsUtils = require("node-os-utils"); -const getJanDataFolderPath = require("@janhq/core/node").getJanDataFolderPath; -const path = require("path"); -const { readFileSync } = require("fs"); -const exec = require("child_process").exec; +const nodeOsUtils = require('node-os-utils') +const getJanDataFolderPath = require('@janhq/core/node').getJanDataFolderPath +const path = require('path') +const { readFileSync } = require('fs') +const exec = require('child_process').exec const NVIDIA_INFO_FILE = path.join( getJanDataFolderPath(), - "settings", - "settings.json" -); + 'settings', + 'settings.json' +) const getResourcesInfo = () => new Promise((resolve) => { nodeOsUtils.mem.used().then((ramUsedInfo) => { - const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024; - const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024; + const totalMemory = ramUsedInfo.totalMemMb * 1024 * 1024 + const usedMemory = ramUsedInfo.usedMemMb * 1024 * 1024 const response = { mem: { totalMemory, usedMemory, }, - }; - resolve(response); - }); - }); + } + resolve(response) + }) + }) const getCurrentLoad = () => new Promise((resolve, reject) => { nodeOsUtils.cpu.usage().then((cpuPercentage) => { let data = { - run_mode: "cpu", + run_mode: 'cpu', gpus_in_use: [], - }; - if (process.platform !== "darwin") { - data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, "utf-8")); } - if (data.run_mode === "gpu" && data.gpus_in_use.length > 0) { - const gpuIds = data["gpus_in_use"].join(","); - if (gpuIds !== "") { + if (process.platform !== 'darwin') { + data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8')) + } + if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) { + const gpuIds = data['gpus_in_use'].join(',') + if (gpuIds !== '' && data['vulkan'] !== true) { exec( `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`, - (error, stdout, stderr) => { + (error, stdout, _) => { if (error) { - console.error(`exec error: ${error}`); - reject(error); - return; + console.error(`exec error: ${error}`) + reject(error) + return } - const gpuInfo = stdout.trim().split("\n").map((line) => { - const [id, name, temperature, utilization, memoryTotal, memoryFree, memoryUtilization] = line.split(", ").map(item => item.replace(/\r/g, "")); - return { id, name, temperature, utilization, memoryTotal, memoryFree, memoryUtilization }; - }); + const gpuInfo = stdout + .trim() + .split('\n') + .map((line) => { + const [ + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + ] = line.split(', ').map((item) => item.replace(/\r/g, '')) + return { + id, + name, + temperature, + utilization, + memoryTotal, + memoryFree, + memoryUtilization, + } + }) resolve({ cpu: { usage: cpuPercentage }, - gpu: gpuInfo - }); + gpu: gpuInfo, + }) } - ); + ) } else { // Handle the case where gpuIds is empty - resolve({ cpu: { usage: cpuPercentage }, gpu: [] }); + resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) } } else { // Handle the case where run_mode is not 'gpu' or no GPUs are in use - resolve({ cpu: { usage: cpuPercentage }, gpu: [] }); + resolve({ cpu: { usage: cpuPercentage }, gpu: [] }) } - }); - }); + }) + }) module.exports = { getResourcesInfo, getCurrentLoad, -}; \ No newline at end of file +} diff --git a/extensions/monitoring-extension/webpack.config.js b/extensions/monitoring-extension/webpack.config.js index f54059222..c8c3a34f7 100644 --- a/extensions/monitoring-extension/webpack.config.js +++ b/extensions/monitoring-extension/webpack.config.js @@ -1,24 +1,24 @@ -const path = require("path"); -const webpack = require("webpack"); -const packageJson = require("./package.json"); +const path = require('path') +const webpack = require('webpack') +const packageJson = require('./package.json') module.exports = { experiments: { outputModule: true }, - entry: "./src/index.ts", // Adjust the entry point to match your project's main file - mode: "production", + entry: './src/index.ts', // Adjust the entry point to match your project's main file + mode: 'production', module: { rules: [ { test: /\.tsx?$/, - use: "ts-loader", + use: 'ts-loader', exclude: /node_modules/, }, ], }, output: { - filename: "index.js", // Adjust the output file name as needed - path: path.resolve(__dirname, "dist"), - library: { type: "module" }, // Specify ESM output format + filename: 'index.js', // Adjust the output file name as needed + path: path.resolve(__dirname, 'dist'), + library: { type: 'module' }, // Specify ESM output format }, plugins: [ new webpack.DefinePlugin({ @@ -26,10 +26,10 @@ module.exports = { }), ], resolve: { - extensions: [".ts", ".js"], + extensions: ['.ts', '.js'], }, optimization: { minimize: false, }, // Add loaders and other configuration as needed for your project -}; +} diff --git a/models/dolphin-phi-2/model.json b/models/dolphin-phi-2/model.json new file mode 100644 index 000000000..c25ff8f69 --- /dev/null +++ b/models/dolphin-phi-2/model.json @@ -0,0 +1,32 @@ +{ + "sources": [ + { + "url": "https://huggingface.co/TheBloke/dolphin-2_6-phi-2-GGUF/resolve/main/dolphin-2_6-phi-2.Q8_0.gguf", + "filename": "dolphin-2_6-phi-2.Q8_0.gguf" + } + ], + "id": "dolphin-phi-2", + "object": "model", + "name": "Dolphin Phi-2 2.7B Q8", + "version": "1.0", + "description": "Dolphin Phi-2 is a 2.7B model, fine-tuned for chat, excelling in common sense and logical reasoning benchmarks.", + "format": "gguf", + "settings": { + "ctx_len": 4096, + "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", + "llama_model_path": "dolphin-2_6-phi-2.Q8_0.gguf" + }, + "parameters": { + "max_tokens": 4096, + "stop": ["<|im_end|>"] + }, + "metadata": { + "author": "Cognitive Computations, Microsoft", + "tags": [ + "3B", + "Finetuned" + ], + "size": 2960000000 + }, + "engine": "nitro" + } diff --git a/models/llamacorn-1.1b/model.json b/models/llamacorn-1.1b/model.json new file mode 100644 index 000000000..056fb9050 --- /dev/null +++ b/models/llamacorn-1.1b/model.json @@ -0,0 +1,37 @@ +{ + "sources": [ + { + "url":"https://huggingface.co/janhq/llamacorn-1.1b-chat-GGUF/resolve/main/llamacorn-1.1b-chat.Q8_0.gguf", + "filename": "llamacorn-1.1b-chat.Q8_0.gguf" + } + ], + "id": "llamacorn-1.1b", + "object": "model", + "name": "LlamaCorn 1.1B Q8", + "version": "1.0", + "description": "LlamaCorn is designed to improve chat functionality from TinyLlama.", + "format": "gguf", + "settings": { + "ctx_len": 2048, + "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant", + "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 2048, + "stop": [], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "Jan", + "tags": [ + "Tiny", + "Finetuned" + ], + "size": 1170000000 + }, + "engine": "nitro" + } \ No newline at end of file diff --git a/models/mistral-ins-7b-q4/cover.png b/models/mistral-ins-7b-q4/cover.png index 000445ecb..73b82e599 100644 Binary files a/models/mistral-ins-7b-q4/cover.png and b/models/mistral-ins-7b-q4/cover.png differ diff --git a/models/mistral-ins-7b-q4/model.json b/models/mistral-ins-7b-q4/model.json index bfdaffa90..75e0cbf9f 100644 --- a/models/mistral-ins-7b-q4/model.json +++ b/models/mistral-ins-7b-q4/model.json @@ -29,7 +29,7 @@ "author": "MistralAI, The Bloke", "tags": ["Featured", "7B", "Foundational Model"], "size": 4370000000, - "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/mistral-ins-7b-q4/cover.png" + "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/mistral-ins-7b-q4/cover.png" }, "engine": "nitro" } diff --git a/models/openchat-3.5-7b/model.json b/models/openchat-3.5-7b/model.json index 294f7d269..18db33f8e 100644 --- a/models/openchat-3.5-7b/model.json +++ b/models/openchat-3.5-7b/model.json @@ -1,8 +1,8 @@ { "sources": [ { - "filename": "openchat-3.5-1210.Q4_K_M.gguf", - "url": "https://huggingface.co/TheBloke/openchat-3.5-1210-GGUF/resolve/main/openchat-3.5-1210.Q4_K_M.gguf" + "filename": "openchat-3.5-0106.Q4_K_M.gguf", + "url": "https://huggingface.co/TheBloke/openchat-3.5-0106-GGUF/resolve/main/openchat-3.5-0106.Q4_K_M.gguf" } ], "id": "openchat-3.5-7b", @@ -14,7 +14,7 @@ "settings": { "ctx_len": 4096, "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:", - "llama_model_path": "openchat-3.5-1210.Q4_K_M.gguf" + "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf" }, "parameters": { "temperature": 0.7, diff --git a/models/openhermes-neural-7b/cover.png b/models/openhermes-neural-7b/cover.png index 5b9da0aef..8976d8449 100644 Binary files a/models/openhermes-neural-7b/cover.png and b/models/openhermes-neural-7b/cover.png differ diff --git a/models/openhermes-neural-7b/model.json b/models/openhermes-neural-7b/model.json index 87e1df143..a13a0f2b8 100644 --- a/models/openhermes-neural-7b/model.json +++ b/models/openhermes-neural-7b/model.json @@ -28,7 +28,7 @@ "author": "Intel, Jan", "tags": ["7B", "Merged", "Featured"], "size": 4370000000, - "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/openhermes-neural-7b/cover.png" + "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/openhermes-neural-7b/cover.png" }, "engine": "nitro" } diff --git a/models/stable-zephyr-3b/model.json b/models/stable-zephyr-3b/model.json new file mode 100644 index 000000000..724299ea5 --- /dev/null +++ b/models/stable-zephyr-3b/model.json @@ -0,0 +1,34 @@ +{ + "sources": [ + { + "url": "https://huggingface.co/TheBloke/stablelm-zephyr-3b-GGUF/resolve/main/stablelm-zephyr-3b.Q8_0.gguf", + "filename": "stablelm-zephyr-3b.Q8_0.gguf" + } + ], + "id": "stable-zephyr-3b", + "object": "model", + "name": "Stable Zephyr 3B Q8", + "version": "1.0", + "description": "StableLM Zephyr 3B is trained for safe and reliable chatting.", + "format": "gguf", + "settings": { + "ctx_len": 4096, + "prompt_template": "<|user|>\n{prompt}<|endoftext|>\n<|assistant|>", + "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf" + }, + "parameters": { + "temperature": 0.7, + "top_p": 0.95, + "stream": true, + "max_tokens": 4096, + "stop": ["<|endoftext|>"], + "frequency_penalty": 0, + "presence_penalty": 0 + }, + "metadata": { + "author": "StabilityAI", + "tags": ["3B", "Finetuned"], + "size": 2970000000 + }, + "engine": "nitro" + } \ No newline at end of file diff --git a/models/trinity-v1.2-7b/cover.png b/models/trinity-v1.2-7b/cover.png index a548e3c17..fbef0bb56 100644 Binary files a/models/trinity-v1.2-7b/cover.png and b/models/trinity-v1.2-7b/cover.png differ diff --git a/models/trinity-v1.2-7b/model.json b/models/trinity-v1.2-7b/model.json index 2dda120e6..947629642 100644 --- a/models/trinity-v1.2-7b/model.json +++ b/models/trinity-v1.2-7b/model.json @@ -28,7 +28,7 @@ "author": "Jan", "tags": ["7B", "Merged", "Featured"], "size": 4370000000, - "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/trinity-v1.2-7b/cover.png" + "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/trinity-v1.2-7b/cover.png" }, "engine": "nitro" } diff --git a/models/yarn-mistral-7b/model.json b/models/yarn-mistral-7b/model.json deleted file mode 100644 index ee6de1319..000000000 --- a/models/yarn-mistral-7b/model.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "sources": [ - { - "url": "https://huggingface.co/TheBloke/Yarn-Mistral-7B-128k-GGUF/resolve/main/yarn-mistral-7b-128k.Q4_K_M.gguf" - } - ], - "id": "yarn-mistral-7b", - "object": "model", - "name": "Yarn Mistral 7B Q4", - "version": "1.0", - "description": "Yarn Mistral 7B is a language model for long context and supports a 128k token context window.", - "format": "gguf", - "settings": { - "ctx_len": 4096, - "prompt_template": "{prompt}" - }, - "parameters": { - "temperature": 0.7, - "top_p": 0.95, - "stream": true, - "max_tokens": 4096, - "frequency_penalty": 0, - "presence_penalty": 0 - }, - "metadata": { - "author": "NousResearch, The Bloke", - "tags": ["7B", "Finetuned"], - "size": 4370000000 - }, - "engine": "nitro" -} diff --git a/package.json b/package.json index 4b8bc4af0..957934fda 100644 --- a/package.json +++ b/package.json @@ -21,22 +21,23 @@ "lint": "yarn workspace jan lint && yarn workspace jan-web lint", "test:unit": "yarn workspace @janhq/core test", "test": "yarn workspace jan test:e2e", - "copy:assets": "cpx \"models/**\" \"electron/models/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"", + "copy:assets": "cpx \"models/**\" \"electron/models/\" && cpx \"pre-install/*.tgz\" \"electron/pre-install/\" && cpx \"docs/openapi/**\" \"electron/docs/openapi\"", "dev:electron": "yarn copy:assets && yarn workspace jan dev", "dev:web": "yarn workspace jan-web dev", - "dev:server": "yarn workspace @janhq/server dev", + "dev:server": "yarn copy:assets && yarn workspace @janhq/server dev", "dev": "concurrently --kill-others \"yarn dev:web\" \"wait-on http://localhost:3000 && yarn dev:electron\"", "test-local": "yarn lint && yarn build:test && yarn test", "dev:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit dev", "build:uikit": "yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build", - "build:server": "cd server && yarn install && yarn run build", + "build:server": "yarn copy:assets && cd server && yarn install && yarn run build", "build:core": "cd core && yarn install && yarn run build", "build:web": "yarn workspace jan-web build && cpx \"web/out/**\" \"electron/renderer/\"", "build:electron": "yarn copy:assets && yarn workspace jan build", "build:electron:test": "yarn workspace jan build:test", - "build:extensions:windows": "rimraf ./electron/pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; npm install; npm run build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"", - "build:extensions:linux": "rimraf ./electron/pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'", - "build:extensions:darwin": "rimraf ./electron/pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'", + "build:extensions:windows": "rimraf ./pre-install/*.tgz && powershell -command \"$jobs = Get-ChildItem -Path './extensions' -Directory | ForEach-Object { Start-Job -Name ($_.Name) -ScriptBlock { param($_dir); try { Set-Location $_dir; npm install; npm run build:publish; Write-Output 'Build successful in ' + $_dir } catch { Write-Error 'Error in ' + $_dir; throw } } -ArgumentList $_.FullName }; $jobs | Wait-Job; $jobs | ForEach-Object { Receive-Job -Job $_ -Keep } | ForEach-Object { Write-Host $_ }; $failed = $jobs | Where-Object { $_.State -ne 'Completed' -or $_.ChildJobs[0].JobStateInfo.State -ne 'Completed' }; if ($failed) { Exit 1 }\"", + "build:extensions:linux": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'", + "build:extensions:darwin": "rimraf ./pre-install/*.tgz && find ./extensions -mindepth 1 -maxdepth 1 -type d -print0 | xargs -0 -n 1 -P 4 -I {} sh -c 'cd {} && npm install && npm run build:publish'", + "build:extensions:server": "yarn workspace build:extensions ", "build:extensions": "run-script-os", "build:test": "yarn copy:assets && yarn build:web && yarn workspace jan build:test", "build": "yarn build:web && yarn build:electron", diff --git a/pre-install/.gitkeep b/pre-install/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/server/helpers/setup.ts b/server/helpers/setup.ts new file mode 100644 index 000000000..7d8f8914a --- /dev/null +++ b/server/helpers/setup.ts @@ -0,0 +1,73 @@ +import { join, extname } from 'path' +import { existsSync, readdirSync, writeFileSync, mkdirSync } from 'fs' +import { init, installExtensions } from '@janhq/core/node' + +export async function setup() { + /** + * Setup Jan Data Directory + */ + const appDir = process.env.JAN_DATA_DIRECTORY ?? join(__dirname, '..', 'jan') + + console.debug(`Create app data directory at ${appDir}...`) + if (!existsSync(appDir)) mkdirSync(appDir) + //@ts-ignore + global.core = { + // Define appPath function for app to retrieve app path globaly + appPath: () => appDir, + } + init({ + extensionsPath: join(appDir, 'extensions'), + }) + + /** + * Write app configurations. See #1619 + */ + console.debug('Writing config file...') + writeFileSync( + join(appDir, 'settings.json'), + JSON.stringify({ + data_folder: appDir, + }), + 'utf-8' + ) + + if (!existsSync(join(appDir, 'settings'))) { + console.debug('Writing nvidia config file...') + mkdirSync(join(appDir, 'settings')) + writeFileSync( + join(appDir, 'settings', 'settings.json'), + JSON.stringify( + { + notify: true, + run_mode: 'cpu', + nvidia_driver: { + exist: false, + version: '', + }, + cuda: { + exist: false, + version: '', + }, + gpus: [], + gpu_highest_vram: '', + gpus_in_use: [], + is_initial: true, + }), + 'utf-8' + ) + } + + /** + * Install extensions + */ + + console.debug('Installing extensions...') + + const baseExtensionPath = join(__dirname, '../../..', 'pre-install') + const extensions = readdirSync(baseExtensionPath) + .filter((file) => extname(file) === '.tgz') + .map((file) => join(baseExtensionPath, file)) + + await installExtensions(extensions) + console.debug('Extensions installed') +} diff --git a/server/index.ts b/server/index.ts index 05bfdca96..98cc8385d 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,26 +1,27 @@ -import fastify from "fastify"; -import dotenv from "dotenv"; +import fastify from 'fastify' +import dotenv from 'dotenv' import { getServerLogPath, v1Router, logServer, getJanExtensionsPath, -} from "@janhq/core/node"; -import { join } from "path"; +} from '@janhq/core/node' +import { join } from 'path' +import tcpPortUsed from 'tcp-port-used' // Load environment variables -dotenv.config(); +dotenv.config() // Define default settings -const JAN_API_HOST = process.env.JAN_API_HOST || "127.0.0.1"; -const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || "1337"); +const JAN_API_HOST = process.env.JAN_API_HOST || '127.0.0.1' +const JAN_API_PORT = Number.parseInt(process.env.JAN_API_PORT || '1337') // Initialize server settings -let server: any | undefined = undefined; -let hostSetting: string = JAN_API_HOST; -let portSetting: number = JAN_API_PORT; -let corsEnabled: boolean = true; -let isVerbose: boolean = true; +let server: any | undefined = undefined +let hostSetting: string = JAN_API_HOST +let portSetting: number = JAN_API_PORT +let corsEnabled: boolean = true +let isVerbose: boolean = true /** * Server configurations @@ -32,80 +33,93 @@ let isVerbose: boolean = true; * @param baseDir - Base directory for the OpenAPI schema file */ export interface ServerConfig { - host?: string; - port?: number; - isCorsEnabled?: boolean; - isVerboseEnabled?: boolean; - schemaPath?: string; - baseDir?: string; + host?: string + port?: number + isCorsEnabled?: boolean + isVerboseEnabled?: boolean + schemaPath?: string + baseDir?: string + storageAdataper?: any } /** * Function to start the server * @param configs - Server configurations */ -export const startServer = async (configs?: ServerConfig) => { +export const startServer = async (configs?: ServerConfig): Promise => { + if (configs?.port && configs?.host) { + const inUse = await tcpPortUsed.check(Number(configs.port), configs.host) + if (inUse) { + const errorMessage = `Port ${configs.port} is already in use.` + logServer(errorMessage) + throw new Error(errorMessage) + } + } + // Update server settings - isVerbose = configs?.isVerboseEnabled ?? true; - hostSetting = configs?.host ?? JAN_API_HOST; - portSetting = configs?.port ?? JAN_API_PORT; - corsEnabled = configs?.isCorsEnabled ?? true; - const serverLogPath = getServerLogPath(); + isVerbose = configs?.isVerboseEnabled ?? true + hostSetting = configs?.host ?? JAN_API_HOST + portSetting = configs?.port ?? JAN_API_PORT + corsEnabled = configs?.isCorsEnabled ?? true + const serverLogPath = getServerLogPath() // Start the server try { // Log server start - if (isVerbose) logServer(`Debug: Starting JAN API server...`); + if (isVerbose) logServer(`Debug: Starting JAN API server...`) // Initialize Fastify server with logging server = fastify({ logger: { - level: "info", + level: 'info', file: serverLogPath, }, - }); + }) // Register CORS if enabled - if (corsEnabled) await server.register(require("@fastify/cors"), {}); + if (corsEnabled) await server.register(require('@fastify/cors'), {}) // Register Swagger for API documentation - await server.register(require("@fastify/swagger"), { - mode: "static", + await server.register(require('@fastify/swagger'), { + mode: 'static', specification: { - path: configs?.schemaPath ?? "./../docs/openapi/jan.yaml", - baseDir: configs?.baseDir ?? "./../docs/openapi", + path: configs?.schemaPath ?? './../docs/openapi/jan.yaml', + baseDir: configs?.baseDir ?? './../docs/openapi', }, - }); + }) // Register Swagger UI - await server.register(require("@fastify/swagger-ui"), { - routePrefix: "/", - baseDir: configs?.baseDir ?? join(__dirname, "../..", "./docs/openapi"), + await server.register(require('@fastify/swagger-ui'), { + routePrefix: '/', + baseDir: configs?.baseDir ?? join(__dirname, '../..', './docs/openapi'), uiConfig: { - docExpansion: "full", + docExpansion: 'full', deepLinking: false, }, staticCSP: false, transformSpecificationClone: true, - }); + }) // Register static file serving for extensions // TODO: Watch extension files changes and reload await server.register( (childContext: any, _: any, done: any) => { - childContext.register(require("@fastify/static"), { + childContext.register(require('@fastify/static'), { root: getJanExtensionsPath(), wildcard: false, - }); + }) - done(); + done() }, - { prefix: "extensions" } - ); + { prefix: 'extensions' } + ) + + // Register proxy middleware + if (configs?.storageAdataper) + server.addHook('preHandler', configs.storageAdataper) // Register API routes - await server.register(v1Router, { prefix: "/v1" }); - + await server.register(v1Router, { prefix: '/v1' }) // Start listening for requests await server .listen({ @@ -117,13 +131,15 @@ export const startServer = async (configs?: ServerConfig) => { if (isVerbose) logServer( `Debug: JAN API listening at: http://${hostSetting}:${portSetting}` - ); - }); + ) + }) + return true } catch (e) { // Log any errors - if (isVerbose) logServer(`Error: ${e}`); + if (isVerbose) logServer(`Error: ${e}`) } -}; + return false +} /** * Function to stop the server @@ -131,11 +147,11 @@ export const startServer = async (configs?: ServerConfig) => { export const stopServer = async () => { try { // Log server stop - if (isVerbose) logServer(`Debug: Server stopped`); + if (isVerbose) logServer(`Debug: Server stopped`) // Stop the server - await server.close(); + await server.close() } catch (e) { // Log any errors - if (isVerbose) logServer(`Error: ${e}`); + if (isVerbose) logServer(`Error: ${e}`) } -}; +} diff --git a/server/main.ts b/server/main.ts index c3eb69135..71fb11106 100644 --- a/server/main.ts +++ b/server/main.ts @@ -1,3 +1,7 @@ -import { startServer } from "./index"; - -startServer(); +import { s3 } from './middleware/s3' +import { setup } from './helpers/setup' +import { startServer as start } from './index' +/** + * Setup extensions and start the server + */ +setup().then(() => start({ storageAdataper: s3 })) diff --git a/server/middleware/s3.ts b/server/middleware/s3.ts new file mode 100644 index 000000000..28971a42b --- /dev/null +++ b/server/middleware/s3.ts @@ -0,0 +1,70 @@ +import { join } from 'path' + +// Middleware to intercept requests and proxy if certain conditions are met +const config = { + endpoint: process.env.AWS_ENDPOINT, + region: process.env.AWS_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, +} + +const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME + +const fs = require('@cyclic.sh/s3fs')(S3_BUCKET_NAME, config) +const PROXY_PREFIX = '/v1/fs' +const PROXY_ROUTES = ['/threads', '/messages'] + +export const s3 = (req: any, reply: any, done: any) => { + // Proxy FS requests to S3 using S3FS + if (req.url.startsWith(PROXY_PREFIX)) { + const route = req.url.split('/').pop() + const args = parseRequestArgs(req) + + // Proxy matched requests to the s3fs module + if (args.length && PROXY_ROUTES.some((route) => args[0].includes(route))) { + try { + // Handle customized route + // S3FS does not handle appendFileSync + if (route === 'appendFileSync') { + let result = handAppendFileSync(args) + + reply.status(200).send(result) + return + } + // Reroute the other requests to the s3fs module + const result = fs[route](...args) + reply.status(200).send(result) + return + } catch (ex) { + console.log(ex) + } + } + } + // Let other requests go through + done() +} + +const parseRequestArgs = (req: Request) => { + const { + getJanDataFolderPath, + normalizeFilePath, + } = require('@janhq/core/node') + + return JSON.parse(req.body as any).map((arg: any) => + typeof arg === 'string' && + (arg.startsWith(`file:/`) || arg.startsWith(`file:\\`)) + ? join(getJanDataFolderPath(), normalizeFilePath(arg)) + : arg + ) +} + +const handAppendFileSync = (args: any[]) => { + if (fs.existsSync(args[0])) { + const data = fs.readFileSync(args[0], 'utf-8') + return fs.writeFileSync(args[0], data + args[1]) + } else { + return fs.writeFileSync(args[0], args[1]) + } +} diff --git a/server/nodemon.json b/server/nodemon.json deleted file mode 100644 index 0ea41ca96..000000000 --- a/server/nodemon.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "watch": ["main.ts", "v1"], - "ext": "ts, json", - "exec": "tsc && node ./build/main.js" -} \ No newline at end of file diff --git a/server/package.json b/server/package.json index f61730da4..d9a2bbc9a 100644 --- a/server/package.json +++ b/server/package.json @@ -18,26 +18,29 @@ }, "dependencies": { "@alumna/reflect": "^1.1.3", + "@cyclic.sh/s3fs": "^1.2.9", "@fastify/cors": "^8.4.2", "@fastify/static": "^6.12.0", "@fastify/swagger": "^8.13.0", "@fastify/swagger-ui": "2.0.1", "@janhq/core": "link:./core", + "@npmcli/arborist": "^7.3.1", "dotenv": "^16.3.1", "fastify": "^4.24.3", - "request": "^2.88.2", "fetch-retry": "^5.0.6", - "tcp-port-used": "^1.0.2", - "request-progress": "^3.0.0" + "node-fetch": "2", + "request": "^2.88.2", + "request-progress": "^3.0.0", + "tcp-port-used": "^1.0.2" }, "devDependencies": { "@types/body-parser": "^1.19.5", "@types/npmcli__arborist": "^5.6.4", + "@types/tcp-port-used": "^1.0.4", "@typescript-eslint/eslint-plugin": "^6.7.3", "@typescript-eslint/parser": "^6.7.3", "eslint-plugin-react": "^7.33.2", "run-script-os": "^1.1.6", - "@types/tcp-port-used": "^1.0.4", "typescript": "^5.2.2" } } diff --git a/server/tsconfig.json b/server/tsconfig.json index 2c4fc4a64..dd27b8932 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -20,5 +20,5 @@ // "sourceMap": true, "include": ["./**/*.ts"], - "exclude": ["core", "build", "dist", "tests", "node_modules"] + "exclude": ["core", "build", "dist", "tests", "node_modules", "extensions"] } diff --git a/uikit/src/input/styles.scss b/uikit/src/input/styles.scss index 9990da8b4..e649f494d 100644 --- a/uikit/src/input/styles.scss +++ b/uikit/src/input/styles.scss @@ -1,6 +1,6 @@ .input { @apply border-border placeholder:text-muted-foreground flex h-9 w-full rounded-lg border bg-transparent px-3 py-1 transition-colors; - @apply disabled:cursor-not-allowed disabled:bg-zinc-100 disabled:dark:bg-zinc-800 disabled:dark:text-zinc-600; + @apply disabled:text-muted-foreground disabled:cursor-not-allowed disabled:bg-zinc-100 disabled:dark:bg-zinc-800 disabled:dark:text-zinc-600; @apply focus-within:outline-none focus-visible:outline-0 focus-visible:ring-2 focus-visible:ring-blue-500 focus-visible:ring-offset-1; @apply file:border-0 file:bg-transparent file:font-medium; } diff --git a/uikit/src/select/styles.scss b/uikit/src/select/styles.scss index bc5b6c0cc..90485723a 100644 --- a/uikit/src/select/styles.scss +++ b/uikit/src/select/styles.scss @@ -1,6 +1,6 @@ .select { @apply placeholder:text-muted-foreground border-border flex h-9 w-full items-center justify-between whitespace-nowrap rounded-md border bg-transparent px-3 py-2 text-sm shadow-sm disabled:cursor-not-allowed [&>span]:line-clamp-1; - @apply disabled:cursor-not-allowed disabled:bg-zinc-100 disabled:dark:bg-zinc-800 disabled:dark:text-zinc-600; + @apply disabled:text-muted-foreground disabled:cursor-not-allowed disabled:bg-zinc-100 disabled:dark:bg-zinc-800 disabled:dark:text-zinc-600; @apply focus-within:outline-none focus-visible:outline-0 focus-visible:ring-2 focus-visible:ring-blue-500 focus-visible:ring-offset-1; &-caret { @@ -21,6 +21,7 @@ &-item { @apply hover:bg-secondary relative my-1 block w-full cursor-pointer select-none items-center rounded-sm px-4 py-2 text-sm data-[disabled]:pointer-events-none data-[disabled]:opacity-50; + @apply focus:outline-none focus-visible:outline-0; } &-trigger-viewport { diff --git a/uikit/types/declaration.d.ts b/uikit/types/declaration.d.ts index 85b1a7136..f8e975fa5 100644 --- a/uikit/types/declaration.d.ts +++ b/uikit/types/declaration.d.ts @@ -1,4 +1,4 @@ declare module '*.scss' { - const content: Record; - export default content; -} \ No newline at end of file + const content: Record + export default content +} diff --git a/web/.prettierignore b/web/.prettierignore deleted file mode 100644 index 02d9145c1..000000000 --- a/web/.prettierignore +++ /dev/null @@ -1,5 +0,0 @@ -.next/ -node_modules/ -dist/ -*.hbs -*.mdx \ No newline at end of file diff --git a/web/.prettierrc b/web/.prettierrc deleted file mode 100644 index 46f1abcb0..000000000 --- a/web/.prettierrc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "semi": false, - "singleQuote": true, - "quoteProps": "consistent", - "trailingComma": "es5", - "endOfLine": "auto", - "plugins": ["prettier-plugin-tailwindcss"] -} diff --git a/web/app/error.tsx b/web/app/error.tsx new file mode 100644 index 000000000..25b24b9ef --- /dev/null +++ b/web/app/error.tsx @@ -0,0 +1,89 @@ +'use client' // Error components must be Client Components + +import { useEffect, useState } from 'react' + +export default function Error({ + error, +}: { + error: Error & { digest?: string } + reset: () => void +}) { + const [showFull, setShowFull] = useState(false) + useEffect(() => { + // Log the error to an error reporting service + console.error(error) + }, [error]) + + return ( + <> +
+
+
+
+ + + + +
+
+

+ Oops! Unexpected error occurred. +

+

+ Something went wrong. Try to{' '} + {' '} + or
feel free to{' '} + + contact us + {' '} + if the problem presists. +

+
+ Error: + {error.message} +
+ 
+                {showFull ? error.stack : error.stack?.slice(0, 200)}
+
+ +
+
+
+
+ + ) +} diff --git a/web/app/page.tsx b/web/app/page.tsx index 20b15a235..92d654528 100644 --- a/web/app/page.tsx +++ b/web/app/page.tsx @@ -11,7 +11,6 @@ import ExploreModelsScreen from '@/screens/ExploreModels' import LocalServerScreen from '@/screens/LocalServer' import SettingsScreen from '@/screens/Settings' -import SystemMonitorScreen from '@/screens/SystemMonitor' export default function Page() { const { mainViewState } = useMainViewState() @@ -26,10 +25,6 @@ export default function Page() { children = break - case MainViewState.SystemMonitor: - children = - break - case MainViewState.LocalServer: children = break diff --git a/web/constants/screens.ts b/web/constants/screens.ts index 6a8adc185..74b441b17 100644 --- a/web/constants/screens.ts +++ b/web/constants/screens.ts @@ -3,6 +3,5 @@ export enum MainViewState { MyModels, Settings, Thread, - SystemMonitor, LocalServer, } diff --git a/web/containers/CardSidebar/index.tsx b/web/containers/CardSidebar/index.tsx index 38a8678d9..89ff60e66 100644 --- a/web/containers/CardSidebar/index.tsx +++ b/web/containers/CardSidebar/index.tsx @@ -22,6 +22,7 @@ interface Props { rightAction?: ReactNode title: string asChild?: boolean + isShow?: boolean hideMoreVerticalAction?: boolean } export default function CardSidebar({ @@ -30,8 +31,9 @@ export default function CardSidebar({ asChild, rightAction, hideMoreVerticalAction, + isShow, }: Props) { - const [show, setShow] = useState(true) + const [show, setShow] = useState(isShow ?? false) const [more, setMore] = useState(false) const [menu, setMenu] = useState(null) const [toggle, setToggle] = useState(null) @@ -67,8 +69,8 @@ export default function CardSidebar({ show && 'rotate-180' )} /> + {title} - {title}
{rightAction && rightAction} @@ -156,7 +158,10 @@ export default function CardSidebar({ ) : ( <> - Opens {title}.json. + Opens{' '} + + {title === 'Tools' ? 'assistant' : title}.json. +  Changes affect all new threads. )} diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index 140a1aba1..191c7bcbe 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -14,7 +14,14 @@ import { import { atom, useAtom, useAtomValue, useSetAtom } from 'jotai' -import { MonitorIcon } from 'lucide-react' +import { + MonitorIcon, + LayoutGridIcon, + FoldersIcon, + GlobeIcon, + CheckIcon, + CopyIcon, +} from 'lucide-react' import { twMerge } from 'tailwind-merge' @@ -22,6 +29,7 @@ import { MainViewState } from '@/constants/screens' import { useActiveModel } from '@/hooks/useActiveModel' +import { useClipboard } from '@/hooks/useClipboard' import { useMainViewState } from '@/hooks/useMainViewState' import useRecommendedModel from '@/hooks/useRecommendedModel' @@ -42,6 +50,8 @@ import { export const selectedModelAtom = atom(undefined) +const engineOptions = ['Local', 'Remote'] + // TODO: Move all of the unscoped logics outside of the component const DropdownListSidebar = ({ strictedThread = true, @@ -51,13 +61,24 @@ const DropdownListSidebar = ({ const activeThread = useAtomValue(activeThreadAtom) const [selectedModel, setSelectedModel] = useAtom(selectedModelAtom) const setThreadModelParams = useSetAtom(setThreadModelParamsAtom) - + const [isTabActive, setIsTabActive] = useState(0) const { stateModel } = useActiveModel() const [serverEnabled, setServerEnabled] = useAtom(serverEnabledAtom) const { setMainViewState } = useMainViewState() const [loader, setLoader] = useState(0) const { recommendedModel, downloadedModels } = useRecommendedModel() const { updateModelParameter } = useUpdateModelParameters() + const clipboard = useClipboard({ timeout: 1000 }) + const [copyId, setCopyId] = useState('') + + const localModel = downloadedModels.filter( + (model) => model.engine === InferenceEngine.nitro + ) + const remoteModel = downloadedModels.filter( + (model) => model.engine === InferenceEngine.openai + ) + + const modelOptions = isTabActive === 0 ? localModel : remoteModel useEffect(() => { if (!activeThread) return @@ -73,7 +94,7 @@ const DropdownListSidebar = ({ // This is fake loader please fix this when we have realtime percentage when load model useEffect(() => { - if (stateModel.loading) { + if (stateModel.model === selectedModel?.id && stateModel.loading) { if (loader === 24) { setTimeout(() => { setLoader(loader + 1) @@ -94,7 +115,7 @@ const DropdownListSidebar = ({ } else { setLoader(0) } - }, [stateModel.loading, loader]) + }, [stateModel.loading, loader, selectedModel, stateModel.model]) const onValueSelected = useCallback( async (modelId: string) => { @@ -138,12 +159,16 @@ const DropdownListSidebar = ({ return null } + const selectedModelLoading = + stateModel.model === selectedModel?.id && stateModel.loading + return ( <>