Merge branch 'website/feb-2024-update' of https://github.com/janhq/jan into website/feb-2024-update

.github/workflows/jan-server-build-nightly.yml

@@ -0,0 +1,40 @@
+name: Jan Build Docker Nightly or Manual
+
+on:
+  push:
+    branches:
+      - main
+      - feature/helmchart-and-ci-jan-server
+    paths-ignore:
+      - 'README.md'
+      - 'docs/**'
+  schedule:
+    - cron: '0 20 * * 1,2,3' # At 8 PM UTC on Monday, Tuesday, and Wednesday which is 3 AM UTC+7 Tuesday, Wednesday, and Thursday
+  workflow_dispatch:
+
+jobs:
+  # Job create Update app version based on latest release tag with build number and save to output
+  get-update-version:
+    uses: ./.github/workflows/template-get-update-version.yml
+
+  build-cpu:
+    uses: ./.github/workflows/template-build-jan-server.yml
+    permissions:
+      packages: write
+    secrets: inherit
+    needs: [get-update-version]
+    with:
+      dockerfile_path: ./Dockerfile
+      docker_image_tag: "ghcr.io/janhq/jan-server:dev-cpu-latest,ghcr.io/janhq/jan-server:dev-cpu-${{ needs.get-update-version.outputs.new_version }}"
+
+  build-gpu:
+    uses: ./.github/workflows/template-build-jan-server.yml
+    permissions:
+      packages: write
+    secrets: inherit
+    needs: [get-update-version]
+    with:
+      dockerfile_path: ./Dockerfile.gpu
+      docker_image_tag: "ghcr.io/janhq/jan-server:dev-cuda-12.2-latest,ghcr.io/janhq/jan-server:dev-cuda-12.2-${{ needs.get-update-version.outputs.new_version }}"
+    
+

.github/workflows/jan-server-build.yml

@@ -0,0 +1,30 @@
+name: Jan Build Docker
+
+on:
+  push:
+    tags: ["v[0-9]+.[0-9]+.[0-9]+"]
+
+jobs:
+  # Job create Update app version based on latest release tag with build number and save to output
+  get-update-version:
+    uses: ./.github/workflows/template-get-update-version.yml
+
+  build-cpu:
+    permissions:
+      packages: write
+    uses: ./.github/workflows/template-build-jan-server.yml
+    secrets: inherit
+    needs: [get-update-version]
+    with:
+      dockerfile_path: ./Dockerfile
+      docker_image_tag: "ghcr.io/janhq/jan-server:cpu-latest,ghcr.io/janhq/jan-server:cpu-${{ needs.get-update-version.outputs.new_version }}"
+
+  build-gpu:
+    permissions:
+      packages: write
+    uses: ./.github/workflows/template-build-jan-server.yml
+    secrets: inherit
+    needs: [get-update-version]
+    with:
+      dockerfile_path: ./Dockerfile.gpu
+      docker_image_tag: "ghcr.io/janhq/jan-server:cuda-12.2-latest,ghcr.io/janhq/jan-server:cuda-12.2-${{ needs.get-update-version.outputs.new_version }}"

.github/workflows/template-build-jan-server.yml

@@ -0,0 +1,39 @@
+name: build-jan-server
+on:
+  workflow_call:
+    inputs:
+      dockerfile_path:
+        required: false
+        type: string
+        default: './Dockerfile'
+      docker_image_tag:
+        required: true
+        type: string
+        default: 'ghcr.io/janhq/jan-server:dev-latest'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: janhq/jan-server
+    permissions:
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ${{ inputs.dockerfile_path }}
+          push: true
+          tags: ${{ inputs.docker_image_tag }}

.gitignore

@@ -5,6 +5,7 @@
 error.log
 node_modules
 *.tgz
+!charts/server/charts/*.tgz
 yarn.lock
 dist
 build
@@ -28,4 +29,5 @@ extensions/inference-nitro-extension/bin/*/*.exp
 extensions/inference-nitro-extension/bin/*/*.lib
 extensions/inference-nitro-extension/bin/saved-*
 extensions/inference-nitro-extension/bin/*.tar.gz
-
+extensions/inference-nitro-extension/bin/vulkaninfoSDK.exe
+extensions/inference-nitro-extension/bin/vulkaninfo

Dockerfile

@@ -14,9 +14,6 @@ COPY . ./
 RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \
     jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json
 RUN make install-and-build
-RUN yarn workspace jan-web install
-
-RUN export NODE_ENV=production && yarn workspace jan-web build
 
 # # 2. Rebuild the source code only when needed
 FROM base AS runner
@@ -42,12 +39,13 @@ COPY --from=builder /app/docs/openapi ./docs/openapi/
 COPY --from=builder /app/pre-install ./pre-install/
 
 # Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache
-COPY --from=builder /app/web/out ./web/out/
+COPY --from=builder /app/uikit ./uikit/
-COPY --from=builder /app/web/.next ./web/.next/
+COPY --from=builder /app/web ./web/
-COPY --from=builder /app/web/package.json ./web/package.json
-COPY --from=builder /app/web/yarn.lock ./web/yarn.lock
 COPY --from=builder /app/models ./models/
 
+RUN yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build
+RUN yarn workspace jan-web install
+
 RUN npm install -g serve@latest
 
 EXPOSE 1337 3000 3928
@@ -55,7 +53,9 @@ EXPOSE 1337 3000 3928
 ENV JAN_API_HOST 0.0.0.0
 ENV JAN_API_PORT 1337
 
-CMD ["sh", "-c", "cd server && node build/main.js & cd web && npx serve out"]
+ENV API_BASE_URL http://localhost:1337
+
+CMD ["sh", "-c", "export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out & cd server && node build/main.js"]
 
 # docker build -t jan .
 # docker run -p 1337:1337 -p 3000:3000 -p 3928:3928 jan

Dockerfile.gpu

@@ -28,9 +28,6 @@ COPY . ./
 RUN export NITRO_VERSION=$(cat extensions/inference-nitro-extension/bin/version.txt) && \
     jq --arg nitroVersion $NITRO_VERSION '(.scripts."downloadnitro:linux" | gsub("\\${NITRO_VERSION}"; $nitroVersion)) | gsub("\r"; "")' extensions/inference-nitro-extension/package.json > /tmp/newcommand.txt && export NEW_COMMAND=$(sed 's/^"//;s/"$//' /tmp/newcommand.txt) && jq --arg newCommand "$NEW_COMMAND" '.scripts."downloadnitro:linux" = $newCommand' extensions/inference-nitro-extension/package.json > /tmp/package.json && mv /tmp/package.json extensions/inference-nitro-extension/package.json
 RUN make install-and-build
-RUN yarn workspace jan-web install
-
-RUN export NODE_ENV=production && yarn workspace jan-web build
 
 # # 2. Rebuild the source code only when needed
 FROM base AS runner
@@ -66,12 +63,13 @@ COPY --from=builder /app/docs/openapi ./docs/openapi/
 COPY --from=builder /app/pre-install ./pre-install/
 
 # Copy the package.json, yarn.lock, and output of web yarn space to leverage Docker cache
-COPY --from=builder /app/web/out ./web/out/
+COPY --from=builder /app/uikit ./uikit/
-COPY --from=builder /app/web/.next ./web/.next/
+COPY --from=builder /app/web ./web/
-COPY --from=builder /app/web/package.json ./web/package.json
-COPY --from=builder /app/web/yarn.lock ./web/yarn.lock
 COPY --from=builder /app/models ./models/
 
+RUN yarn workspace @janhq/uikit install && yarn workspace @janhq/uikit build
+RUN yarn workspace jan-web install
+
 RUN npm install -g serve@latest
 
 EXPOSE 1337 3000 3928
@@ -81,7 +79,9 @@ ENV LD_LIBRARY_PATH=/usr/local/cuda/targets/x86_64-linux/lib:/usr/local/cuda-12.
 ENV JAN_API_HOST 0.0.0.0
 ENV JAN_API_PORT 1337
 
-CMD ["sh", "-c", "cd server && node build/main.js & cd web && npx serve out"]
+ENV API_BASE_URL http://localhost:1337
+
+CMD ["sh", "-c", "export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out & cd server && node build/main.js"]
 
 # pre-requisites: nvidia-docker
 # docker build -t jan-gpu . -f Dockerfile.gpu

Makefile

@@ -52,18 +52,28 @@ build: check-file-counts
 
 clean:
 ifeq ($(OS),Windows_NT)
-	powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist -Recurse -Directory | Remove-Item -Recurse -Force"
+	powershell -Command "Get-ChildItem -Path . -Include node_modules, .next, dist, build, out -Recurse -Directory | Remove-Item -Recurse -Force"
+	powershell -Command "Remove-Item -Recurse -Force ./pre-install/*.tgz"
+	powershell -Command "Remove-Item -Recurse -Force ./electron/pre-install/*.tgz"
 	rmdir /s /q "%USERPROFILE%\jan\extensions"
 else ifeq ($(shell uname -s),Linux)
 	find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 	find . -name ".next" -type d -exec rm -rf '{}' +
 	find . -name "dist" -type d -exec rm -rf '{}' +
+	find . -name "build" -type d -exec rm -rf '{}' +
+	find . -name "out" -type d -exec rm -rf '{}' +
+	rm -rf ./pre-install/*.tgz
+	rm -rf ./electron/pre-install/*.tgz
 	rm -rf "~/jan/extensions"
 	rm -rf "~/.cache/jan*"
 else
 	find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
 	find . -name ".next" -type d -exec rm -rf '{}' +
 	find . -name "dist" -type d -exec rm -rf '{}' +
+	find . -name "build" -type d -exec rm -rf '{}' +
+	find . -name "out" -type d -exec rm -rf '{}' +
+	rm -rf ./pre-install/*.tgz
+	rm -rf ./electron/pre-install/*.tgz
 	rm -rf ~/jan/extensions
 	rm -rf ~/Library/Caches/jan*
 endif

README.md

@@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute
   <tr style="text-align:center">
     <td style="text-align:center"><b>Experimental (Nightly Build)</b></td>
     <td style="text-align:center">
-      <a href='https://delta.jan.ai/latest/jan-win-x64-0.4.6-273.exe'>
+      <a href='https://delta.jan.ai/latest/jan-win-x64-0.4.6-281.exe'>
         <img src='./docs/static/img/windows.png' style="height:14px; width: 14px" />
         <b>jan.exe</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://delta.jan.ai/latest/jan-mac-x64-0.4.6-273.dmg'>
+      <a href='https://delta.jan.ai/latest/jan-mac-x64-0.4.6-281.dmg'>
         <img src='./docs/static/img/mac.png' style="height:15px; width: 15px" />
         <b>Intel</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://delta.jan.ai/latest/jan-mac-arm64-0.4.6-273.dmg'>
+      <a href='https://delta.jan.ai/latest/jan-mac-arm64-0.4.6-281.dmg'>
         <img src='./docs/static/img/mac.png' style="height:15px; width: 15px" />
         <b>M1/M2</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://delta.jan.ai/latest/jan-linux-amd64-0.4.6-273.deb'>
+      <a href='https://delta.jan.ai/latest/jan-linux-amd64-0.4.6-281.deb'>
         <img src='./docs/static/img/linux.png' style="height:14px; width: 14px" />
         <b>jan.deb</b>
       </a>
     </td>
     <td style="text-align:center">
-      <a href='https://delta.jan.ai/latest/jan-linux-x86_64-0.4.6-273.AppImage'>
+      <a href='https://delta.jan.ai/latest/jan-linux-x86_64-0.4.6-281.AppImage'>
         <img src='./docs/static/img/linux.png' style="height:14px; width: 14px" />
         <b>jan.AppImage</b>
       </a>
@@ -235,61 +235,84 @@ This will build the app MacOS m1/m2 for production (with code signing already do
 
 - Run Jan in Docker mode
 
-  - **Option 1**: Run Jan in CPU mode
+| Docker compose Profile | Description                                  |
+| ---------------------- | -------------------------------------------- |
+| `cpu-fs`               | Run Jan in CPU mode with default file system |
+| `cpu-s3fs`             | Run Jan in CPU mode with S3 file system      |
+| `gpu-fs`               | Run Jan in GPU mode with default file system |
+| `gpu-s3fs`             | Run Jan in GPU mode with S3 file system      |
+
+| Environment Variable    | Description                                                                                             |
+| ----------------------- | ------------------------------------------------------------------------------------------------------- |
+| `S3_BUCKET_NAME`        | S3 bucket name - leave blank for default file system                                                    |
+| `AWS_ACCESS_KEY_ID`     | AWS access key ID - leave blank for default file system                                                 |
+| `AWS_SECRET_ACCESS_KEY` | AWS secret access key - leave blank for default file system                                             |
+| `AWS_ENDPOINT`          | AWS endpoint URL - leave blank for default file system                                                  |
+| `AWS_REGION`            | AWS region - leave blank for default file system                                                        |
+| `API_BASE_URL`          | Jan Server URL, please modify it as your public ip address or domain name default http://localhost:1377 |
+
+- **Option 1**: Run Jan in CPU mode
+
+  ```bash
+  # cpu mode with default file system
+  docker compose --profile cpu-fs up -d
+
+  # cpu mode with S3 file system
+  docker compose --profile cpu-s3fs up -d
+  ```
+
+- **Option 2**: Run Jan in GPU mode
+
+  - **Step 1**: Check CUDA compatibility with your NVIDIA driver by running `nvidia-smi` and check the CUDA version in the output
 
     ```bash
-    docker compose --profile cpu up -d
+    nvidia-smi
+
+    # Output
+    +---------------------------------------------------------------------------------------+
+    | NVIDIA-SMI 531.18                 Driver Version: 531.18       CUDA Version: 12.1     |
+    |-----------------------------------------+----------------------+----------------------+
+    | GPU  Name                      TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |
+    | Fan  Temp  Perf            Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
+    |                                         |                      |               MIG M. |
+    |=========================================+======================+======================|
+    |   0  NVIDIA GeForce RTX 4070 Ti    WDDM | 00000000:01:00.0  On |                  N/A |
+    |  0%   44C    P8               16W / 285W|   1481MiB / 12282MiB |      2%      Default |
+    |                                         |                      |                  N/A |
+    +-----------------------------------------+----------------------+----------------------+
+    |   1  NVIDIA GeForce GTX 1660 Ti    WDDM | 00000000:02:00.0 Off |                  N/A |
+    |  0%   49C    P8               14W / 120W|      0MiB /  6144MiB |      0%      Default |
+    |                                         |                      |                  N/A |
+    +-----------------------------------------+----------------------+----------------------+
+    |   2  NVIDIA GeForce GTX 1660 Ti    WDDM | 00000000:05:00.0 Off |                  N/A |
+    | 29%   38C    P8               11W / 120W|      0MiB /  6144MiB |      0%      Default |
+    |                                         |                      |                  N/A |
+    +-----------------------------------------+----------------------+----------------------+
+
+    +---------------------------------------------------------------------------------------+
+    | Processes:                                                                            |
+    |  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
+    |        ID   ID                                                             Usage      |
+    |=======================================================================================|
     ```
 
-  - **Option 2**: Run Jan in GPU mode
+  - **Step 2**: Visit [NVIDIA NGC Catalog ](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags) and find the smallest minor version of image tag that matches your CUDA version (e.g., 12.1 -> 12.1.0)
 
-    - **Step 1**: Check CUDA compatibility with your NVIDIA driver by running `nvidia-smi` and check the CUDA version in the output
+  - **Step 3**: Update the `Dockerfile.gpu` line number 5 with the latest minor version of the image tag from step 2 (e.g. change `FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base` to `FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS base`)
 
-      ```bash
+  - **Step 4**: Run command to start Jan in GPU mode
-      nvidia-smi
 
-      # Output
+    ```bash
-      +---------------------------------------------------------------------------------------+
+    # GPU mode with default file system
-      | NVIDIA-SMI 531.18                 Driver Version: 531.18       CUDA Version: 12.1     |
+    docker compose --profile gpu up -d
-      |-----------------------------------------+----------------------+----------------------+
-      | GPU  Name                      TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |
-      | Fan  Temp  Perf            Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
-      |                                         |                      |               MIG M. |
-      |=========================================+======================+======================|
-      |   0  NVIDIA GeForce RTX 4070 Ti    WDDM | 00000000:01:00.0  On |                  N/A |
-      |  0%   44C    P8               16W / 285W|   1481MiB / 12282MiB |      2%      Default |
-      |                                         |                      |                  N/A |
-      +-----------------------------------------+----------------------+----------------------+
-      |   1  NVIDIA GeForce GTX 1660 Ti    WDDM | 00000000:02:00.0 Off |                  N/A |
-      |  0%   49C    P8               14W / 120W|      0MiB /  6144MiB |      0%      Default |
-      |                                         |                      |                  N/A |
-      +-----------------------------------------+----------------------+----------------------+
-      |   2  NVIDIA GeForce GTX 1660 Ti    WDDM | 00000000:05:00.0 Off |                  N/A |
-      | 29%   38C    P8               11W / 120W|      0MiB /  6144MiB |      0%      Default |
-      |                                         |                      |                  N/A |
-      +-----------------------------------------+----------------------+----------------------+
 
-      +---------------------------------------------------------------------------------------+
+    # GPU mode with S3 file system
-      | Processes:                                                                            |
+    docker compose --profile gpu-s3fs up -d
-      |  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
+    ```
-      |        ID   ID                                                             Usage      |
-      |=======================================================================================|
-      ```
 
-    - **Step 2**: Visit [NVIDIA NGC Catalog ](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags) and find the smallest minor version of image tag that matches your CUDA version (e.g., 12.1 -> 12.1.0)
+This will start the web server and you can access Jan at `http://localhost:3000`.
 
-    - **Step 3**: Update the `Dockerfile.gpu` line number 5 with the latest minor version of the image tag from step 2 (e.g. change `FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS base` to `FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 AS base`)
+> Note: RAG feature is not supported in Docker mode with s3fs yet.
-
-    - **Step 4**: Run command to start Jan in GPU mode
-
-      ```bash
-      # GPU mode
-      docker compose --profile gpu up -d
-      ```
-
-  This will start the web server and you can access Jan at `http://localhost:3000`.
-
-  > Note: Currently, Docker mode is only work for development and localhost, production is not supported yet. RAG feature is not supported in Docker mode yet.
 
 ## Acknowledgements
 

charts/server/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: common
+  repository: oci://ghcr.io/janhq/charts
+  version: 0.1.2
+digest: sha256:35e98bde174130787755b0f8ea2359b7b6790d965a7157c2f7cabf1bc8c04471
+generated: "2024-02-20T16:20:37.6530108+07:00"

charts/server/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v2
+name: jan-server
+description: A Helm chart for Kubernetes
+type: application
+version: 0.1.0
+appVersion: '1.0.0'
+dependencies:
+  - name: common
+    version: 0.1.2 # common-chart-version
+    repository: oci://ghcr.io/janhq/charts

charts/server/config.json

@@ -0,0 +1,4 @@
+{
+    "image-list": "server=ghcr.io/janhq/jan-server",
+    "platforms": "linux/amd64"
+}

charts/server/values.yaml

@@ -0,0 +1,256 @@
+common:
+  imageTag: v0.4.6-cpu
+  # DO NOT CHANGE THE LINE ABOVE. MAKE ALL CHANGES BELOW
+
+  # Global pvc for all workload
+  pvc:
+    enabled: false
+    name: 'janroot'
+    accessModes: 'ReadWriteOnce'
+    storageClassName: ''
+    capacity: '50Gi'
+
+  # Global image pull secret
+  imagePullSecrets: []
+
+  externalSecret:
+    create: false
+    name: ''
+    annotations: {}
+
+  nameOverride: 'jan-server'
+  fullnameOverride: 'jan-server'
+
+  serviceAccount:
+    create: true
+    annotations: {}
+    name: 'jan-server-service-account'
+
+  podDisruptionBudget:
+    create: false
+    minAvailable: 1
+
+  workloads:
+    - name: server
+      image:
+        repository: ghcr.io/janhq/jan-server
+        pullPolicy: Always
+
+      command: ['/bin/sh', '-c']
+      args: ['cd server && node build/main.js']
+
+      replicaCount: 1
+      ports:
+        containerPort: 1337
+
+      strategy:
+        canary:
+          steps:
+            - setWeight: 50
+            - pause: { duration: 1m }
+
+      ingress:
+        enabled: true
+        className: 'nginx'
+        annotations:
+          nginx.ingress.kubernetes.io/proxy-body-size: '100m'
+          nginx.ingress.kubernetes.io/proxy-read-timeout: '1800'
+          nginx.ingress.kubernetes.io/proxy-send-timeout: '1800'
+          # cert-manager.io/cluster-issuer: 'jan-ai-dns01-cluster-issuer'
+          # nginx.ingress.kubernetes.io/force-ssl-redirect: 'true'
+          nginx.ingress.kubernetes.io/backend-protocol: HTTP
+        hosts:
+          - host: server.local
+            paths:
+              - path: /
+                pathType: Prefix
+        tls:
+          []
+          # - hosts:
+          #     - server-dev.jan.ai
+          #   secretName: jan-server-prod-tls-v2
+
+      instrumentation:
+        enabled: false
+      podAnnotations: {}
+
+      podSecurityContext: {}
+
+      securityContext: {}
+
+      service:
+        extenalLabel: {}
+        type: ClusterIP
+        port: 1337
+        targetPort: 1337
+
+      # If you want to use GPU, please uncomment the following lines and change imageTag to the one with GPU support
+      resources:
+        # limits:
+        #   nvidia.com/gpu: 1
+        requests:
+          cpu: 2000m
+          memory: 8192M
+
+      # If you want to use pv, please uncomment the following lines and enable pvc.enabled
+      volumes:
+        []
+        # - name: janroot
+        #   persistentVolumeClaim:
+        #     claimName: janroot
+
+      volumeMounts:
+        []
+        # - name: janroot
+        #   mountPath: /app/server/build/jan
+
+      # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME, AWS_ENDPOINT, AWS_REGION should mount as a secret env instead of plain text here
+      # Change API_BASE_URL to your server's public domain
+      env:
+        - name: API_BASE_URL
+          value: 'http://server.local'
+
+      lifecycle: {}
+      autoscaling:
+        enabled: false
+        minReplicas: 2
+        maxReplicas: 3
+        targetCPUUtilizationPercentage: 95
+        targetMemoryUtilizationPercentage: 95
+
+      kedaScaling:
+        enabled: false # ignore if autoscaling.enable = true
+        cooldownPeriod: 30
+        pollingInterval: 2
+        minReplicas: 1
+        maxReplicas: 5
+        metricName: celery_queue_length
+        query: celery_queue_length{queue_name="myqueue"} # change queue_name here
+        serverAddress: http://prometheus-prod-kube-prome-prometheus.monitoring.svc:9090
+        threshold: '3'
+
+      nodeSelector: {}
+
+      tolerations: []
+
+      podSecurityGroup:
+        enabled: false
+        securitygroupid: []
+
+      # Reloader Option
+      reloader: 'false'
+      vpa:
+        enabled: false
+
+    - name: web
+      image:
+        repository: ghcr.io/janhq/jan-server
+        pullPolicy: Always
+
+      command: ['/bin/sh', '-c']
+      args:
+        [
+          'export NODE_ENV=production && yarn workspace jan-web build && cd web && npx serve out',
+        ]
+
+      replicaCount: 1
+      ports:
+        containerPort: 3000
+
+      strategy:
+        canary:
+          steps:
+            - setWeight: 50
+            - pause: { duration: 1m }
+
+      ingress:
+        enabled: true
+        className: 'nginx'
+        annotations:
+          nginx.ingress.kubernetes.io/proxy-body-size: '100m'
+          nginx.ingress.kubernetes.io/proxy-read-timeout: '1800'
+          nginx.ingress.kubernetes.io/proxy-send-timeout: '1800'
+          # cert-manager.io/cluster-issuer: 'jan-ai-dns01-cluster-issuer'
+          # nginx.ingress.kubernetes.io/force-ssl-redirect: 'true'
+          nginx.ingress.kubernetes.io/backend-protocol: HTTP
+        hosts:
+          - host: web.local
+            paths:
+              - path: /
+                pathType: Prefix
+        tls:
+          []
+          # - hosts:
+          #     - server-dev.jan.ai
+          #   secretName: jan-server-prod-tls-v2
+
+      instrumentation:
+        enabled: false
+      podAnnotations: {}
+
+      podSecurityContext: {}
+
+      securityContext: {}
+
+      service:
+        extenalLabel: {}
+        type: ClusterIP
+        port: 3000
+        targetPort: 3000
+
+      resources:
+        limits:
+          cpu: 1000m
+          memory: 2048M
+        requests:
+          cpu: 50m
+          memory: 500M
+
+      volumes:
+        []
+        # - name: janroot
+        #   persistentVolumeClaim:
+        #     claimName: janroot
+
+      volumeMounts:
+        []
+        # - name: janroot
+        #   mountPath: /app/server/build/jan
+
+      # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET_NAME, AWS_ENDPOINT, AWS_REGION should mount as a secret env instead of plain text here
+      # Change API_BASE_URL to your server's public domain
+      env:
+        - name: API_BASE_URL
+          value: 'http://server.local'
+
+      lifecycle: {}
+      autoscaling:
+        enabled: true
+        minReplicas: 1
+        maxReplicas: 3
+        targetCPUUtilizationPercentage: 95
+        targetMemoryUtilizationPercentage: 95
+
+      kedaScaling:
+        enabled: false # ignore if autoscaling.enable = true
+        cooldownPeriod: 30
+        pollingInterval: 2
+        minReplicas: 1
+        maxReplicas: 5
+        metricName: celery_queue_length
+        query: celery_queue_length{queue_name="myqueue"} # change queue_name here
+        serverAddress: http://prometheus-prod-kube-prome-prometheus.monitoring.svc:9090
+        threshold: '3'
+
+      nodeSelector: {}
+
+      tolerations: []
+
+      podSecurityGroup:
+        enabled: false
+        securitygroupid: []
+
+      # Reloader Option
+      reloader: 'false'
+      vpa:
+        enabled: false

core/src/node/api/processors/app.ts

@@ -1,6 +1,5 @@
 import { basename, isAbsolute, join, relative } from 'path'
 
-import { AppRoute } from '../../../api'
 import { Processor } from './Processor'
 import { getAppConfigurations as appConfiguration, updateAppConfiguration } from '../../helper'
 import { log as writeLog, logServer as writeServerLog } from '../../helper/log'

core/src/node/api/restful/common.ts

@@ -8,7 +8,7 @@ import {
   createMessage,
   createThread,
   getMessages,
-  retrieveMesasge,
+  retrieveMessage,
   updateThread,
 } from './helper/builder'
 
@@ -17,10 +17,18 @@ import { startModel, stopModel } from './helper/startStopModel'
 import { ModelSettingParams } from '../../../types'
 
 export const commonRouter = async (app: HttpServer) => {
+  const normalizeData = (data: any) => {
+    return {
+      object: 'list',
+      data,
+    }
+  }
   // Common Routes
   // Read & Delete :: Threads | Models | Assistants
   Object.keys(JanApiRouteConfiguration).forEach((key) => {
-    app.get(`/${key}`, async (_request) => getBuilder(JanApiRouteConfiguration[key]))
+    app.get(`/${key}`, async (_request) =>
+      getBuilder(JanApiRouteConfiguration[key]).then(normalizeData)
+    )
 
     app.get(`/${key}/:id`, async (request: any) =>
       retrieveBuilder(JanApiRouteConfiguration[key], request.params.id)
@@ -34,10 +42,12 @@ export const commonRouter = async (app: HttpServer) => {
   // Threads
   app.post(`/threads/`, async (req, res) => createThread(req.body))
 
-  app.get(`/threads/:threadId/messages`, async (req, res) => getMessages(req.params.threadId))
+  app.get(`/threads/:threadId/messages`, async (req, res) =>
+    getMessages(req.params.threadId).then(normalizeData)
+  )
 
   app.get(`/threads/:threadId/messages/:messageId`, async (req, res) =>
-    retrieveMesasge(req.params.threadId, req.params.messageId)
+    retrieveMessage(req.params.threadId, req.params.messageId)
   )
 
   app.post(`/threads/:threadId/messages`, async (req, res) =>

core/src/node/api/restful/helper/builder.ts

@@ -125,7 +125,7 @@ export const getMessages = async (threadId: string): Promise<ThreadMessage[]> =>
   }
 }
 
-export const retrieveMesasge = async (threadId: string, messageId: string) => {
+export const retrieveMessage = async (threadId: string, messageId: string) => {
   const messages = await getMessages(threadId)
   const filteredMessages = messages.filter((m) => m.id === messageId)
   if (!filteredMessages || filteredMessages.length === 0) {
@@ -318,13 +318,6 @@ export const chatCompletions = async (request: any, reply: any) => {
     apiUrl = engineConfiguration.full_url
   }
 
-  reply.raw.writeHead(200, {
-    'Content-Type': 'text/event-stream',
-    'Cache-Control': 'no-cache',
-    'Connection': 'keep-alive',
-    'Access-Control-Allow-Origin': '*',
-  })
-
   const headers: Record<string, any> = {
     'Content-Type': 'application/json',
   }
@@ -343,8 +336,14 @@ export const chatCompletions = async (request: any, reply: any) => {
   })
   if (response.status !== 200) {
     console.error(response)
-    return
+    reply.code(400).send(response)
   } else {
+    reply.raw.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive',
+      'Access-Control-Allow-Origin': '*',
+    })
     response.body.pipe(reply.raw)
   }
 }

core/src/node/helper/config.ts

@@ -1,8 +1,7 @@
-import { AppConfiguration, SystemResourceInfo } from '../../types'
+import { AppConfiguration } from '../../types'
 import { join } from 'path'
 import fs from 'fs'
 import os from 'os'
-import { log, logServer } from './log'
 import childProcess from 'child_process'
 
 // TODO: move this to core

docker-compose.yml

@@ -42,10 +42,10 @@ services:
       vpcbr:
 
   # app_cpu service for running the CPU version of the application
-  app_cpu:
+  app_cpu_s3fs:
     image: jan:latest
     volumes:
-      - app_data:/app/server/build/jan
+      - app_data_cpu_s3fs:/app/server/build/jan
     build:
       context: .
       dockerfile: Dockerfile
@@ -56,9 +56,10 @@ services:
       S3_BUCKET_NAME: mybucket
       AWS_ENDPOINT: http://10.5.0.2:9000
       AWS_REGION: us-east-1
+      API_BASE_URL: http://localhost:1337
     restart: always
     profiles:
-      - cpu
+      - cpu-s3fs
     ports:
       - "3000:3000"
       - "1337:1337"
@@ -68,7 +69,7 @@ services:
         ipv4_address: 10.5.0.3
 
   # app_gpu service for running the GPU version of the application
-  app_gpu:
+  app_gpu_s3fs:
     deploy:
       resources:
         reservations:
@@ -78,7 +79,7 @@ services:
             capabilities: [gpu]
     image: jan-gpu:latest
     volumes:
-      - app_data:/app/server/build/jan
+      - app_data_gpu_s3fs:/app/server/build/jan
     build:
       context: .
       dockerfile: Dockerfile.gpu
@@ -90,8 +91,9 @@ services:
       S3_BUCKET_NAME: mybucket
       AWS_ENDPOINT: http://10.5.0.2:9000
       AWS_REGION: us-east-1
+      API_BASE_URL: http://localhost:1337
     profiles:
-      - gpu
+      - gpu-s3fs
     ports:
       - "3000:3000"
       - "1337:1337"
@@ -100,9 +102,60 @@ services:
       vpcbr:
         ipv4_address: 10.5.0.4
 
+  app_cpu_fs:
+    image: jan:latest
+    volumes:
+      - app_data_cpu_fs:/app/server/build/jan
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      API_BASE_URL: http://localhost:1337
+    restart: always
+    profiles:
+      - cpu-fs
+    ports:
+      - "3000:3000"
+      - "1337:1337"
+      - "3928:3928"
+    networks:
+      vpcbr:
+        ipv4_address: 10.5.0.5
+
+  # app_gpu service for running the GPU version of the application
+  app_gpu_fs:
+    deploy:
+      resources:
+        reservations:
+          devices:
+          - driver: nvidia
+            count: all
+            capabilities: [gpu]
+    image: jan-gpu:latest
+    volumes:
+      - app_data_gpu_fs:/app/server/build/jan
+    build:
+      context: .
+      dockerfile: Dockerfile.gpu
+    restart: always
+    environment:
+      API_BASE_URL: http://localhost:1337
+    profiles:
+      - gpu-fs
+    ports:
+      - "3000:3000"
+      - "1337:1337"
+      - "3928:3928"
+    networks:
+      vpcbr:
+        ipv4_address: 10.5.0.6
+
 volumes:
   minio_data:
-  app_data:
+  app_data_cpu_s3fs:
+  app_data_gpu_s3fs:
+  app_data_cpu_fs:
+  app_data_gpu_fs:
 
 networks:
   vpcbr:
@@ -113,5 +166,7 @@ networks:
          gateway: 10.5.0.1
 
 # Usage:
-# - Run 'docker-compose --profile cpu up -d' to start the app_cpu service
+# - Run 'docker compose --profile cpu-s3fs up -d' to start the app_cpu service
-# - Run 'docker-compose --profile gpu up -d' to start the app_gpu service
+# - Run 'docker compose --profile gpu-s3fs up -d' to start the app_gpu service
+# - Run 'docker compose --profile cpu-fs up -d' to start the app_cpu service
+# - Run 'docker compose --profile gpu-fs up -d' to start the app_gpu service

docs/docs/guides/04-using-models/02-import-manually.mdx

@@ -29,6 +29,10 @@ In this section, we will show you how to import a GGUF model from [HuggingFace](
 
 > We are fast shipping a UI to make this easier, but it's a bit manual for now. Apologies.
 
+## Import Models Using Absolute Filepath (version 0.4.7)
+
+Starting from version 0.4.7, Jan has introduced the capability to import models using an absolute file path. It allows you to import models from any directory on your computer. Please check the [import models using absolute filepath](../import-models-using-absolute-filepath) guide for more information. 
+
 ## Manually Importing a Downloaded Model (nightly versions and v0.4.4+)
 
 ### 1. Create a Model Folder
@@ -186,7 +190,6 @@ This means that you can easily reconfigure your models, export them, and share y
 
 Edit `model.json` and include the following configurations:
 
-- Ensure the filename must be `model.json`.
 - Ensure the `id` property matches the folder name you created.
 - Ensure the GGUF filename should match the `id` property exactly.
 - Ensure the `source.url` property is the direct binary download link ending in `.gguf`. In HuggingFace, you can find the direct links in the `Files and versions` tab.

docs/docs/guides/04-using-models/03-import-models-using-absolute-filepath.mdx

@@ -0,0 +1,84 @@
+---
+title: Import Models Using Absolute Filepath
+slug: /guides/using-models/import-models-using-absolute-filepath
+description: Guide to import model using absolute filepath in Jan.
+keywords:
+  [
+    Jan AI,
+    Jan,
+    ChatGPT alternative,
+    local AI,
+    private AI,
+    conversational AI,
+    no-subscription fee,
+    large language model,
+    import-models-manually,
+    absolute-filepath,
+  ]
+---
+
+In this guide, we will walk you through the process of importing a model using an absolute filepath in Jan, using our latest model, [Trinity](https://huggingface.co/janhq/trinity-v1-GGUF), as an example.
+
+### 1. Get the Absolute Filepath of the Model
+
+After downloading .gguf model, you can get the absolute filepath of the model file.
+
+### 2. Configure the Model JSON
+
+1. Navigate to the `~/jan/models` folder. 
+2. Create a folder named `<modelname>`, for example, `tinyllama`.
+3. Create a `model.json` file inside the folder, including the following configurations:
+
+- Ensure the `id` property matches the folder name you created.
+- Ensure the `url` property is the direct binary download link ending in `.gguf`. Now, you can use the absolute filepath of the model file.
+- Ensure the `engine` property is set to `nitro`.
+
+```json
+{
+  "sources": [
+    {
+      "filename": "tinyllama.gguf",
+      // highlight-next-line
+      "url": "<absolute-filepath-of-the-model-file>"
+    }
+  ],
+  "id": "tinyllama-1.1b",
+  "object": "model",
+  "name": "(Absolute Path) TinyLlama Chat 1.1B Q4",
+  "version": "1.0",
+  "description": "TinyLlama is a tiny model with only 1.1B. It's a good model for less powerful computers.",
+  "format": "gguf",
+  "settings": {
+    "ctx_len": 4096,
+    "prompt_template": "<|system|>\n{system_message}<|user|>\n{prompt}<|assistant|>",
+    "llama_model_path": "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf"
+  },
+  "parameters": {
+    "temperature": 0.7,
+    "top_p": 0.95,
+    "stream": true,
+    "max_tokens": 2048,
+    "stop": [],
+    "frequency_penalty": 0,
+    "presence_penalty": 0
+  },
+  "metadata": {
+    "author": "TinyLlama",
+    "tags": ["Tiny", "Foundation Model"],
+    "size": 669000000
+  },
+  "engine": "nitro"
+}
+```
+
+:::warning
+
+- If you are using Windows, you need to use double backslashes in the url property, for example: `C:\\Users\\username\\filename.gguf`.
+
+:::
+
+### 3. Start the Model
+
+Restart Jan and navigate to the Hub. Locate your model and click the Use button.
+
+![Demo](assets/03-demo-absolute-filepath.gif)

docs/docs/guides/04-using-models/04-integrate-with-remote-server.mdx

@@ -88,7 +88,7 @@ You can find your API keys in the [OpenAI Platform](https://platform.openai.com/
 
 Restart Jan and navigate to the Hub. Then, select your configured model and start the model.
 
-![image-01](assets/03-openai-platform-configuration.png)
+![image-01](assets/04-openai-platform-configuration.png)
 
 ## Engines with OAI Compatible Configuration
 
@@ -159,7 +159,7 @@ Navigate to the `~/jan/models` folder. Create a folder named `mistral-ins-7b-q4`
 
 Restart Jan and navigate to the Hub. Locate your model and click the Use button.
 
-![image-02](assets/03-oai-compatible-configuration.png)
+![image-02](assets/04-oai-compatible-configuration.png)
 
 ## Assistance and Support
 

docs/docs/guides/05-using-server/01-start-server.md

@@ -1,6 +1,6 @@
 ---
 title: Start Local Server
-slug: /guides/using-server/server
+slug: /guides/using-server/start-server
 description: How to run Jan's built-in API server.
 keywords:
   [

docs/docs/guides/07-integrations/01-integrate-continue.mdx

@@ -35,7 +35,7 @@ To get started with Continue in VS Code, please follow this [guide to install Co
 
 ### 2. Enable Jan API Server
 
-To configure the Continue to use Jan's Local Server, you need to enable Jan API Server with your preferred model, please follow this [guide to enable Jan API Server](../05-using-server/01-server.md)
+To configure the Continue to use Jan's Local Server, you need to enable Jan API Server with your preferred model, please follow this [guide to enable Jan API Server](/guides/using-server/start-server).
 
 ### 3. Configure Continue to Use Jan's Local Server
 

docs/docs/guides/07-integrations/04-integrate-mistral-ai.mdx

@@ -0,0 +1,89 @@
+---
+title: Integrate Mistral AI with Jan
+slug: /guides/integrations/mistral-ai
+description: Guide to integrate Mistral AI with Jan
+keywords:
+  [
+    Jan AI,
+    Jan,
+    ChatGPT alternative,
+    local AI,
+    private AI,
+    conversational AI,
+    no-subscription fee,
+    large language model,
+    Mistral integration,
+  ]
+---
+
+## Quick Introduction
+
+[Mistral AI](https://docs.mistral.ai/) currently provides two ways of accessing their Large Language Models (LLM) - via their API or via open source models available on Hugging Face.  In this guide, we will show you how to integrate Mistral AI with Jan using the API method.
+
+## Steps to Integrate Mistral AI with Jan
+
+### 1. Configure Mistral API key
+
+You can find your API keys in the [Mistral API Key](https://console.mistral.ai/user/api-keys/) and set the Mistral AI API key in `~/jan/engines/openai.json` file.
+
+```json title="~/jan/engines/openai.json"
+{
+  // highlight-start
+  "full_url": "https://api.mistral.ai/v1/chat/completions",
+  "api_key": "<your-mistral-ai-api-key>"
+  // highlight-end
+}
+```
+
+### 2. Modify a Model JSON
+
+Navigate to the `~/jan/models` folder. Create a folder named `<mistral-modelname>`, for example, `mistral-tiny` and create a `model.json` file inside the folder including the following configurations:
+
+- Ensure the filename must be `model.json`.
+- Ensure the `id` property is set to the model id from Mistral AI.
+- Ensure the `format` property is set to `api`.
+- Ensure the `engine` property is set to `openai`.
+- Ensure the `state` property is set to `ready`.
+
+```json title="~/jan/models/mistral-tiny/model.json"
+{
+  "sources": [
+    {
+      "filename": "mistral-tiny",
+      "url": "https://mistral.ai/"
+    }
+  ],
+  "id": "mistral-tiny",
+  "object": "model",
+  "name": "Mistral-7B-v0.2 (Tiny Endpoint)",
+  "version": "1.0",
+  "description": "Currently powered by Mistral-7B-v0.2, a better fine-tuning of the initial Mistral-7B released, inspired by the fantastic work of the community.",
+  // highlight-next-line
+  "format": "api",
+  "settings": {},
+  "parameters": {},
+  "metadata": {
+    "author": "Mistral AI",
+    "tags": ["General", "Big Context Length"]
+  },
+  // highlight-start
+  "engine": "openai"
+  // highlight-end
+}
+```
+
+:::tip
+
+Mistral AI provides different endpoints. Please check out their [endpoint documentation](https://docs.mistral.ai/platform/endpoints/) to find the one that suits your needs. In this example, we will use the `mistral-tiny` model.
+
+:::
+
+### 3. Start the Model
+
+Restart Jan and navigate to the Hub. Locate your model and click the Use button.
+
+![Mitral AI Tiny Model](assets/04-mistral-ai-tiny-hub.png)
+
+### 4. Try Out the Integration of Jan and Mistral AI
+
+![Mistral AI Integration Demo](assets/04-mistral-ai-integration-demo.gif)

docs/docs/guides/07-integrations/05-integrate-lmstudio.mdx

@@ -0,0 +1,184 @@
+---
+title: Integrate LM Studio with Jan
+slug: /guides/integrations/lmstudio
+description: Guide to integrate LM Studio with Jan
+keywords:
+  [
+    Jan AI,
+    Jan,
+    ChatGPT alternative,
+    local AI,
+    private AI,
+    conversational AI,
+    no-subscription fee,
+    large language model,
+    LM Studio integration,
+  ]
+---
+
+## Quick Introduction
+
+With [LM Studio](https://lmstudio.ai/), you can discover, download, and run local Large Language Models (LLMs). In this guide, we will show you how to integrate and use your current models on LM Studio with Jan using 2 methods.  The first method is integrating LM Studio server with Jan UI. The second method is migrating your downloaded model from LM Studio to Jan. We will use the [Phi 2 - GGUF](https://huggingface.co/TheBloke/phi-2-GGUF) model on Hugging Face as an example.
+
+## Steps to Integrate LM Studio Server with Jan UI
+
+### 1. Start the LM Studio Server
+
+1. Navigate to the `Local Inference Server` on the LM Studio application.  
+2. Select the model you want to use. 
+3. Start the server after configuring the server port and options. 
+
+![LM Studio Server](assets/05-setting-lmstudio-server.gif)
+
+<br></br>
+
+Modify the `openai.json` file in the `~/jan/engines` folder to include the full URL of the LM Studio server.
+
+```json title="~/jan/engines/openai.json"
+{
+  "full_url": "http://localhost:<port>/v1/chat/completions"
+}
+```
+
+:::tip
+
+- Replace `<port>` with the port number you set in the LM Studio server. The default port is `1234`.
+
+:::
+
+### 2. Modify a Model JSON
+
+Navigate to the `~/jan/models` folder. Create a folder named `<lmstudio-modelname>`, for example, `lmstudio-phi-2` and create a `model.json` file inside the folder including the following configurations:
+
+- Set the `format` property to `api`.
+- Set the `engine` property to `openai`.
+- Set the `state` property to `ready`.
+
+```json title="~/jan/models/lmstudio-phi-2/model.json"
+{
+  "sources": [
+    {
+      "filename": "phi-2-GGUF",
+      "url": "https://huggingface.co/TheBloke/phi-2-GGUF"
+    }
+  ],
+  "id": "lmstudio-phi-2",
+  "object": "model",
+  "name": "LM Studio - Phi 2 - GGUF",
+  "version": "1.0",
+  "description": "TheBloke/phi-2-GGUF",
+  // highlight-next-line
+  "format": "api",
+  "settings": {},
+  "parameters": {},
+  "metadata": {
+    "author": "Microsoft",
+    "tags": ["General", "Big Context Length"]
+  },
+  // highlight-start
+  "engine": "openai"
+  // highlight-end
+}
+```
+
+### 3. Start the Model
+
+1. Restart Jan and navigate to the **Hub**. 
+2. Locate your model and click the **Use** button.
+
+![LM Studio Model](assets/05-lmstudio-run.png)
+
+### 4. Try Out the Integration of Jan and LM Studio
+
+![LM Studio Integration Demo](assets/05-lmstudio-integration-demo.gif)
+
+## Steps to Migrate Your Downloaded Model from LM Studio to Jan (version 0.4.6 and older)
+
+### 1. Migrate Your Downloaded Model
+
+1. Navigate to `My Models` in the LM Studio application and reveal the model folder.
+
+![Reveal-model-folder-lmstudio](assets/05-reveal-model-folder-lmstudio.gif)
+
+2. Copy the model folder that you want to migrate to `~/jan/models` folder.
+
+3. Ensure the folder name property is the same as the model name of `.gguf` filename by changing the folder name if necessary. For example, in this case, we changed foldername from `TheBloke` to `phi-2.Q4_K_S`.
+
+### 2. Start the Model
+
+1. Restart Jan and navigate to the **Hub**. Jan will automatically detect the model and display it in the **Hub**. 
+2. Locate your model and click the **Use** button to try the migrating model.
+
+![Demo](assets/05-demo-migrating-model.gif)
+
+## Steps to Pointing to the Downloaded Model of LM Studio from Jan (version 0.4.7+)
+
+Starting from version 0.4.7, Jan supports importing models using an absolute filepath, so you can directly use the model from the LM Studio folder.
+
+### 1. Reveal the Model Absolute Path
+
+Navigate to `My Models` in the LM Studio application and reveal the model folder. Then, you can get the absolute path of your model.
+
+![Reveal-model-folder-lmstudio](assets/05-reveal-model-folder-lmstudio.gif)
+
+### 2. Modify a Model JSON
+
+Navigate to the `~/jan/models` folder. Create a folder named `<modelname>`, for example, `phi-2.Q4_K_S` and create a `model.json` file inside the folder including the following configurations:
+
+- Ensure the `id` property matches the folder name you created.
+- Ensure the `url` property is the direct binary download link ending in `.gguf`. Now, you can use the absolute filepath of the model file. In this example, the absolute filepath is `/Users/<username>/.cache/lm-studio/models/TheBloke/phi-2-GGUF/phi-2.Q4_K_S.gguf`.
+- Ensure the `engine` property is set to `nitro`.
+
+```json
+{
+  "object": "model",
+  "version": 1,
+  "format": "gguf",
+  "sources": [
+    {
+      "filename": "phi-2.Q4_K_S.gguf",
+      "url": "<absolute-path-of-model-file>"
+    }
+  ],
+  "id": "phi-2.Q4_K_S",
+  "name": "phi-2.Q4_K_S",
+  "created": 1708308111506,
+  "description": "phi-2.Q4_K_S - user self import model",
+  "settings": {
+    "ctx_len": 4096,
+    "embedding": false,
+    "prompt_template": "{system_message}\n### Instruction: {prompt}\n### Response:",
+    "llama_model_path": "phi-2.Q4_K_S.gguf"
+  },
+  "parameters": {
+    "temperature": 0.7,
+    "top_p": 0.95,
+    "stream": true,
+    "max_tokens": 2048,
+    "stop": ["<endofstring>"],
+    "frequency_penalty": 0,
+    "presence_penalty": 0
+  },
+  "metadata": {
+    "size": 1615568736,
+    "author": "User",
+    "tags": []
+  },
+  "engine": "nitro"
+}
+```
+
+:::warning
+
+- If you are using Windows, you need to use double backslashes in the url property, for example: `C:\\Users\\username\\filename.gguf`.
+
+:::
+
+
+### 3. Start the Model
+
+1. Restart Jan and navigate to the **Hub**. 
+2. Jan will automatically detect the model and display it in the **Hub**. 
+3. Locate your model and click the **Use** button to try the migrating model.  
+
+![Demo](assets/05-demo-pointing-model.gif)

docs/docs/guides/07-integrations/06-integrate-ollama.mdx

@@ -0,0 +1,90 @@
+---
+title: Integrate Ollama with Jan
+slug: /guides/integrations/ollama
+description: Guide to integrate Ollama with Jan
+keywords:
+  [
+    Jan AI,
+    Jan,
+    ChatGPT alternative,
+    local AI,
+    private AI,
+    conversational AI,
+    no-subscription fee,
+    large language model,
+    Ollama integration,
+  ]
+---
+
+## Quick Introduction
+
+With [Ollama](https://ollama.com/), you can run large language models locally. In this guide, we will show you how to integrate and use your current models on Ollama with Jan using 2 methods.  The first method is integrating Ollama server with Jan UI. The second method is migrating your downloaded model from Ollama to Jan. We will use the [llama2](https://ollama.com/library/llama2) model as an example.
+
+## Steps to Integrate Ollama Server with Jan UI
+
+### 1. Start the Ollama Server
+
+1. Select the model you want to use from the [Ollama library](https://ollama.com/library). 
+2. Run your model by using the following command:
+  
+```bash
+ollama run <model-name>
+```
+
+3. According to the [Ollama documentation on OpenAI compatibility](https://github.com/ollama/ollama/blob/main/docs/openai.md), you can use the `http://localhost:11434/v1/chat/completions` endpoint to interact with the Ollama server. Thus, modify the `openai.json` file in the `~/jan/engines` folder to include the full URL of the Ollama server.
+
+```json title="~/jan/engines/openai.json"
+{
+  "full_url": "http://localhost:11434/v1/chat/completions"
+}
+```
+
+### 2. Modify a Model JSON
+
+1. Navigate to the `~/jan/models` folder. 
+2. Create a folder named `<ollam-modelname>`, for example, `lmstudio-phi-2`.
+3. Create a `model.json` file inside the folder including the following configurations:
+
+- Set the `id` property to the model name as Ollama model name.
+- Set the `format` property to `api`.
+- Set the `engine` property to `openai`.
+- Set the `state` property to `ready`.
+
+```json title="~/jan/models/llama2/model.json"
+{
+  "sources": [
+    {
+      "filename": "llama2",
+      "url": "https://ollama.com/library/llama2"
+    }
+  ],
+  // highlight-next-line
+  "id": "llama2",
+  "object": "model",
+  "name": "Ollama - Llama2",
+  "version": "1.0",
+  "description": "Llama 2 is a collection of foundation language models ranging from 7B to 70B parameters.",
+  // highlight-next-line
+  "format": "api",
+  "settings": {},
+  "parameters": {},
+  "metadata": {
+    "author": "Meta",
+    "tags": ["General", "Big Context Length"]
+  },
+  // highlight-next-line
+  "engine": "openai"
+}
+```
+
+### 3. Start the Model
+
+1. Restart Jan and navigate to the **Hub**.
+2. Locate your model and click the **Use** button.
+
+![Ollama Model](assets/06-ollama-run.png)
+
+### 4. Try Out the Integration of Jan and Ollama
+
+![Ollama Integration Demo](assets/06-ollama-integration-demo.gif)
+

docs/docs/guides/09-advanced-settings/README.mdx

@@ -16,6 +16,50 @@ keywords:
   ]
 ---
 
-import DocCardList from "@theme/DocCardList";
+This guide will show you how to use the advanced settings in Jan.
 
-<DocCardList />
+## Keyboard Shortcuts
+
+Keyboard shortcuts are a great way to speed up your workflow. Here are some of the keyboard shortcuts that you can use in Jan.
+
+| Combination     | Description                                        |
+| --------------- | -------------------------------------------------- |
+| `⌘ E`           | Show list your models                              |
+| `⌘ K`           | Show list navigation pages                         |
+| `⌘ B`           | Toggle collapsible left panel                      |
+| `⌘ ,`           | Navigate to setting page                           |
+| `Enter`         | Send a message                                     |
+| `Shift + Enter` | Insert new line in input box                       |
+| `Arrow Up`      | Navigate to previous option (within search dialog) |
+| `Arrow Down`    | Navigate to next option (within search dialog)     |
+
+<br></br>
+
+:::note
+`⌘` is the command key on macOS, and `Ctrl` on Windows.
+:::
+
+## Experimental Mode
+
+Experimental mode allows you to enable experimental features that may be unstable tested.
+
+## Jan Data Folder
+
+The Jan data folder is the location where messages, model configurations, and other user data are placed. You can change the location of the data folder to a different location.
+
+![00-changing-folder](./assets/00-changing-folder.gif)
+
+## HTTPS Proxy & Ignore SSL Certificate
+
+HTTPS Proxy allows you to use a proxy server to connect to the internet. You can also ignore SSL certificates if you are using a self-signed certificate.
+Please check out the guide on [how to set up your own HTTPS proxy server and configure Jan to use it](../advanced-settings/https-proxy) for more information.
+
+## Clear Logs
+
+Clear logs will remove all logs from the Jan application.
+
+## Reset To Factory Default
+
+Reset the application to its original state, deleting all your usage data, including model customizations and conversation history. This action is irreversible and recommended only if the application is in a corrupted state.
+
+![00-reset-factory-settings](./assets/00-reset-factory-settings.gif)

electron/handlers/update.ts

@@ -11,7 +11,8 @@ export function handleAppUpdates() {
   /* New Update Available */
   autoUpdater.on('update-available', async (_info: any) => {
     const action = await dialog.showMessageBox({
-      message: `Update available. Do you want to download the latest update?`,
+      title: 'Update Available',
+      message: 'Would you like to download and install it now?',
       buttons: ['Download', 'Later'],
     })
     if (action.response === 0) await autoUpdater.downloadUpdate()

electron/main.ts

@@ -83,6 +83,7 @@ function createMainWindow() {
 
   /* Enable dev tools for development */
   if (!app.isPackaged) mainWindow.webContents.openDevTools()
+  log(`Version: ${app.getVersion()}`)
 }
 
 /**

electron/package.json

@@ -4,6 +4,7 @@
   "main": "./build/main.js",
   "author": "Jan <service@jan.ai>",
   "license": "MIT",
+  "productName": "Jan",
   "homepage": "https://github.com/janhq/jan/tree/main/electron",
   "description": "Use offline LLMs with your own data. Run open source models like Llama2 or Falcon on your internal computers/servers.",
   "build": {

electron/utils/menu.ts

@@ -1,18 +1,41 @@
 // @ts-nocheck
-import { app, Menu, shell } from 'electron'
+import { app, Menu, shell, dialog } from 'electron'
-const isMac = process.platform === 'darwin'
 import { autoUpdater } from 'electron-updater'
+import { log } from '@janhq/core/node'
+const isMac = process.platform === 'darwin'
 
 const template: (Electron.MenuItemConstructorOptions | Electron.MenuItem)[] = [
   {
     label: app.name,
     submenu: [
-      { role: 'about' },
+      {
+        label: `About ${app.name}`,
+        click: () =>
+          dialog.showMessageBox({
+            title: `Jan`,
+            message: `Jan Version v${app.getVersion()}\n\nCopyright © 2024 Jan`,
+          }),
+      },
       {
         label: 'Check for Updates...',
         click: () =>
           // Check for updates and notify user if there are any
-          autoUpdater.checkForUpdatesAndNotify(),
+          autoUpdater
+            .checkForUpdatesAndNotify()
+            .then((updateCheckResult) => {
+              if (
+                !updateCheckResult?.updateInfo ||
+                updateCheckResult?.updateInfo.version === app.getVersion()
+              ) {
+                dialog.showMessageBox({
+                  message: `No updates available.`,
+                })
+                return
+              }
+            })
+            .catch((error) => {
+              log('Error checking for updates:' + JSON.stringify(error))
+            }),
       },
       { type: 'separator' },
       { role: 'services' },

extensions/inference-nitro-extension/download.bat

@@ -1,3 +1,3 @@
 @echo off
 set /p NITRO_VERSION=<./bin/version.txt
-.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu
+.\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/win-cuda-12-0 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/win-cuda-11-7 && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64.tar.gz -e --strip 1 -o ./bin/win-cpu && .\node_modules\.bin\download https://github.com/janhq/nitro/releases/download/v%NITRO_VERSION%/nitro-%NITRO_VERSION%-win-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/win-vulkan && .\node_modules\.bin\download https://delta.jan.ai/vulkaninfoSDK.exe -o ./bin

extensions/inference-nitro-extension/package.json

@@ -8,7 +8,7 @@
   "license": "AGPL-3.0",
   "scripts": {
     "build": "tsc --module commonjs && rollup -c rollup.config.ts",
-    "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro",
+    "downloadnitro:linux": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64.tar.gz -e --strip 1 -o ./bin/linux-cpu && chmod +x ./bin/linux-cpu/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-12-0.tar.gz -e --strip 1 -o ./bin/linux-cuda-12-0 && chmod +x ./bin/linux-cuda-12-0/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-cuda-11-7.tar.gz -e --strip 1 -o ./bin/linux-cuda-11-7 && chmod +x ./bin/linux-cuda-11-7/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-linux-amd64-vulkan.tar.gz -e --strip 1 -o ./bin/linux-vulkan && chmod +x ./bin/linux-vulkan/nitro && download https://delta.jan.ai/vulkaninfo -o ./bin && chmod +x ./bin/vulkaninfo",
     "downloadnitro:darwin": "NITRO_VERSION=$(cat ./bin/version.txt) && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-arm64.tar.gz -e --strip 1 -o ./bin/mac-arm64 && chmod +x ./bin/mac-arm64/nitro && download https://github.com/janhq/nitro/releases/download/v${NITRO_VERSION}/nitro-${NITRO_VERSION}-mac-amd64.tar.gz -e --strip 1 -o ./bin/mac-x64 && chmod +x ./bin/mac-x64/nitro",
     "downloadnitro:win32": "download.bat",
     "downloadnitro": "run-script-os",

extensions/inference-nitro-extension/src/index.ts

@@ -88,7 +88,7 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
     const electronApi = window?.electronAPI
     this.inferenceUrl = INFERENCE_URL
     if (!electronApi) {
-      this.inferenceUrl = JAN_SERVER_INFERENCE_URL
+      this.inferenceUrl = `${window.core?.api?.baseApiUrl}/v1/chat/completions`
     }
     console.debug('Inference url: ', this.inferenceUrl)
 
@@ -154,7 +154,10 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
     })
 
     if (nitroInitResult?.error) {
-      events.emit(ModelEvent.OnModelFail, model)
+      events.emit(ModelEvent.OnModelFail, {
+        ...model,
+        error: nitroInitResult.error,
+      })
       return
     }
 

extensions/inference-nitro-extension/src/node/accelerator.ts

@@ -1,10 +1,11 @@
 import { writeFileSync, existsSync, readFileSync } from 'fs'
-import { exec } from 'child_process'
+import { exec, spawn } from 'child_process'
 import path from 'path'
-import { getJanDataFolderPath } from '@janhq/core/node'
+import { getJanDataFolderPath, log } from '@janhq/core/node'
 
 /**
  * Default GPU settings
+ * TODO: This needs to be refactored to support multiple accelerators
  **/
 const DEFALT_SETTINGS = {
   notify: true,
@@ -21,12 +22,17 @@ const DEFALT_SETTINGS = {
   gpu_highest_vram: '',
   gpus_in_use: [],
   is_initial: true,
+  // TODO: This needs to be set based on user toggle in settings
+  vulkan: {
+    enabled: true,
+    gpu_in_use: '1',
+  },
 }
 
 /**
  * Path to the settings file
  **/
-export const NVIDIA_INFO_FILE = path.join(
+export const GPU_INFO_FILE = path.join(
   getJanDataFolderPath(),
   'settings',
   'settings.json'
@@ -52,10 +58,10 @@ export async function updateNvidiaInfo() {
   if (process.platform !== 'darwin') {
     let data
     try {
-      data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
+      data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
     } catch (error) {
       data = DEFALT_SETTINGS
-      writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
+      writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2))
     }
     updateNvidiaDriverInfo()
     updateGpuInfo()
@@ -79,7 +85,7 @@ export async function updateNvidiaDriverInfo(): Promise<void> {
   exec(
     'nvidia-smi --query-gpu=driver_version --format=csv,noheader',
     (error, stdout) => {
-      let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
+      let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
 
       if (!error) {
         const firstLine = stdout.split('\n')[0].trim()
@@ -89,7 +95,7 @@ export async function updateNvidiaDriverInfo(): Promise<void> {
         data['nvidia_driver'].exist = false
       }
 
-      writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
+      writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2))
       Promise.resolve()
     }
   )
@@ -158,42 +164,77 @@ export function updateCudaExistence(
  * Get GPU information
  */
 export async function updateGpuInfo(): Promise<void> {
-  exec(
+  let data = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
-    'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits',
-    (error, stdout) => {
-      let data = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
 
-      if (!error) {
+  // Cuda
-        // Get GPU info and gpu has higher memory first
+  if (data['vulkan'] === true) {
-        let highestVram = 0
+    // Vulkan
-        let highestVramId = '0'
+    exec(
-        let gpus = stdout
+      process.platform === 'win32'
-          .trim()
+        ? `${__dirname}\\..\\bin\\vulkaninfoSDK.exe --summary`
-          .split('\n')
+        : `${__dirname}/../bin/vulkaninfo --summary`,
-          .map((line) => {
+      (error, stdout) => {
-            let [id, vram, name] = line.split(', ')
+        if (!error) {
-            vram = vram.replace(/\r/g, '')
+          const output = stdout.toString()
-            if (parseFloat(vram) > highestVram) {
+          log(output)
-              highestVram = parseFloat(vram)
+          const gpuRegex = /GPU(\d+):(?:[\s\S]*?)deviceName\s*=\s*(.*)/g
-              highestVramId = id
-            }
-            return { id, vram, name }
-          })
 
-        data.gpus = gpus
+          let gpus = []
-        data.gpu_highest_vram = highestVramId
+          let match
-      } else {
+          while ((match = gpuRegex.exec(output)) !== null) {
-        data.gpus = []
+            const id = match[1]
-        data.gpu_highest_vram = ''
+            const name = match[2]
+            gpus.push({ id, vram: 0, name })
+          }
+          data.gpus = gpus
+
+          if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) {
+            data.gpus_in_use = [data.gpus.length > 1 ? '1' : '0']
+          }
+
+          data = updateCudaExistence(data)
+          writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2))
+        }
+        Promise.resolve()
       }
+    )
+  } else {
+    exec(
+      'nvidia-smi --query-gpu=index,memory.total,name --format=csv,noheader,nounits',
+      (error, stdout) => {
+        if (!error) {
+          log(stdout)
+          // Get GPU info and gpu has higher memory first
+          let highestVram = 0
+          let highestVramId = '0'
+          let gpus = stdout
+            .trim()
+            .split('\n')
+            .map((line) => {
+              let [id, vram, name] = line.split(', ')
+              vram = vram.replace(/\r/g, '')
+              if (parseFloat(vram) > highestVram) {
+                highestVram = parseFloat(vram)
+                highestVramId = id
+              }
+              return { id, vram, name }
+            })
 
-      if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) {
+          data.gpus = gpus
-        data.gpus_in_use = [data['gpu_highest_vram']]
+          data.gpu_highest_vram = highestVramId
+        } else {
+          data.gpus = []
+          data.gpu_highest_vram = ''
+        }
+
+        if (!data['gpus_in_use'] || data['gpus_in_use'].length === 0) {
+          data.gpus_in_use = [data['gpu_highest_vram']]
+        }
+
+        data = updateCudaExistence(data)
+        writeFileSync(GPU_INFO_FILE, JSON.stringify(data, null, 2))
+        Promise.resolve()
       }
-
+    )
-      data = updateCudaExistence(data)
+  }
-      writeFileSync(NVIDIA_INFO_FILE, JSON.stringify(data, null, 2))
-      Promise.resolve()
-    }
-  )
 }

extensions/inference-nitro-extension/src/node/execute.ts

@@ -1,10 +1,11 @@
 import { readFileSync } from 'fs'
 import * as path from 'path'
-import { NVIDIA_INFO_FILE } from './nvidia'
+import { GPU_INFO_FILE } from './accelerator'
 
 export interface NitroExecutableOptions {
   executablePath: string
   cudaVisibleDevices: string
+  vkVisibleDevices: string
 }
 /**
  * Find which executable file to run based on the current platform.
@@ -13,24 +14,30 @@ export interface NitroExecutableOptions {
 export const executableNitroFile = (): NitroExecutableOptions => {
   let binaryFolder = path.join(__dirname, '..', 'bin') // Current directory by default
   let cudaVisibleDevices = ''
+  let vkVisibleDevices = ''
   let binaryName = 'nitro'
   /**
    * The binary folder is different for each platform.
    */
   if (process.platform === 'win32') {
     /**
-     *  For Windows: win-cpu, win-cuda-11-7, win-cuda-12-0
+     *  For Windows: win-cpu, win-vulkan, win-cuda-11-7, win-cuda-12-0
      */
-    let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
+    let gpuInfo = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
-    if (nvidiaInfo['run_mode'] === 'cpu') {
+    if (gpuInfo['run_mode'] === 'cpu') {
       binaryFolder = path.join(binaryFolder, 'win-cpu')
     } else {
-      if (nvidiaInfo['cuda'].version === '11') {
+      if (gpuInfo['cuda']?.version === '11') {
         binaryFolder = path.join(binaryFolder, 'win-cuda-11-7')
       } else {
         binaryFolder = path.join(binaryFolder, 'win-cuda-12-0')
       }
-      cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',')
+      cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',')
+    }
+    if (gpuInfo['vulkan'] === true) {
+      binaryFolder = path.join(__dirname, '..', 'bin')
+      binaryFolder = path.join(binaryFolder, 'win-vulkan')
+      vkVisibleDevices = gpuInfo['gpus_in_use'].toString()
     }
     binaryName = 'nitro.exe'
   } else if (process.platform === 'darwin') {
@@ -44,22 +51,29 @@ export const executableNitroFile = (): NitroExecutableOptions => {
     }
   } else {
     /**
-     *  For Linux: linux-cpu, linux-cuda-11-7, linux-cuda-12-0
+     *  For Linux: linux-cpu, linux-vulkan, linux-cuda-11-7, linux-cuda-12-0
      */
-    let nvidiaInfo = JSON.parse(readFileSync(NVIDIA_INFO_FILE, 'utf-8'))
+    let gpuInfo = JSON.parse(readFileSync(GPU_INFO_FILE, 'utf-8'))
-    if (nvidiaInfo['run_mode'] === 'cpu') {
+    if (gpuInfo['run_mode'] === 'cpu') {
       binaryFolder = path.join(binaryFolder, 'linux-cpu')
     } else {
-      if (nvidiaInfo['cuda'].version === '11') {
+      if (gpuInfo['cuda']?.version === '11') {
         binaryFolder = path.join(binaryFolder, 'linux-cuda-11-7')
       } else {
         binaryFolder = path.join(binaryFolder, 'linux-cuda-12-0')
       }
-      cudaVisibleDevices = nvidiaInfo['gpus_in_use'].join(',')
+      cudaVisibleDevices = gpuInfo['gpus_in_use'].join(',')
+    }
+
+    if (gpuInfo['vulkan'] === true) {
+      binaryFolder = path.join(__dirname, '..', 'bin')
+      binaryFolder = path.join(binaryFolder, 'win-vulkan')
+      vkVisibleDevices = gpuInfo['gpus_in_use'].toString()
     }
   }
   return {
     executablePath: path.join(binaryFolder, binaryName),
     cudaVisibleDevices,
+    vkVisibleDevices,
   }
 }

extensions/inference-nitro-extension/src/node/index.ts

@@ -4,7 +4,7 @@ import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
 import tcpPortUsed from 'tcp-port-used'
 import fetchRT from 'fetch-retry'
 import { log, getSystemResourceInfo } from '@janhq/core/node'
-import { getNitroProcessInfo, updateNvidiaInfo } from './nvidia'
+import { getNitroProcessInfo, updateNvidiaInfo } from './accelerator'
 import {
   Model,
   InferenceEngine,
@@ -310,9 +310,15 @@ async function killSubprocess(): Promise<void> {
       subprocess?.kill()
       subprocess = undefined
     })
-    .catch(() => {})
+    .catch(() => {}) // Do nothing with this attempt
     .then(() => tcpPortUsed.waitUntilFree(PORT, 300, 5000))
     .then(() => log(`[NITRO]::Debug: Nitro process is terminated`))
+    .catch((err) => {
+      log(
+        `[NITRO]::Debug: Could not kill running process on port ${PORT}. Might be another process running on the same port? ${err}`
+      )
+      throw 'PORT_NOT_AVAILABLE'
+    })
 }
 
 /**
@@ -339,6 +345,10 @@ function spawnNitroProcess(): Promise<any> {
         env: {
           ...process.env,
           CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices,
+          // Vulkan - Support 1 device at a time for now
+          ...(executableOptions.vkVisibleDevices?.length > 0 && {
+            GGML_VULKAN_DEVICE: executableOptions.vkVisibleDevices[0],
+          }),
         },
       }
     )

extensions/model-extension/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@janhq/model-extension",
-  "version": "1.0.23",
+  "version": "1.0.25",
   "description": "Model Management Extension provides model exploration and seamless downloads",
   "main": "dist/index.js",
   "module": "dist/module.js",

extensions/monitoring-extension/src/module.ts

@@ -37,10 +37,10 @@ const getCurrentLoad = () =>
       }
       if (data.run_mode === 'gpu' && data.gpus_in_use.length > 0) {
         const gpuIds = data['gpus_in_use'].join(',')
-        if (gpuIds !== '') {
+        if (gpuIds !== '' && data['vulkan'] !== true) {
           exec(
             `nvidia-smi --query-gpu=index,name,temperature.gpu,utilization.gpu,memory.total,memory.free,utilization.memory --format=csv,noheader,nounits --id=${gpuIds}`,
-            (error, stdout, stderr) => {
+            (error, stdout, _) => {
               if (error) {
                 console.error(`exec error: ${error}`)
                 reject(error)

models/dolphin-phi-2/model.json

@@ -0,0 +1,32 @@
+{
+    "sources": [
+      {
+        "url": "https://huggingface.co/TheBloke/dolphin-2_6-phi-2-GGUF/resolve/main/dolphin-2_6-phi-2.Q8_0.gguf",
+        "filename": "dolphin-2_6-phi-2.Q8_0.gguf"
+      }
+    ],
+    "id": "dolphin-phi-2",
+    "object": "model",
+    "name": "Dolphin Phi-2 2.7B Q8",
+    "version": "1.0",
+    "description": "Dolphin Phi-2 is a 2.7B model, fine-tuned for chat, excelling in common sense and logical reasoning benchmarks.",
+    "format": "gguf",
+    "settings": {
+      "ctx_len": 4096,
+      "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
+      "llama_model_path": "dolphin-2_6-phi-2.Q8_0.gguf"
+    },
+    "parameters": {
+      "max_tokens": 4096,
+      "stop": ["<|im_end|>"]
+    },
+    "metadata": {
+      "author": "Cognitive Computations, Microsoft",
+      "tags": [
+        "3B",
+        "Finetuned"
+      ],
+      "size": 2960000000
+    },
+    "engine": "nitro"
+  }

models/llamacorn-1.1b/model.json

@@ -0,0 +1,37 @@
+{
+    "sources": [
+        {
+            "url":"https://huggingface.co/janhq/llamacorn-1.1b-chat-GGUF/resolve/main/llamacorn-1.1b-chat.Q8_0.gguf",
+            "filename": "llamacorn-1.1b-chat.Q8_0.gguf"
+        }
+    ],
+    "id": "llamacorn-1.1b",
+    "object": "model",
+    "name": "LlamaCorn 1.1B Q8",
+    "version": "1.0",
+    "description": "LlamaCorn is designed to improve chat functionality from TinyLlama.",
+    "format": "gguf",
+    "settings": {
+      "ctx_len": 2048,
+      "prompt_template": "<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant",
+      "llama_model_path": "llamacorn-1.1b-chat.Q8_0.gguf"
+    },
+    "parameters": {
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true,
+      "max_tokens": 2048,
+      "stop": [],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
+    },
+    "metadata": {
+      "author": "Jan",
+      "tags": [
+        "Tiny",
+        "Finetuned"
+      ],
+      "size": 1170000000
+    },
+    "engine": "nitro"
+  }

models/mistral-ins-7b-q4/model.json

@@ -29,7 +29,7 @@
     "author": "MistralAI, The Bloke",
     "tags": ["Featured", "7B", "Foundational Model"],
     "size": 4370000000,
-    "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/mistral-ins-7b-q4/cover.png"
+    "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/mistral-ins-7b-q4/cover.png"
   },
   "engine": "nitro"
 }

models/openchat-3.5-7b/model.json

@@ -1,8 +1,8 @@
 {
   "sources": [
     {
-      "filename": "openchat-3.5-1210.Q4_K_M.gguf",
+      "filename": "openchat-3.5-0106.Q4_K_M.gguf",
-      "url": "https://huggingface.co/TheBloke/openchat-3.5-1210-GGUF/resolve/main/openchat-3.5-1210.Q4_K_M.gguf"
+      "url": "https://huggingface.co/TheBloke/openchat-3.5-0106-GGUF/resolve/main/openchat-3.5-0106.Q4_K_M.gguf"
     }
   ],
   "id": "openchat-3.5-7b",
@@ -14,7 +14,7 @@
   "settings": {
     "ctx_len": 4096,
     "prompt_template": "GPT4 Correct User: {prompt}<|end_of_turn|>GPT4 Correct Assistant:",
-    "llama_model_path": "openchat-3.5-1210.Q4_K_M.gguf"
+    "llama_model_path": "openchat-3.5-0106.Q4_K_M.gguf"
   },
   "parameters": {
     "temperature": 0.7,

models/openhermes-neural-7b/model.json

@@ -28,7 +28,7 @@
     "author": "Intel, Jan",
     "tags": ["7B", "Merged", "Featured"],
     "size": 4370000000,
-    "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/openhermes-neural-7b/cover.png"
+    "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/openhermes-neural-7b/cover.png"
   },
   "engine": "nitro"
 }

models/stable-zephyr-3b/model.json

@@ -0,0 +1,34 @@
+{
+    "sources": [
+      {
+        "url": "https://huggingface.co/TheBloke/stablelm-zephyr-3b-GGUF/resolve/main/stablelm-zephyr-3b.Q8_0.gguf",
+        "filename": "stablelm-zephyr-3b.Q8_0.gguf"
+      }
+    ],
+    "id": "stable-zephyr-3b",
+    "object": "model",
+    "name": "Stable Zephyr 3B Q8",
+    "version": "1.0",
+    "description": "StableLM Zephyr 3B is trained for safe and reliable chatting.",
+    "format": "gguf",
+    "settings": {
+      "ctx_len": 4096,
+      "prompt_template": "<|user|>\n{prompt}<|endoftext|>\n<|assistant|>",
+      "llama_model_path": "stablelm-zephyr-3b.Q8_0.gguf"
+    },
+    "parameters": {
+      "temperature": 0.7,
+      "top_p": 0.95,
+      "stream": true,
+      "max_tokens": 4096,
+      "stop": ["<|endoftext|>"],
+      "frequency_penalty": 0,
+      "presence_penalty": 0
+    },
+    "metadata": {
+      "author": "StabilityAI",
+      "tags": ["3B", "Finetuned"],
+      "size": 2970000000
+    },
+    "engine": "nitro"
+  }

models/trinity-v1.2-7b/model.json

@@ -28,7 +28,7 @@
     "author": "Jan",
     "tags": ["7B", "Merged", "Featured"],
     "size": 4370000000,
-    "cover": "https://raw.githubusercontent.com/janhq/jan/main/models/trinity-v1.2-7b/cover.png"
+    "cover": "https://raw.githubusercontent.com/janhq/jan/dev/models/trinity-v1.2-7b/cover.png"
   },
   "engine": "nitro"
 }

models/yarn-mistral-7b/model.json

@@ -1,31 +0,0 @@
-{
-  "sources": [
-    {
-      "url": "https://huggingface.co/TheBloke/Yarn-Mistral-7B-128k-GGUF/resolve/main/yarn-mistral-7b-128k.Q4_K_M.gguf"
-    }
-  ],
-  "id": "yarn-mistral-7b",
-  "object": "model",
-  "name": "Yarn Mistral 7B Q4",
-  "version": "1.0",
-  "description": "Yarn Mistral 7B is a language model for long context and supports a 128k token context window.",
-  "format": "gguf",
-  "settings": {
-    "ctx_len": 4096,
-    "prompt_template": "{prompt}"
-  },
-  "parameters": {
-    "temperature": 0.7,
-    "top_p": 0.95,
-    "stream": true,
-    "max_tokens": 4096,
-    "frequency_penalty": 0,
-    "presence_penalty": 0
-  },
-  "metadata": {
-    "author": "NousResearch, The Bloke",
-    "tags": ["7B", "Finetuned"],
-    "size": 4370000000
-  },
-  "engine": "nitro"
-}

server/helpers/setup.ts

@@ -31,6 +31,32 @@ export async function setup() {
     'utf-8'
   )
 
+  if (!existsSync(join(appDir, 'settings'))) {
+    console.debug('Writing nvidia config file...')
+    mkdirSync(join(appDir, 'settings'))
+    writeFileSync(
+      join(appDir, 'settings', 'settings.json'),
+      JSON.stringify(
+        {
+          notify: true,
+          run_mode: 'cpu',
+          nvidia_driver: {
+            exist: false,
+            version: '',
+          },
+          cuda: {
+            exist: false,
+            version: '',
+          },
+          gpus: [],
+          gpu_highest_vram: '',
+          gpus_in_use: [],
+          is_initial: true,
+        }),
+      'utf-8'
+    )
+  }
+
   /**
    * Install extensions
    */

server/index.ts

@@ -7,6 +7,7 @@ import {
   getJanExtensionsPath,
 } from '@janhq/core/node'
 import { join } from 'path'
+import tcpPortUsed from 'tcp-port-used'
 
 // Load environment variables
 dotenv.config()
@@ -46,6 +47,15 @@ export interface ServerConfig {
  * @param configs - Server configurations
  */
 export const startServer = async (configs?: ServerConfig): Promise<boolean> => {
+  if (configs?.port && configs?.host) {
+    const inUse = await tcpPortUsed.check(Number(configs.port), configs.host)
+    if (inUse) {
+      const errorMessage = `Port ${configs.port} is already in use.`
+      logServer(errorMessage)
+      throw new Error(errorMessage)
+    }
+  }
+
   // Update server settings
   isVerbose = configs?.isVerboseEnabled ?? true
   hostSetting = configs?.host ?? JAN_API_HOST

server/package.json

@@ -27,19 +27,20 @@
     "@npmcli/arborist": "^7.3.1",
     "dotenv": "^16.3.1",
     "fastify": "^4.24.3",
-    "request": "^2.88.2",
     "fetch-retry": "^5.0.6",
-    "tcp-port-used": "^1.0.2",
+    "node-fetch": "2",
-    "request-progress": "^3.0.0"
+    "request": "^2.88.2",
+    "request-progress": "^3.0.0",
+    "tcp-port-used": "^1.0.2"
   },
   "devDependencies": {
     "@types/body-parser": "^1.19.5",
     "@types/npmcli__arborist": "^5.6.4",
+    "@types/tcp-port-used": "^1.0.4",
     "@typescript-eslint/eslint-plugin": "^6.7.3",
     "@typescript-eslint/parser": "^6.7.3",
     "eslint-plugin-react": "^7.33.2",
     "run-script-os": "^1.1.6",
-    "@types/tcp-port-used": "^1.0.4",
     "typescript": "^5.2.2"
   }
 }

web/containers/CardSidebar/index.tsx

@@ -22,6 +22,7 @@ interface Props {
   rightAction?: ReactNode
   title: string
   asChild?: boolean
+  isShow?: boolean
   hideMoreVerticalAction?: boolean
 }
 export default function CardSidebar({
@@ -30,8 +31,9 @@ export default function CardSidebar({
   asChild,
   rightAction,
   hideMoreVerticalAction,
+  isShow,
 }: Props) {
-  const [show, setShow] = useState(false)
+  const [show, setShow] = useState(isShow ?? false)
   const [more, setMore] = useState(false)
   const [menu, setMenu] = useState<HTMLDivElement | null>(null)
   const [toggle, setToggle] = useState<HTMLDivElement | null>(null)
@@ -67,8 +69,8 @@ export default function CardSidebar({
                 show && 'rotate-180'
               )}
             />
+            <span className="font-bold">{title}</span>
           </button>
-          <span className="font-bold">{title}</span>
         </div>
         <div className="flex">
           {rightAction && rightAction}

web/containers/DropdownListSidebar/index.tsx

@@ -195,7 +195,12 @@ const DropdownListSidebar = ({
             </SelectValue>
           </SelectTrigger>
           <SelectPortal>
-            <SelectContent className="right-2  block w-full min-w-[450px] pr-0">
+            <SelectContent
+              className={twMerge(
+                'right-2  block w-full min-w-[450px] pr-0',
+                isTabActive === 1 && '[&_.select-scroll-down-button]:hidden'
+              )}
+            >
               <div className="relative px-2 py-2 dark:bg-secondary/50">
                 <ul className="inline-flex w-full space-x-2 rounded-lg bg-zinc-100 px-1 dark:bg-secondary">
                   {engineOptions.map((name, i) => {

web/containers/Layout/BottomBar/index.tsx

@@ -152,7 +152,7 @@ const BottomBar = () => {
         {gpus.length > 0 && (
           <Tooltip>
             <TooltipTrigger>
-              <div className="flex cursor-pointer items-center">
+              <div className="flex items-center">
                 <SystemItem
                   name={`${gpus.length} GPU `}
                   value={`${calculateUtilization()}% `}

web/containers/Layout/Ribbon/index.tsx

@@ -6,7 +6,7 @@ import {
 } from '@janhq/uikit'
 import { motion as m } from 'framer-motion'
 
-import { useAtom } from 'jotai'
+import { useAtom, useSetAtom } from 'jotai'
 import {
   MessageCircleIcon,
   SettingsIcon,
@@ -23,16 +23,19 @@ import { MainViewState } from '@/constants/screens'
 
 import { useMainViewState } from '@/hooks/useMainViewState'
 
+import { editMessageAtom } from '@/helpers/atoms/ChatMessage.atom'
 import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom'
 
 export default function RibbonNav() {
   const { mainViewState, setMainViewState } = useMainViewState()
   const [serverEnabled] = useAtom(serverEnabledAtom)
+  const setEditMessage = useSetAtom(editMessageAtom)
 
   const onMenuClick = (state: MainViewState) => {
     if (mainViewState === state) return
     if (serverEnabled && state === MainViewState.Thread) return
     setMainViewState(state)
+    setEditMessage('')
   }
 
   const primaryMenus = [

web/containers/OpenAiKeyInput/index.tsx

@@ -30,7 +30,7 @@ const OpenAiKeyInput: React.FC = () => {
   }
 
   return (
-    <div className="mt-4">
+    <div className="my-4">
       <label
         id="thread-title"
         className="mb-2 inline-block font-bold text-gray-600 dark:text-gray-300"

web/containers/Providers/EventHandler.tsx

@@ -114,8 +114,8 @@ export default function EventHandler({ children }: { children: ReactNode }) {
 
   const onModelInitFailed = useCallback(
     (res: any) => {
-      const errorMessage = `${res.error}`
+      const errorMessage = res?.error ?? res
-      console.error('Failed to load model: ' + errorMessage)
+      console.error('Failed to load model: ', errorMessage)
       setStateModel(() => ({
         state: 'start',
         loading: false,

web/context/FeatureToggle.tsx

@@ -4,18 +4,26 @@ interface FeatureToggleContextType {
   experimentalFeature: boolean
   ignoreSSL: boolean
   proxy: string
+  proxyEnabled: boolean
+  vulkanEnabled: boolean
   setExperimentalFeature: (on: boolean) => void
+  setVulkanEnabled: (on: boolean) => void
   setIgnoreSSL: (on: boolean) => void
   setProxy: (value: string) => void
+  setProxyEnabled: (on: boolean) => void
 }
 
 const initialContext: FeatureToggleContextType = {
   experimentalFeature: false,
   ignoreSSL: false,
   proxy: '',
+  proxyEnabled: false,
+  vulkanEnabled: false,
   setExperimentalFeature: () => {},
+  setVulkanEnabled: () => {},
   setIgnoreSSL: () => {},
   setProxy: () => {},
+  setProxyEnabled: () => {},
 }
 
 export const FeatureToggleContext =
@@ -27,10 +35,15 @@ export default function FeatureToggleWrapper({
   children: ReactNode
 }) {
   const EXPERIMENTAL_FEATURE = 'experimentalFeature'
+  const VULKAN_ENABLED = 'vulkanEnabled'
   const IGNORE_SSL = 'ignoreSSLFeature'
   const HTTPS_PROXY_FEATURE = 'httpsProxyFeature'
+  const PROXY_FEATURE_ENABLED = 'proxyFeatureEnabled'
+
   const [experimentalFeature, directSetExperimentalFeature] =
     useState<boolean>(false)
+  const [proxyEnabled, directSetProxyEnabled] = useState<boolean>(false)
+  const [vulkanEnabled, directEnableVulkan] = useState<boolean>(false)
   const [ignoreSSL, directSetIgnoreSSL] = useState<boolean>(false)
   const [proxy, directSetProxy] = useState<string>('')
 
@@ -40,6 +53,9 @@ export default function FeatureToggleWrapper({
     )
     directSetIgnoreSSL(localStorage.getItem(IGNORE_SSL) === 'true')
     directSetProxy(localStorage.getItem(HTTPS_PROXY_FEATURE) ?? '')
+    directSetProxyEnabled(
+      localStorage.getItem(PROXY_FEATURE_ENABLED) === 'true'
+    )
   }, [])
 
   const setExperimentalFeature = (on: boolean) => {
@@ -47,6 +63,11 @@ export default function FeatureToggleWrapper({
     directSetExperimentalFeature(on)
   }
 
+  const setVulkanEnabled = (on: boolean) => {
+    localStorage.setItem(VULKAN_ENABLED, on ? 'true' : 'false')
+    directEnableVulkan(on)
+  }
+
   const setIgnoreSSL = (on: boolean) => {
     localStorage.setItem(IGNORE_SSL, on ? 'true' : 'false')
     directSetIgnoreSSL(on)
@@ -57,15 +78,24 @@ export default function FeatureToggleWrapper({
     directSetProxy(proxy)
   }
 
+  const setProxyEnabled = (on: boolean) => {
+    localStorage.setItem(PROXY_FEATURE_ENABLED, on ? 'true' : 'false')
+    directSetProxyEnabled(on)
+  }
+
   return (
     <FeatureToggleContext.Provider
       value={{
         experimentalFeature,
         ignoreSSL,
         proxy,
+        proxyEnabled,
+        vulkanEnabled,
         setExperimentalFeature,
+        setVulkanEnabled,
         setIgnoreSSL,
         setProxy,
+        setProxyEnabled,
       }}
     >
       {children}

web/extension/ExtensionManager.ts

@@ -83,7 +83,10 @@ export class ExtensionManager {
     // Import class
     const extensionUrl = window.electronAPI
       ? extension.url
-      : extension.url.replace('extension://', `${API_BASE_URL}/extensions/`)
+      : extension.url.replace(
+          'extension://',
+          `${window.core?.api?.baseApiUrl ?? ''}/extensions/`
+        )
     await import(/* webpackIgnore: true */ extensionUrl).then(
       (extensionClass) => {
         // Register class if it has a default export

web/hooks/useDownloadModel.ts

@@ -20,7 +20,7 @@ import { extensionManager } from '@/extension/ExtensionManager'
 import { addDownloadingModelAtom } from '@/helpers/atoms/Model.atom'
 
 export default function useDownloadModel() {
-  const { ignoreSSL, proxy } = useContext(FeatureToggleContext)
+  const { ignoreSSL, proxy, proxyEnabled } = useContext(FeatureToggleContext)
   const setDownloadState = useSetAtom(setDownloadStateAtom)
   const addDownloadingModel = useSetAtom(addDownloadingModelAtom)
 
@@ -64,9 +64,9 @@ export default function useDownloadModel() {
 
       addDownloadingModel(model)
 
-      await localDownloadModel(model, ignoreSSL, proxy)
+      await localDownloadModel(model, ignoreSSL, proxyEnabled ? proxy : '')
     },
-    [ignoreSSL, proxy, addDownloadingModel, setDownloadState]
+    [ignoreSSL, proxy, proxyEnabled, addDownloadingModel, setDownloadState]
   )
 
   const abortModelDownload = useCallback(async (model: Model) => {

web/hooks/useDownloadState.ts

@@ -130,8 +130,8 @@ export const setDownloadStateAtom = atom(
         0
       )
       modelDownloadState.size.transferred = transferredSize
-      modelDownloadState.percent = transferredSize / parentTotalSize
+      modelDownloadState.percent =
-
+        parentTotalSize === 0 ? 0 : transferredSize / parentTotalSize
       currentState[state.modelId] = modelDownloadState
     }
 

web/hooks/useSendChatMessage.ts

@@ -297,6 +297,10 @@ export default function useSendChatMessage() {
     const updatedThread: Thread = {
       ...activeThread,
       updated: timestamp,
+      metadata: {
+        ...(activeThread.metadata ?? {}),
+        lastMessage: prompt,
+      },
     }
 
     // change last update thread when send message

web/hooks/useSettings.ts

@@ -48,17 +48,33 @@ export const useSettings = () => {
     runMode,
     notify,
     gpusInUse,
+    vulkan,
   }: {
     runMode?: string | undefined
     notify?: boolean | undefined
     gpusInUse?: string[] | undefined
+    vulkan?: boolean | undefined
   }) => {
     const settingsFile = await joinPath(['file://settings', 'settings.json'])
     const settings = await readSettings()
     if (runMode != null) settings.run_mode = runMode
     if (notify != null) settings.notify = notify
     if (gpusInUse != null) settings.gpus_in_use = gpusInUse
+    if (vulkan != null) {
+      settings.vulkan = vulkan
+      // GPU enabled, set run_mode to 'gpu'
+      if (settings.vulkan) {
+        settings.run_mode = 'gpu'
+      } else {
+        settings.run_mode = settings.gpus?.length > 0 ? 'gpu' : 'cpu'
+      }
+    }
     await fs.writeFileSync(settingsFile, JSON.stringify(settings))
+
+    // Relaunch to apply settings
+    if (vulkan != null) {
+      window.location.reload()
+    }
   }
 
   return {

web/next.config.js

@@ -6,6 +6,11 @@ const webpack = require('webpack')
 const packageJson = require('./package.json')
 
 const nextConfig = {
+  eslint: {
+    // Warning: This allows production builds to successfully complete even if
+    // your project has ESLint errors.
+    ignoreDuringBuilds: true,
+  },
   output: 'export',
   assetPrefix: '.',
   images: {

web/screens/Chat/ChatInput/index.tsx

@@ -145,7 +145,7 @@ const ChatInput: React.FC = () => {
 
         <Textarea
           className={twMerge(
-            'max-h-[400px] resize-none overflow-y-hidden pr-20',
+            'max-h-[400px] resize-none pr-20',
             fileUpload.length && 'rounded-t-none'
           )}
           style={{ height: '40px' }}

web/screens/Chat/EditChatInput/index.tsx

@@ -1,5 +1,5 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { useEffect, useRef } from 'react'
+import { useEffect, useRef, useState } from 'react'
 
 import {
   ConversationalExtension,
@@ -10,7 +10,17 @@ import {
   events,
 } from '@janhq/core'
 
-import { Textarea, Button } from '@janhq/uikit'
+import {
+  Textarea,
+  Button,
+  Modal,
+  ModalClose,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalPortal,
+  ModalTitle,
+} from '@janhq/uikit'
 import { useAtom, useAtomValue, useSetAtom } from 'jotai'
 
 import { twMerge } from 'tailwind-merge'
@@ -51,6 +61,7 @@ const EditChatInput: React.FC<Props> = ({ message }) => {
   const [isWaitingToSend, setIsWaitingToSend] = useAtom(waitingToSendMessage)
   const textareaRef = useRef<HTMLTextAreaElement>(null)
   const setEditMessage = useSetAtom(editMessageAtom)
+  const [showDialog, setshowDialog] = useState(false)
 
   const onPromptChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
     setEditPrompt(e.target.value)
@@ -152,6 +163,28 @@ const EditChatInput: React.FC<Props> = ({ message }) => {
           Cancel
         </Button>
       </div>
+
+      <Modal open={showDialog} onOpenChange={() => setshowDialog(false)}>
+        <ModalPortal />
+        <ModalContent>
+          <ModalHeader>
+            <ModalTitle>Edit Message</ModalTitle>
+          </ModalHeader>
+          <p className="text-muted-foreground">
+            Do you want to discard the change
+          </p>
+          <ModalFooter>
+            <div className="flex gap-x-2">
+              <ModalClose asChild onClick={() => setshowDialog(false)}>
+                <Button themes="outline">Cancel</Button>
+              </ModalClose>
+              <ModalClose asChild onClick={() => setEditMessage('')}>
+                <Button autoFocus>Yes</Button>
+              </ModalClose>
+            </div>
+          </ModalFooter>
+        </ModalContent>
+      </Modal>
     </div>
   )
 }

web/screens/Chat/ErrorMessage/index.tsx

@@ -7,6 +7,7 @@ import ModalTroubleShooting, {
   modalTroubleShootingAtom,
 } from '@/containers/ModalTroubleShoot'
 
+import { loadModelErrorAtom } from '@/hooks/useActiveModel'
 import useSendChatMessage from '@/hooks/useSendChatMessage'
 
 import { getCurrentChatMessagesAtom } from '@/helpers/atoms/ChatMessage.atom'
@@ -15,6 +16,8 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => {
   const messages = useAtomValue(getCurrentChatMessagesAtom)
   const { resendChatMessage } = useSendChatMessage()
   const setModalTroubleShooting = useSetAtom(modalTroubleShootingAtom)
+  const loadModelError = useAtomValue(loadModelErrorAtom)
+  const PORT_NOT_AVAILABLE = 'PORT_NOT_AVAILABLE'
 
   const regenerateMessage = async () => {
     const lastMessageIndex = messages.length - 1
@@ -23,9 +26,9 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => {
   }
 
   return (
-    <>
+    <div className="mt-10">
       {message.status === MessageStatus.Stopped && (
-        <div key={message.id} className="mt-10 flex flex-col items-center">
+        <div key={message.id} className="flex flex-col items-center">
           <span className="mb-3 text-center text-sm font-medium text-gray-500">
             Oops! The generation was interrupted. Let&apos;s give it another go!
           </span>
@@ -41,25 +44,47 @@ const ErrorMessage = ({ message }: { message: ThreadMessage }) => {
         </div>
       )}
       {message.status === MessageStatus.Error && (
-        <div
+        <>
-          key={message.id}
+          {loadModelError === PORT_NOT_AVAILABLE ? (
-          className="flex flex-col items-center text-center text-sm font-medium text-gray-500"
+            <div
-        >
+              key={message.id}
-          <p>{`Apologies, something’s amiss!`}</p>
+              className="flex flex-col items-center text-center text-sm font-medium text-gray-500 w-full"
-          <p>
-            Jan’s in beta. Access&nbsp;
-            <span
-              className="cursor-pointer text-primary dark:text-blue-400"
-              onClick={() => setModalTroubleShooting(true)}
             >
-              troubleshooting assistance
+              <p className="w-[90%]">
-            </span>
+                Port 3928 is currently unavailable. Check for conflicting apps,
-            &nbsp;now.
+                or access&nbsp;
-          </p>
+                <span
-          <ModalTroubleShooting />
+                  className="cursor-pointer text-primary dark:text-blue-400"
-        </div>
+                  onClick={() => setModalTroubleShooting(true)}
+                >
+                  troubleshooting assistance
+                </span>
+                &nbsp;for further support.
+              </p>
+              <ModalTroubleShooting />
+            </div>
+          ) : (
+            <div
+              key={message.id}
+              className="flex flex-col items-center text-center text-sm font-medium text-gray-500"
+            >
+              <p>{`Apologies, something’s amiss!`}</p>
+              <p>
+                Jan’s in beta. Access&nbsp;
+                <span
+                  className="cursor-pointer text-primary dark:text-blue-400"
+                  onClick={() => setModalTroubleShooting(true)}
+                >
+                  troubleshooting assistance
+                </span>
+                &nbsp;now.
+              </p>
+              <ModalTroubleShooting />
+            </div>
+          )}
+        </>
       )}
-    </>
+    </div>
   )
 }
 export default ErrorMessage

web/screens/Chat/MessageToolbar/index.tsx

@@ -6,7 +6,7 @@ import {
   ConversationalExtension,
   ContentType,
 } from '@janhq/core'
-import { useAtom, useAtomValue, useSetAtom } from 'jotai'
+import { useAtomValue, useSetAtom } from 'jotai'
 import {
   RefreshCcw,
   CopyIcon,
@@ -30,7 +30,7 @@ import { activeThreadAtom } from '@/helpers/atoms/Thread.atom'
 
 const MessageToolbar = ({ message }: { message: ThreadMessage }) => {
   const deleteMessage = useSetAtom(deleteMessageAtom)
-  const [editMessage, setEditMessage] = useAtom(editMessageAtom)
+  const setEditMessage = useSetAtom(editMessageAtom)
   const thread = useAtomValue(activeThreadAtom)
   const messages = useAtomValue(getCurrentChatMessagesAtom)
   const { resendChatMessage } = useSendChatMessage()
@@ -49,11 +49,7 @@ const MessageToolbar = ({ message }: { message: ThreadMessage }) => {
   }
 
   const onEditClick = async () => {
-    if (!editMessage.length) {
+    setEditMessage(message.id ?? '')
-      setEditMessage(message.id ?? '')
-    } else {
-      setEditMessage('')
-    }
   }
 
   const onRegenerateClick = async () => {

web/screens/Chat/Sidebar/index.tsx

@@ -116,7 +116,7 @@ const Sidebar: React.FC = () => {
           </div>
         </div>
 
-        <CardSidebar title="Assistant">
+        <CardSidebar title="Assistant" isShow={true}>
           <div className="flex flex-col space-y-4 p-2">
             <div className="flex items-center space-x-2">
               <LogoMark width={24} height={24} />
@@ -152,11 +152,52 @@ const Sidebar: React.FC = () => {
           </div>
         </CardSidebar>
 
+        <CardSidebar title="Model" isShow={true}>
+          <div className="px-2 pt-4">
+            <DropdownListSidebar />
+
+            {componentDataRuntimeSetting.length > 0 && (
+              <div className="mt-6">
+                <CardSidebar title="Inference Parameters" asChild>
+                  <div className="px-2 py-4">
+                    <ModelSetting componentData={componentDataRuntimeSetting} />
+                  </div>
+                </CardSidebar>
+              </div>
+            )}
+
+            {componentDataEngineSetting.filter(
+              (x) => x.name === 'prompt_template'
+            ).length !== 0 && (
+              <div className="mt-4">
+                <CardSidebar title="Model Parameters" asChild>
+                  <div className="px-2 py-4">
+                    <SettingComponentBuilder
+                      componentData={componentDataEngineSetting}
+                      selector={(x: any) => x.name === 'prompt_template'}
+                    />
+                  </div>
+                </CardSidebar>
+              </div>
+            )}
+
+            {componentDataEngineSetting.length > 0 && (
+              <div className="my-4">
+                <CardSidebar title="Engine Parameters" asChild>
+                  <div className="px-2 py-4">
+                    <EngineSetting componentData={componentDataEngineSetting} />
+                  </div>
+                </CardSidebar>
+              </div>
+            )}
+          </div>
+        </CardSidebar>
+
         {experimentalFeature && (
           <div>
             {activeThread?.assistants[0]?.tools &&
               componentDataAssistantSetting.length > 0 && (
-                <CardSidebar title="Tools">
+                <CardSidebar title="Tools" isShow={true}>
                   <div className="px-2 pt-4">
                     <div className="mb-2">
                       <div className="flex items-center justify-between">
@@ -312,47 +353,6 @@ const Sidebar: React.FC = () => {
               )}
           </div>
         )}
-
-        <CardSidebar title="Model">
-          <div className="px-2 pt-4">
-            <DropdownListSidebar />
-
-            {componentDataRuntimeSetting.length > 0 && (
-              <div className="mt-6">
-                <CardSidebar title="Inference Parameters" asChild>
-                  <div className="px-2 py-4">
-                    <ModelSetting componentData={componentDataRuntimeSetting} />
-                  </div>
-                </CardSidebar>
-              </div>
-            )}
-
-            {componentDataEngineSetting.filter(
-              (x) => x.name === 'prompt_template'
-            ).length !== 0 && (
-              <div className="mt-4">
-                <CardSidebar title="Model Parameters" asChild>
-                  <div className="px-2 py-4">
-                    <SettingComponentBuilder
-                      componentData={componentDataEngineSetting}
-                      selector={(x: any) => x.name === 'prompt_template'}
-                    />
-                  </div>
-                </CardSidebar>
-              </div>
-            )}
-
-            {componentDataEngineSetting.length > 0 && (
-              <div className="my-4">
-                <CardSidebar title="Engine Parameters" asChild>
-                  <div className="px-2 py-4">
-                    <EngineSetting componentData={componentDataEngineSetting} />
-                  </div>
-                </CardSidebar>
-              </div>
-            )}
-          </div>
-        </CardSidebar>
       </div>
     </div>
   )

web/screens/Chat/ThreadList/index.tsx

@@ -3,7 +3,7 @@ import { useCallback, useEffect } from 'react'
 import { Thread } from '@janhq/core/'
 
 import { motion as m } from 'framer-motion'
-import { useAtomValue } from 'jotai'
+import { useAtomValue, useSetAtom } from 'jotai'
 import { GalleryHorizontalEndIcon, MoreVerticalIcon } from 'lucide-react'
 
 import { twMerge } from 'tailwind-merge'
@@ -18,6 +18,7 @@ import CleanThreadModal from '../CleanThreadModal'
 import DeleteThreadModal from '../DeleteThreadModal'
 
 import { assistantsAtom } from '@/helpers/atoms/Assistant.atom'
+import { editMessageAtom } from '@/helpers/atoms/ChatMessage.atom'
 import {
   getActiveThreadIdAtom,
   threadDataReadyAtom,
@@ -33,12 +34,14 @@ export default function ThreadList() {
   const assistants = useAtomValue(assistantsAtom)
   const threadDataReady = useAtomValue(threadDataReadyAtom)
   const { requestCreateNewThread } = useCreateNewThread()
+  const setEditMessage = useSetAtom(editMessageAtom)
 
   const onThreadClick = useCallback(
     (thread: Thread) => {
       setActiveThread(thread)
+      setEditMessage('')
     },
-    [setActiveThread]
+    [setActiveThread, setEditMessage]
   )
 
   /**

web/screens/LocalServer/index.tsx

@@ -39,6 +39,8 @@ import ModalTroubleShooting, {
 } from '@/containers/ModalTroubleShoot'
 import ServerLogs from '@/containers/ServerLogs'
 
+import { toaster } from '@/containers/Toast'
+
 import { loadModelErrorAtom, useActiveModel } from '@/hooks/useActiveModel'
 import { useLogs } from '@/hooks/useLogs'
 
@@ -106,6 +108,45 @@ const LocalServerScreen = () => {
     handleChangePort(port)
   }, [handleChangePort, port])
 
+  const onStartServerClick = async () => {
+    if (selectedModel == null) return
+    try {
+      const isStarted = await window.core?.api?.startServer({
+        host,
+        port,
+        isCorsEnabled,
+        isVerboseEnabled,
+      })
+      await startModel(selectedModel.id)
+      if (isStarted) setServerEnabled(true)
+      if (firstTimeVisitAPIServer) {
+        localStorage.setItem(FIRST_TIME_VISIT_API_SERVER, 'false')
+        setFirstTimeVisitAPIServer(false)
+      }
+    } catch (e) {
+      console.error(e)
+      toaster({
+        title: `Failed to start server!`,
+        description: 'Please check Server Logs for more details.',
+        type: 'error',
+      })
+    }
+  }
+
+  const onStopServerClick = async () => {
+    window.core?.api?.stopServer()
+    setServerEnabled(false)
+    setLoadModelError(undefined)
+  }
+
+  const onToggleServer = async () => {
+    if (serverEnabled) {
+      await onStopServerClick()
+    } else {
+      await onStartServerClick()
+    }
+  }
+
   return (
     <div className="flex h-full w-full" data-testid="local-server-testid">
       {/* Left SideBar */}
@@ -122,26 +163,7 @@ const LocalServerScreen = () => {
               block
               themes={serverEnabled ? 'danger' : 'primary'}
               disabled={stateModel.loading || errorRangePort || !selectedModel}
-              onClick={async () => {
+              onClick={onToggleServer}
-                if (serverEnabled) {
-                  window.core?.api?.stopServer()
-                  setServerEnabled(false)
-                  setLoadModelError(undefined)
-                } else {
-                  startModel(String(selectedModel?.id))
-                  const isStarted = await window.core?.api?.startServer({
-                    host,
-                    port,
-                    isCorsEnabled,
-                    isVerboseEnabled,
-                  })
-                  if (isStarted) setServerEnabled(true)
-                  if (firstTimeVisitAPIServer) {
-                    localStorage.setItem(FIRST_TIME_VISIT_API_SERVER, 'false')
-                    setFirstTimeVisitAPIServer(false)
-                  }
-                }
-              }}
             >
               {serverEnabled ? 'Stop' : 'Start'} Server
             </Button>

web/screens/Settings/Advanced/DataFolder/index.tsx

@@ -83,7 +83,8 @@ const DataFolder = () => {
         await window.core?.api?.getAppConfigurations()
       const currentJanDataFolder = appConfiguration.data_folder
       appConfiguration.data_folder = destinationPath
-      await fs.syncFile(currentJanDataFolder, destinationPath)
+      const { err } = await fs.syncFile(currentJanDataFolder, destinationPath)
+      if (err) throw err
       await window.core?.api?.updateAppConfiguration(appConfiguration)
       console.debug(
         `File sync finished from ${currentJanDataFolder} to ${destinationPath}`
@@ -94,7 +95,7 @@ const DataFolder = () => {
       }, 1200)
       await window.core?.api?.relaunch()
     } catch (e) {
-      console.error(`Error: ${e}`)
+      console.error(e)
       setShowLoader(false)
       setShowChangeFolderError(true)
     }

web/screens/Settings/Advanced/index.tsx

@@ -36,6 +36,7 @@ import { snackbar, toaster } from '@/containers/Toast'
 
 import { FeatureToggleContext } from '@/context/FeatureToggle'
 
+import { useActiveModel } from '@/hooks/useActiveModel'
 import { useSettings } from '@/hooks/useSettings'
 
 import DataFolder from './DataFolder'
@@ -55,6 +56,10 @@ const Advanced = () => {
     setIgnoreSSL,
     proxy,
     setProxy,
+    proxyEnabled,
+    setProxyEnabled,
+    vulkanEnabled,
+    setVulkanEnabled,
   } = useContext(FeatureToggleContext)
   const [partialProxy, setPartialProxy] = useState<string>(proxy)
   const [gpuEnabled, setGpuEnabled] = useState<boolean>(false)
@@ -62,6 +67,7 @@ const Advanced = () => {
   const [gpusInUse, setGpusInUse] = useState<string[]>([])
   const { readSettings, saveSettings, validateSettings, setShowNotification } =
     useSettings()
+  const { stopModel } = useActiveModel()
 
   const selectedGpu = gpuList
     .filter((x) => gpusInUse.includes(x.id))
@@ -85,14 +91,15 @@ const Advanced = () => {
   useEffect(() => {
     const setUseGpuIfPossible = async () => {
       const settings = await readSettings()
-      setGpuEnabled(settings.run_mode === 'gpu' && gpuList.length > 0)
+      setGpuEnabled(settings.run_mode === 'gpu' && settings.gpus?.length > 0)
       setGpusInUse(settings.gpus_in_use || [])
+      setVulkanEnabled(settings.vulkan || false)
       if (settings.gpus) {
         setGpuList(settings.gpus)
       }
     }
     setUseGpuIfPossible()
-  }, [readSettings, gpuList])
+  }, [readSettings, setGpuList, setGpuEnabled, setGpusInUse, setVulkanEnabled])
 
   const clearLogs = async () => {
     if (await fs.existsSync(`file://logs`)) {
@@ -106,14 +113,21 @@ const Advanced = () => {
   }
 
   const handleGPUChange = (gpuId: string) => {
-    // TODO detect current use GPU nvidia or AMD
     let updatedGpusInUse = [...gpusInUse]
     if (updatedGpusInUse.includes(gpuId)) {
       updatedGpusInUse = updatedGpusInUse.filter((id) => id !== gpuId)
       if (gpuEnabled && updatedGpusInUse.length === 0) {
+        // Vulkan support only allow 1 active device at a time
+        if (vulkanEnabled) {
+          updatedGpusInUse = []
+        }
         updatedGpusInUse.push(gpuId)
       }
     } else {
+      // Vulkan support only allow 1 active device at a time
+      if (vulkanEnabled) {
+        updatedGpusInUse = []
+      }
       updatedGpusInUse.push(gpuId)
     }
     setGpusInUse(updatedGpusInUse)
@@ -169,8 +183,8 @@ const Advanced = () => {
                 </h6>
               </div>
               <p className="pr-8 leading-relaxed">
-                Enable to enhance model performance by utilizing your devices
+                Enable to enhance model performance by utilizing your GPU
-                GPU for acceleration. Read{' '}
+                devices for acceleration. Read{' '}
                 <span>
                   {' '}
                   <span
@@ -198,7 +212,7 @@ const Advanced = () => {
                   className="max-w-[240px]"
                 >
                   <span>
-                    Disabling GPU Acceleration may result in reduced
+                    Disabling NVIDIA GPU Acceleration may result in reduced
                     performance. It is recommended to keep this enabled for
                     optimal user experience.
                   </span>
@@ -210,7 +224,7 @@ const Advanced = () => {
             <Tooltip>
               <TooltipTrigger>
                 <Switch
-                  disabled={gpuList.length === 0}
+                  disabled={gpuList.length === 0 || vulkanEnabled}
                   checked={gpuEnabled}
                   onCheckedChange={(e) => {
                     if (e === true) {
@@ -232,6 +246,8 @@ const Advanced = () => {
                         type: 'success',
                       })
                     }
+                    // Stop any running model to apply the changes
+                    if (e !== gpuEnabled) stopModel()
                   }}
                 />
               </TooltipTrigger>
@@ -253,12 +269,14 @@ const Advanced = () => {
             </Tooltip>
           </div>
           <div className="mt-2 w-full rounded-lg bg-secondary p-4">
-            <label className="mb-1 inline-block font-medium">Choose GPU</label>
+            <label className="mb-1 inline-block font-medium">
+              Choose device(s)
+            </label>
             <Select
               disabled={gpuList.length === 0 || !gpuEnabled}
               value={selectedGpu.join()}
             >
-              <SelectTrigger className="w-[340px] bg-white">
+              <SelectTrigger className="w-[340px] dark:bg-gray-500 bg-white">
                 <SelectValue placeholder={gpuSelectionPlaceHolder}>
                   <span className="line-clamp-1 w-full pr-8">
                     {selectedGpu.join()}
@@ -268,12 +286,16 @@ const Advanced = () => {
               <SelectPortal>
                 <SelectContent className="w-[400px] px-1 pb-2">
                   <SelectGroup>
-                    <SelectLabel>Nvidia</SelectLabel>
+                    <SelectLabel>
+                      {vulkanEnabled ? 'Vulkan Supported GPUs' : 'Nvidia'}
+                    </SelectLabel>
                     <div className="px-4 pb-2">
                       <div className="rounded-lg bg-secondary p-3">
                         {gpuList
                           .filter((gpu) =>
-                            gpu.name.toLowerCase().includes('nvidia')
+                            vulkanEnabled
+                              ? gpu.name
+                              : gpu.name?.toLowerCase().includes('nvidia')
                           )
                           .map((gpu) => (
                             <div
@@ -293,7 +315,9 @@ const Advanced = () => {
                                 htmlFor={`gpu-${gpu.id}`}
                               >
                                 <span>{gpu.name}</span>
-                                <span>{gpu.vram}MB VRAM</span>
+                                {!vulkanEnabled && (
+                                  <span>{gpu.vram}MB VRAM</span>
+                                )}
                               </label>
                             </div>
                           ))}
@@ -322,12 +346,47 @@ const Advanced = () => {
         </div>
       )}
 
+      {/* Vulkan for AMD GPU/ APU and Intel Arc GPU */}
+      {!isMac && experimentalFeature && (
+        <div className="flex w-full items-start justify-between border-b border-border py-4 first:pt-0 last:border-none">
+          <div className="flex-shrink-0 space-y-1.5">
+            <div className="flex gap-x-2">
+              <h6 className="text-sm font-semibold capitalize">
+                Vulkan Support
+              </h6>
+            </div>
+            <p className="text-xs leading-relaxed">
+              Enable Vulkan with AMD GPU/APU and Intel Arc GPU for better model
+              performance (reload needed).
+            </p>
+          </div>
+
+          <Switch
+            checked={vulkanEnabled}
+            onCheckedChange={(e) => {
+              toaster({
+                title: 'Reload',
+                description:
+                  'Vulkan settings updated. Reload now to apply the changes.',
+              })
+              stopModel()
+              saveSettings({ vulkan: e, gpusInUse: [] })
+              setVulkanEnabled(e)
+            }}
+          />
+        </div>
+      )}
+
       <DataFolder />
       {/* Proxy */}
       <div className="flex w-full items-start justify-between border-b border-border py-4 first:pt-0 last:border-none">
-        <div className="flex-shrink-0 space-y-1.5">
+        <div className="flex-shrink-0 space-y-1.5 w-full">
-          <div className="flex gap-x-2">
+          <div className="flex gap-x-2 justify-between w-full">
             <h6 className="text-sm font-semibold capitalize">HTTPS Proxy</h6>
+            <Switch
+              checked={proxyEnabled}
+              onCheckedChange={(_) => setProxyEnabled(!proxyEnabled)}
+            />
           </div>
           <p className="leading-relaxed">
             Specify the HTTPS proxy or leave blank (proxy auto-configuration and
@@ -337,6 +396,7 @@ const Advanced = () => {
             placeholder={'http://<user>:<password>@<domain or IP>:<port>'}
             value={partialProxy}
             onChange={onProxyChange}
+            className="w-2/3"
           />
         </div>
       </div>

web/services/restService.ts

@@ -18,11 +18,14 @@ export const restAPI = {
       ...acc,
       [proxy.route]: (...args: any) => {
         // For each route, define a function that sends a request to the API
-        return fetch(`${API_BASE_URL}/v1/${proxy.path}/${proxy.route}`, {
+        return fetch(
-          method: 'POST',
+          `${window.core?.api.baseApiUrl}/v1/${proxy.path}/${proxy.route}`,
-          body: JSON.stringify(args),
+          {
-          headers: { contentType: 'application/json' },
+            method: 'POST',
-        }).then(async (res) => {
+            body: JSON.stringify(args),
+            headers: { contentType: 'application/json' },
+          }
+        ).then(async (res) => {
           try {
             if (proxy.path === 'fs') {
               const text = await res.text()
@@ -38,6 +41,6 @@ export const restAPI = {
   }, {}),
   openExternalUrl,
   // Jan Server URL
-  baseApiUrl: API_BASE_URL,
+  baseApiUrl: process.env.API_BASE_URL ?? API_BASE_URL,
   pollingInterval: 5000,
 }