Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/issue-255-adr-001-jand-cloud-native

Browse Source
This commit is contained in:
0xSage 2023-10-16 10:35:16 +08:00 committed by GitHub
commit 856bd6bd37
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
292 changed files with 41960 additions and 12110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@ -28,11 +28,5 @@ If applicable, add screenshots to help explain your problem.
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Smartphone (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Browser [e.g. stock browser, safari]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.

4
.github/ISSUE_TEMPLATE/discussion-thread.md vendored
View File

@ -2,7 +2,7 @@
name: Discussion thread
about: Start an open ended discussion
title: 'Discussion: [TOPIC HERE]'
labels: ''
labels: 'type: discussion'
assignees: ''
---
@ -11,4 +11,6 @@ assignees: ''
**Discussion**
**Alternatives**
**Resources**

26
.github/release-drafter.yml vendored Normal file
View File

@ -0,0 +1,26 @@
categories:
- title: '🚀 Features'
labels:
- 'type: enhancement'
- 'type: epic'
- 'type: feature request'
- title: '🐛 Bug Fixes'
labels:
- 'type: bug'
- title: '🧰 Maintenance'
labels:
- 'type: chore'
- 'type: ci'
- title: '📖 Documentaion'
labels:
- 'type: documentation'
change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
template: |
## Changes
$CHANGES
## Contributor
$CONTRIBUTORS

182
.github/workflows/build-app.yml vendored Normal file
View File

@ -0,0 +1,182 @@
name: Jan Build MacOS App
on:
push:
tags: ['v*.*.*']
jobs:
build-macos:
runs-on: macos-latest
environment: production
permissions:
contents: write
steps:
- name: Getting the repo
uses: actions/checkout@v3
- name: Installing node
uses: actions/setup-node@v1
with:
node-version: 20
- name: Install jq
uses: dcarbone/install-jq-action@v2.0.1
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
- name: Update app version base on tag
run: |
if [[ ! "${VERSION_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Error: Tag is not valid!"
exit 1
fi
jq --arg version "${VERSION_TAG#v}" '.version = $version' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
env:
VERSION_TAG: ${{ steps.tag.outputs.tag }}
- name: Get Cer for code signing
run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
shell: bash
env:
CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
- uses: apple-actions/import-codesign-certs@v2
with:
p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
- name: Install yarn dependencies
run: |
yarn install
yarn build:plugins-darwin
- name: Build and publish app
run: |
yarn build:publish-darwin
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CSC_LINK: "/tmp/codesign.p12"
CSC_KEY_PASSWORD: ${{ secrets.CODE_SIGN_P12_PASSWORD }}
CSC_IDENTITY_AUTO_DISCOVERY: "true"
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
build-windows-x64:
runs-on: windows-latest
permissions:
contents: write
steps:
- name: Getting the repo
uses: actions/checkout@v3
- name: Installing node
uses: actions/setup-node@v1
with:
node-version: 20
- name: Install jq
uses: dcarbone/install-jq-action@v2.0.1
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
- name: Update app version base on tag
shell: bash
run: |
if [[ ! "${VERSION_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Error: Tag is not valid!"
exit 1
fi
jq --arg version "${VERSION_TAG#v}" '.version = $version' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
env:
VERSION_TAG: ${{ steps.tag.outputs.tag }}
- name: Install yarn dependencies
run: |
yarn config set network-timeout 300000
yarn install
yarn build:plugins
- name: Build and publish app
run: |
yarn build:publish-win32
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build-linux-x64:
runs-on: ubuntu-latest
environment: production
env:
SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAPCRAFT_TOKEN }}
permissions:
contents: write
steps:
- name: Getting the repo
uses: actions/checkout@v3
- name: Installing node
uses: actions/setup-node@v1
with:
node-version: 20
- name: Install jq
uses: dcarbone/install-jq-action@v2.0.1
- name: Install Snapcraft
uses: samuelmeuli/action-snapcraft@v2
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
- name: Update app version base on tag
run: |
if [[ ! "${VERSION_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Error: Tag is not valid!"
exit 1
fi
jq --arg version "${VERSION_TAG#v}" '.version = $version' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
env:
VERSION_TAG: ${{ steps.tag.outputs.tag }}
- name: Install yarn dependencies
run: |
yarn config set network-timeout 300000
yarn install
yarn build:plugins
- name: Build and publish app
run: |
yarn build:publish-linux
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
update_release_draft:
needs: [build-macos, build-windows-x64, build-linux-x64]
permissions:
# write permission is required to create a github release
contents: write
# write permission is required for autolabeler
# otherwise, read permission is required at least
pull-requests: write
runs-on: ubuntu-latest
steps:
# (Optional) GitHub Enterprise requires GHE_HOST variable set
#- name: Set GHE_HOST
# run: |
# echo "GHE_HOST=${GITHUB_SERVER_URL##https:\/\/}" >> $GITHUB_ENV
# Drafts your next Release notes as Pull Requests are merged into "master"
- uses: release-drafter/release-drafter@v5
# (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml
# with:
# config-name: my-config.yml
# disable-autolabeler: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

2
.github/workflows/deploy-jan-docs.yml vendored
View File

@ -26,7 +26,7 @@ jobs:
run: yarn install
working-directory: docs
- name: Build website
run: yarn build
run: sed -i '/process.env.DEBUG = namespaces;/c\// process.env.DEBUG = namespaces;' ./node_modules/debug/src/node.js && yarn build
working-directory: docs
- name: Add Custome Domain file

29
.github/workflows/jan-docs-test.yml vendored Normal file
View File

@ -0,0 +1,29 @@
name: Jan Docs Test Build
on:
pull_request:
branches:
- main
paths:
- 'docs/**'
- '.github/workflows/deploy-jan-docs.yml'
- '.github/workflows/jan-docs-test.yml'
jobs:
deploy:
name: Test Build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 18
cache: 'yarn'
cache-dependency-path: './docs/yarn.lock'
- name: Install dependencies
run: yarn install
working-directory: docs
- name: Test Build Command
run: sed -i '/process.env.DEBUG = namespaces;/c\// process.env.DEBUG = namespaces;' ./node_modules/debug/src/node.js && yarn build
working-directory: docs

106
.github/workflows/linter-and-test.yml vendored Normal file
View File

@ -0,0 +1,106 @@
name: Linter & Test
on:
push:
branches:
- main
paths:
- 'electron/**'
- .github/workflows/linter-and-test.yml
- 'web/**'
- 'package.json'
- 'node_modules/**'
- 'yarn.lock'
pull_request:
branches:
- main
paths:
- 'electron/**'
- .github/workflows/linter-and-test.yml
- 'web/**'
- 'package.json'
- 'node_modules/**'
- 'yarn.lock'
jobs:
test-on-macos:
runs-on: [self-hosted, macOS, macos-desktop]
steps:
- name: 'Cleanup build folder'
run: |
ls -la ./
rm -rf ./* || true
rm -rf ./.??* || true
ls -la ./
- name: Getting the repo
uses: actions/checkout@v3
- name: Installing node
uses: actions/setup-node@v1
with:
node-version: 20
- name: Linter and test
run: |
yarn config set network-timeout 300000
yarn build:core
yarn install
yarn lint
yarn build:plugins
yarn build
yarn test
env:
CSC_IDENTITY_AUTO_DISCOVERY: "false"
test-on-windows:
runs-on: [self-hosted, Windows, windows-desktop]
steps:
- name: Clean workspace
run: |
Remove-Item -Path .\* -Force -Recurse
- name: Getting the repo
uses: actions/checkout@v3
- name: Installing node
uses: actions/setup-node@v1
with:
node-version: 20
- name: Linter and test
run: |
yarn config set network-timeout 300000
yarn build:core
yarn install
yarn lint
yarn build:plugins
yarn build:win32
yarn test
test-on-ubuntu:
runs-on: [self-hosted, Linux, ubuntu-desktop]
steps:
- name: 'Cleanup build folder'
run: |
ls -la ./
rm -rf ./* || true
rm -rf ./.??* || true
ls -la ./
- name: Getting the repo
uses: actions/checkout@v3
- name: Installing node
uses: actions/setup-node@v1
with:
node-version: 20
- name: Linter and test
run: |
export DISPLAY=$(w -h | awk 'NR==1 {print $2}')
echo -e "Display ID: $DISPLAY"
yarn config set network-timeout 300000
yarn build:core
yarn install
yarn lint
yarn build:plugins
yarn build:linux
yarn test

50
.github/workflows/macos-build-app.yml vendored
View File

@ -1,50 +0,0 @@
name: Jan Build MacOS App
on:
push:
tags: ['v*.*.*']
jobs:
build-macos-app:
runs-on: macos-latest
permissions:
contents: write
steps:
- name: Getting the repo
uses: actions/checkout@v3
- name: Installing node
uses: actions/setup-node@v1
with:
node-version: 20
- name: Install jq
uses: dcarbone/install-jq-action@v2.0.1
- name: Get tag
id: tag
uses: dawidd6/action-get-tag@v1
- name: Update app version base on tag
run: |
if [[ ! "${VERSION_TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Error: Tag is not valid!"
exit 1
fi
jq --arg version "${VERSION_TAG#v}" '.version = $version' electron/package.json > /tmp/package.json
mv /tmp/package.json electron/package.json
env:
VERSION_TAG: ${{ steps.tag.outputs.tag }}
- name: Install yarn dependencies
run: |
yarn install
yarn build:plugins
- name: Build and publish app
run: |
yarn build:publish
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

66
.github/workflows/publish-plugin-core.yml vendored Normal file
View File

@ -0,0 +1,66 @@
name: Publish Plugin-core Package to npmjs
on:
push:
branches:
- main
paths:
- "plugin-core/**"
- ".github/workflows/publish-plugin-core.yml"
- "!plugin-core/package.json"
jobs:
build:
runs-on: ubuntu-latest
environment: production
steps:
- uses: actions/checkout@v4
with:
fetch-depth: "0"
token: ${{ secrets.PAT_SERVICE_ACCOUNT }}
- name: Install jq
uses: dcarbone/install-jq-action@v2.0.1
- name: "Auto Increase package Version"
run: |
# Extract current version
current_version=$(jq -r '.version' plugin-core/package.json)
# Break the version into its components
major_version=$(echo $current_version | cut -d "." -f 1)
minor_version=$(echo $current_version | cut -d "." -f 2)
patch_version=$(echo $current_version | cut -d "." -f 3)
# Increment the patch version by one
new_patch_version=$((patch_version+1))
# Construct the new version
new_version="$major_version.$minor_version.$new_patch_version"
# Replace the old version with the new version in package.json
jq --arg version "$new_version" '.version = $version' plugin-core/package.json > /tmp/package.json && mv /tmp/package.json plugin-core/package.json
# Print the new version
echo "Updated package.json version to: $new_version"
# Setup .npmrc file to publish to npm
- uses: actions/setup-node@v3
with:
node-version: "20.x"
registry-url: "https://registry.npmjs.org"
- run: npm install && npm run build
working-directory: ./plugin-core
- run: npm publish --access public
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
working-directory: ./plugin-core
- name: "Commit new version to main and create tag"
run: |
version=$(jq -r '.version' plugin-core/package.json)
git config --global user.email "service@jan.ai"
git config --global user.name "Service Account"
git add plugin-core/package.json
git commit -m "${GITHUB_REPOSITORY}: Update tag build $version"
git -c http.extraheader="AUTHORIZATION: bearer ${{ secrets.PAT_SERVICE_ACCOUNT }}" push origin HEAD:main
git tag -a plugin-core-$version -m "${GITHUB_REPOSITORY}: Update tag build $version for plugin-core"
git -c http.extraheader="AUTHORIZATION: bearer ${{ secrets.PAT_SERVICE_ACCOUNT }}" push origin plugin-core-$version

2
.gitignore vendored
View File

@ -5,7 +5,6 @@
models/**
error.log
node_modules
package-lock.json
*.tgz
yarn.lock
dist
@ -14,3 +13,4 @@ build
electron/renderer
*.log
plugin-core/lib

699
LICENSE
View File

@ -1,79 +1,674 @@
# License
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
This software is licensed under the following conditions:
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
- All third party components incorporated into the software are licensed under the original license
provided by the owner of the applicable component.
- Content outside of the above mentioned files or restrictions is available under the "Sustainable Use
License" as defined below.
Preamble
# Sustainable Use License
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
## Acceptance
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
By utilizing the software, you consent to all of the terms and conditions outlined below.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
## Copyright License
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
The licensor provides you a non-exclusive, royalty-free, worldwide, non-sublicensable, non-transferable license
to use, copy, distribute, make available, and prepare derivative works of the software, in each case subject
to the restrictions below.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
## Restrictions
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
The software may be used or modified only for your own internal business purposes or for non-commercial or
personal use. Distribution of the software or provision to others is only permitted if done free of charge for
non-commercial purposes. You are prohibited from altering, removing, or obscuring any licensing, copyright, or other notices of
the licensor in the software. Any use of the licensor’s trademarks is subject to applicable law.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
## Patents
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
The licensor grants you a license, under any patent claims the licensor can license, or becomes able to
license, to make, have made, use, sell, offer for sale, import and have imported the software, in each case
subject to the restrictions and conditions in this license. This license does not cover any patent claims that
you cause to be infringed by modifications or additions to the software. If you or your company make any
written claim that the software infringes or contributes to infringement of any patent, your patent license
for the software granted under these terms ends immediately. If your company makes such a claim, your patent
license ends immediately for work on behalf of your company.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
## Notices
The precise terms and conditions for copying, distribution and
modification follow.
You must ensure that anyone who receives a copy of any part of the software from you also receives a copy of these
terms. If you modify the software, you must include in any modified copies of the software a prominent notice
stating that you have modified the software.
TERMS AND CONDITIONS
## No Other Rights
1. Definitions.
These terms do not imply any licenses other than those expressly granted in these terms.
"This License" refers to version 3 of the GNU General Public License.
## Termination
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
If you use the software in violation of these terms, such use is not licensed, and your license will
automatically terminate. If the licensor provides you with a notice of your violation, and you cease all
violation of this license no later than 30 days after you receive that notice, your license will be reinstated
retroactively. However, if you violate these terms after such reinstatement, any additional violation of these
terms will cause your license to terminate automatically and permanently.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
## No Liability
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
To the extent permitted by law, the software is provided "as is", without any warranty or condition, expressed or implied. In no event shall the authors or licensor be liable to you for any damages arising out of these terms or the use or nature of the software, under
any kind of legal claim.
A "covered work" means either the unmodified Program or a work based
on the Program.
## Definitions
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
The “licensor” is the entity offering these terms.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
The “software” is the software the licensor makes available under these terms, including any portion of it.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
“You” refers to the individual or entity agreeing to these terms.
1. Source Code.
“Your company” is any legal entity, sole proprietorship, or other kind of organization that you work for, plus
all organizations that have control over, are under the control of, or are under common control with that
organization. Control means ownership of substantially all the assets of an entity, or the power to direct its
management and policies by vote, contract, or otherwise. Control can be direct or indirect.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
“Your license” is the license granted to you for the software under these terms.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
“Use” means anything you do with the software requiring your license.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
“Trademark” means trademarks, service marks, and similar rights.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<https://www.gnu.org/licenses/why-not-lgpl.html>.

88
README.md
View File

@ -1,8 +1,6 @@
# Jan - Run your own AI
<p align="center">
<img alt="janlogo" src="https://user-images.githubusercontent.com/69952136/266827788-b37d6f41-fc34-4677-aa1f-3e2ca6d3c91a.png">
</p>
![](./docs/static/img/github-readme-banner.png)
<p align="center">
<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
@ -20,64 +18,77 @@
> ⚠️ **Jan is currently in Development**: Expect breaking changes and bugs!
Jan lets you run AI on your own hardware, with helpful tools to manage models and monitor your hardware performance.
Jan runs Large Language Models and AIs on your own Windows, Mac or Linux computer. Jan can be run as a desktop app, or as a cloud-native deployment.
In the background, Jan runs [Nitro](https://nitro.jan.ai), a C++ inference engine. It runs various model formats (GGUF/TensorRT) on various hardware (Mac M1/M2/Intel, Windows, Linux, and datacenter-grade Nvidia GPUs) with optional GPU acceleration.
Jan is free and open source, under the GPLv3 license.
> See the Nitro codebase at https://nitro.jan.ai.
**Jan runs on any hardware.** From PCs to multi-GPU clusters, Jan supports universal architectures:
<!-- TODO: uncomment this later when we have this feature -->
<!-- Jan can be run as a server or cloud-native application for enterprise. We offer enterprise plugins for LDAP integration and Audit Logs. Contact us at [hello@jan.ai](mailto:hello@jan.ai) for more details. -->
- [x] Nvidia GPUs (fast)
- [x] Apple M-series (fast)
- [x] Apple Intel
- [x] Linux Debian
- [x] Windows x64
> Download Jan at https://jan.ai/
## Demo
<p align="center">
<img style='border:1px solid #000000' src="https://github.com/janhq/jan/assets/69952136/1f9bb48c-2e70-4633-9f68-7881cd925972" alt="Jan Web GIF">
<img style='border:1px solid #000000' src="https://github.com/janhq/jan/assets/69952136/1db9c3d3-79b1-4988-afb5-afd4f4afd0d9" alt="Jan Web GIF">
</p>
_Screenshot: Jan v0.1.3 on Mac M1 Pro, 16GB Sonoma_
## Quicklinks
- Developer documentation: https://jan.ai/docs (Work in Progress)
- Desktop app: Download at https://jan.ai/
- Mobile app shell: Download via [App Store](https://apps.apple.com/us/app/jan-on-device-ai-cloud-ais/id6449664703) | [Android](https://play.google.com/store/apps/details?id=com.jan.ai)
- Nitro (C++ AI Engine): https://nitro.jan.ai
- [Developer docs](https://jan.ai/docs) (WIP)
- Mobile App shell: [App Store](https://apps.apple.com/us/app/jan-on-device-ai-cloud-ais/id6449664703) | [Android](https://play.google.com/store/apps/details?id=com.jan.ai)
- [Nitro Github](https://nitro.jan.ai): Jan's AI engine
## Plugins
Jan supports core & 3rd party extensions:
- [x] **LLM chat**: Self-hosted Llama2 and LLMs
- [x] **Model Manager**: 1-click to install, swap, and delete models
- [x] **Storage**: Optionally store your conversation history and other data in SQLite/your storage of choice
- [x] **Model Manager**: 1-click to install, swap, and delete models with HuggingFace integration
- [x] **Storage**: Optionally save conversation history and other data in SQLite
- [ ] **3rd-party AIs**: Connect to ChatGPT, Claude via API Key (in progress)
- [ ] **Cross device support**: Mobile & Web support for custom shared servers (in progress)
- [ ] **File retrieval**: User can upload private and run a vectorDB (planned)
- [ ] **File retrieval**: User can chat with docs
- [ ] **Multi-user support**: Share a single server across a team/friends (planned)
- [ ] **Compliance**: Auditing and flagging features (planned)
## Hardware Support
## Nitro (Jan's AI engine)
Nitro provides both CPU and GPU support, via [llama.cpp](https://github.com/ggerganov/llama.cpp) and [TensorRT](https://github.com/NVIDIA/TensorRT), respectively.
In the background, Jan runs [Nitro](https://nitro.jan.ai), an open source, C++ inference engine. It runs various model formats (GGUF/TensorRT) on various hardware (Mac M1/M2/Intel, Windows, Linux, and datacenter-grade Nvidia GPUs) with optional GPU acceleration.
- [x] Nvidia GPUs (accelerated)
- [x] Apple M-series (accelerated)
- [x] Linux DEB
- [x] Windows x64
> See the open source Nitro codebase at https://nitro.jan.ai.
Not supported yet: Apple Intel, Linux RPM, Windows x86|ARM64, AMD ROCm
## Troubleshooting
As Jan is development mode, you might get stuck on a broken build.
> See [developer docs](https://docs.jan.ai/docs/) for detailed installation instructions.
To reset your installation:
1. Delete Jan Application from /Applications
1. Clear cache:
`rm -rf /Users/$(whoami)/Library/Application\ Support/jan`
OR
`rm -rf /Users/$(whoami)/Library/Application\ Support/jan`
---
## Contributing
Contributions are welcome! Please read the [CONTRIBUTING.md](CONTRIBUTING.md) file
### Pre-requisites
- node >= 20.0.0
- yarn >= 1.22.0
### Use as complete suite (in progress)
### For interactive development
### Instructions
Note: This instruction is tested on MacOS only.
@ -85,7 +96,7 @@ Note: This instruction is tested on MacOS only.
```
git clone https://github.com/janhq/jan
git checkout feature/hackathon-refactor-jan-into-electron-app
git checkout DESIRED_BRANCH
cd jan
```
@ -98,28 +109,29 @@ Note: This instruction is tested on MacOS only.
yarn build:plugins
```
4. **Run development and Using Jan Desktop**
3. **Run development and Using Jan Desktop**
```
yarn dev
```
This will start the development server and open the desktop app.
In this step, there are a few notification about installing base plugin, just click `OK` and `Next` to continue.
### For production build
```bash
# Do step 1 and 2 in previous section
git clone https://github.com/janhq/jan
cd jan
yarn install
yarn build:plugins
```bash
# Do step 1 and 2 in previous section
git clone https://github.com/janhq/jan
cd jan
yarn install
yarn build:plugins
# Build the app
yarn build
```
# Build the app
yarn build
```
This will build the app MacOS m1/m2 for production (with code signing already done) and put the result in `dist` folder.
This will build the app MacOS m1/m2 for production (with code signing already done) and put the result in `dist` folder.
## License

55
adr/adr-002-jan-ai-apps.md Normal file
View File

@ -0,0 +1,55 @@
# ADR #002: Jan AI apps
## Changelog
- Oct 4th 2023: Initial draft
- Oct 6th 2023: Update sample API
## Authors
- @vuonghoainam - Hiro
- @louis-jan
## Status
Proposed
## Context
### Business context
Jan can be a platform and let builders build their own `AI app` using existing tools
- Use-case 1: Medical AI startup uploads "case notes" to Jan, wants to ask it questions (i.e. medical audit)
- Use-case 2: Legal e-discovery: very large amount of documents (~10-15k pages) are uploaded, data is very private and cannot be leaked
- Use-case 3: Jan wants to use Jan to have a QnA chatbot to answer questions on docs
- Use-case 4: Jan wants to use Jan to have a codellama RAG on its own codebase, to generate new PRs
### Extra context
- There are many use cases that the community can develop and sell to the users through Jan as plugin. Jan needs to streamline higher value chain.
- This brings more value and more option to all kind of user
- This can help building ecosystem and streamline value end to end (Jan, plugins/ model creators, Jan users - enterprise/ individual)
- We at Jan cannot build plugins more on our own, but this one should serve as featured example like [OpenAI Retrieval plugin](https://github.com/openai/chatgpt-retrieval-plugin) does.
- [#232](https://github.com/janhq/jan/issues/232)
## Decision
- User can browse and install plugins (with recommended model - llama2, claude, openai …) - This requires plugin dependencies.
- Jan provide consistent interface for plugin developer to use:
- Use LLM (this can be switched in runtime) - i.e Dev in llama2-7b but user can use with llama2-70b. Can choose another model as well
- Plugin can have API for CRUD indices in vectorDB/ DB, and Jan only exposes corresponding data to the app
- A place for a plugin to store the files for persistence
- This works seamlessly on desktop/ Jan hosted version with Jan API abstraction.
### Simple UX
![UX](images/adr-002-01.png "UX")
### Component design
![Component design](images/adr-002-02.png "Component design")
## API
- `jan.plugin.<plugin_name>.<function_name>(**args)`
- `jan.core.db.sql.command()` -> CRUD/ query
- `jan.plugin.vectra.<function_name>(**args)` -> CRUD/ query for
## Consequences
- Jan user can build their own AI apps (and buy from others too) in an easy way
- Clear design for plugin and Jan platform development
## Reference
- [ADR-003](adr-003-jan-plugins.md)

29
adr/adr-003-jan-plugins.md Normal file
View File

@ -0,0 +1,29 @@
# ADR 003: JAN PLUGINS
## Changelog
- Oct 5th 2023: Initial draft
## Status
Accepted
## Context
Modular Architecture w/ Plugins:
- Jan will have an architecture similar to VSCode or k8Lens
- "Desktop Application" whose functionality can be extended thru plugins
- Jan's architecture will need to accomodate plugins for (a) Persistence(b) IAM(c) Teams and RBAC(d) Policy engines(e) "Apps" (i.e. higher-order business logic)(f) Themes (UI)
- Nitro's architecture will need to accomodate plugins for different "model backends"(a) llama.cpp(b) rkwk (and others)(c) 3rd-party AIs
## Decision
![Architecture](./images/adr-003-01.png)
## Consequences
What becomes easier or more difficult to do because of this change?
## Reference
[Plugin APIs](./adr-003-jan-plugins.md)

37
adr/docs/adr-003-plugins.md Normal file
View File

@ -0,0 +1,37 @@
## JAN service & plugin APIs
Jan frontend components will communicate with plugin functions via Service Interfaces:
All of the available APIs are listed in [CoreService](../../web/shared/coreService.ts)
- Data Service:
- GET_CONVERSATIONS: retrieve all of the conversations
- CREATE_CONVERSATION: start a new conversation
- DELETE_CONVERSATION: delete an existing conversation
- GET_CONVERSATION_MESSAGES: retrieve a certain conversation messages
- CREATE_MESSAGE: store a new message (both sent & received)
- UPDATE_MESSAGE: update an existing message (streaming)
- STORE_MODEL: store new model information (when clicking download)
- UPDATE_FINISHED_DOWNLOAD: mark a model as downloaded
- GET_UNFINISHED_DOWNLOAD_MODELS: retrieve all unfinished downloading model (TBD)
- GET_FINISHED_DOWNLOAD_MODELS: retrieve all finished downloading model (TBD)
- DELETE_DOWNLOAD_MODEL: delete a model (TBD)
- GET_MODEL_BY_ID: retrieve model information by its ID
- Inference Service:
- INFERENCE_URL: retrieve inference endpoint served by plugin
- INIT_MODEL: runs a model
- STOP_MODEL: stop a running model
- Model Management Service: (TBD)
- GET_AVAILABLE_MODELS: retrieve available models (deprecate soon)
- GET_DOWNLOADED_MODELS: (deprecated)
- DELETE_MODEL: (deprecated)
- DOWNLOAD_MODEL: start to download a model
- SEARCH_MODELS: explore models with search query on HuggingFace (TBD)
- Monitoring service:
- GET_RESOURCES_INFORMATION: retrieve total & used memory information
- GET_CURRENT_LOAD_INFORMATION: retrieve CPU load information

BIN
adr/images/adr-002-01.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 87 KiB

BIN
adr/images/adr-002-02.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 179 KiB

BIN
adr/images/adr-003-01.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 335 KiB

6
docs/docs/about/about.md
View File

@ -3,6 +3,8 @@ title: About Jan
slug: /about
---
## Team
## Problem
## History
## Ideal Customer Persona
## Business Model

10
docs/docs/about/roadmap.md
View File

@ -1,10 +0,0 @@
---
title: Roadmap
---
## Problem
## Ideal Customer Persona
## Business Model

4
docs/docs/about/team.md Normal file
View File

@ -0,0 +1,4 @@
---
title: Team
---

51
docs/docs/events/hcmc-oct23.md Normal file
View File

@ -0,0 +1,51 @@
---
title: "Jan's AI Hacker House (Ho Chi Minh City)"
description: "24-27 Oct 2023, Thao Dien, HCMC. AI-focused talks, workshops and social events. Hosted by Jan.ai"
slug: /events/hcmc-oct23
image: /img/hcmc-villa-1.jpeg
---
![](/img/hcmc-villa-1.jpeg)
## Ho Chi Minh City
[Jan's Hacker House](https://jan.ai) is a 4-day event where we host an open AI Hacker House and invite the local AI community to join us. There is fast wifi, free snacks, drinks and pizza.
We also host a series of talks, workshops and social events at night. We usually start off the week with a "Intro to LLMs" that targets local university students, and then progress to more in-depth technical and research areas.
Jan is a fully remote team. We use the money we save from not having an office, to hold Hack Weeks where we meet in a city, eat pizza and work to ship major releases.
### Date & Time
- 24-27 October 2023
### Location
- Thao Dien, District 2, Ho Chi Minh City
- Exact location to be shared later
## Agenda
To help us manage RSVPs, please use the Eventbrite links below to RSVP for each event.
### Schedule
| Day | Eventbrite Link | Signups |
| -------------- | -------------------------- | ------------------------------------------------------ |
| Mon (23 Oct) | Jan Team & Partners Dinner | Invite-only |
| Tues (24 Oct) | AI Talks Day 1 | [Eventbrite](https://jan-tech-talks-1.eventbrite.com) |
| Wed (25 Oct) | AI Talks Day 2 | [Eventbrite](https://jan-tech-talks-2.eventbrite.com) |
| Thurs (26 Oct) | VC Night | [Eventbrite](https://jan-hcmc-vc-night.eventbrite.com) |
| Fri (27 Oct) | Jan Launch Party | [Eventbrite](https://jan-launch-party.eventbrite.com) |
### OKRs
| **Objective** | Jan v1.0 should be bug-free and run on Windows, Max, Linux |
| ------------- | ---------------------------------------------------------- |
| Key Result | "Create Bot" feature w/ Saved Prompts |
| Key Result | *Stretch Goal:* Core Process API for Plugins |
| Key Result | *Stretch Goal:* UI API for Plugins |
## Photos
![](/img/hcmc-villa-2.jpeg)

2
docs/docs/hardware/community.md
View File

@ -1,5 +1,5 @@
---
title: Community Examples
title: Hardware Examples
---
## Add your own example

BIN
docs/docs/hardware/concepts/concepts-images/GPU.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 388 KiB

BIN
docs/docs/hardware/concepts/concepts-images/GPU_Image.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 945 KiB

BIN
docs/docs/hardware/concepts/concepts-images/PCIex16.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.2 MiB

BIN
docs/docs/hardware/concepts/concepts-images/Power.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 453 KiB

BIN
docs/docs/hardware/concepts/concepts-images/RAM-VRAM.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 349 KiB

BIN
docs/docs/hardware/concepts/concepts-images/VRAM-Image.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 553 KiB

BIN
docs/docs/hardware/concepts/concepts-images/slot.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 636 KiB

132
docs/docs/hardware/concepts/gpu-and-vram.md
View File

@ -2,7 +2,131 @@
title: GPUs and VRAM
---
- GPUs plugging in to Motherboard via PCIe
- Multiple GPUs
- NVLink
- PCIe (and Motherboard limitations)
## What Is a GPU?
A Graphics Card, or GPU (Graphics Processing Unit), is a fundamental component in modern computing. Think of it as the powerhouse behind rendering the stunning visuals you see on your screen. Similar to the motherboard in your computer, the graphics card is a printed circuit board. However, it's not just a passive piece of hardware; it's a sophisticated device equipped with essential components like fans, onboard RAM, a dedicated memory controller, BIOS, and various other features. If you want to learn more about GPUs then read here to [Understand the architecture of a GPU.](https://medium.com/codex/understanding-the-architecture-of-a-gpu-d5d2d2e8978b)
![GPU Image](concepts-images/GPU_Image.png)
## What Are GPUs Used For?
Two decades ago, GPUs primarily enhanced real-time 3D graphics in gaming. But as the 21st century dawned, a revelation occurred among computer scientists. They recognized that GPUs held untapped potential to solve some of the world's most intricate computing tasks.
This revelation marked the dawn of the general-purpose GPU era. Today's GPUs have evolved into versatile tools, more adaptable than ever before. They now have the capability to accelerate a diverse range of applications that stretch well beyond their original graphics-focused purpose.
### **Here are some example use cases:**
1. **Gaming**: They make games look good and run smoothly.
2. **Content Creation**: Help with video editing, 3D design, and graphics work.
3. **AI and Machine Learning**: Used for training smart machines.
4. **Science**: Speed up scientific calculations and simulations.
5. **Cryptocurrency Mining**: Mine digital currencies like Bitcoin.
6. **Medical Imaging**: Aid in analyzing medical images.
7. **Self-Driving Cars**: Help cars navigate autonomously.
8. **Simulations**: Create realistic virtual experiences.
9. **Data Analysis**: Speed up data processing and visualization.
10. **Video Streaming**: Improve video quality and streaming efficiency.
## What is VRAM In GPU?
VRAM, or video random-access memory, is a type of high-speed memory that is specifically designed for use with graphics processing units (GPUs). VRAM is used to store the textures, images, and other data that the GPU needs to render graphics. Its allows the GPU to access the data it needs quickly and efficiently. This is essential for rendering complex graphics at high frame rates.
VRAM is different from other types of memory, such as the system RAM that is used by the CPU. VRAM is optimized for high bandwidth and low latency, which means that it can read and write data very quickly. The amount of VRAM that a GPU has is one of the factors that determines its performance. More VRAM allows the GPU to store more data and render more complex graphics. However, VRAM is also one of the most expensive components of a GPU. So when choosing a graphics card, it is important to consider the amount of VRAM that it has. If you are planning on running demanding LLMs or video games, or 3D graphics software, you will need a graphics card with more VRAM.
![VRAM](concepts-images/VRAM-Image.png)
## What makes VRAM and RAM different from each other?
RAM (Random Access Memory) and VRAM (Video Random Access Memory) are both types of memory used in computers, but they have different functions and characteristics. Here are the differences between RAM and VRAM.
### RAM (Random Access Memory):
- RAM is a general-purpose memory that stores data and instructions that the CPU needs to access quickly.
- RAM is used for short-term data storage and is volatile, meaning that it loses its contents when the computer is turned off.
- RAM is connected to the motherboard and is accessed by the CPU.
- RAM typically has a larger capacity compared to VRAM, which is designed to store smaller amounts of data with faster access times.
- RAM stores data related to the operating system and the various programs that are running, including code, program files, and user data.
### VRAM (Video Random Access Memory):
- VRAM is a type of RAM that is specifically used to store image data for a computer display.
- VRAM is a graphics card component that is connected to the GPU (Graphics Processing Unit).
- VRAM is used exclusively by the GPU and doesn’t need to store as much data as the CPU.
- VRAM is similar to RAM in that it is volatile and loses its contents when the computer is turned off.
- VRAM stores data related specifically to graphics, such as textures, frames, and other graphical data.
- VRAM is designed to store smaller amounts of data with faster access times than RAM.
In summary, RAM is used for general-purpose memory, while VRAM is used for graphics-related tasks. RAM has a larger capacity and is accessed by the CPU, while VRAM has a smaller capacity and is accessed by the GPU.
**Key differences between VRAM and RAM:**
| Characteristic | VRAM | RAM |
| -------------- | --------------------- | --------------------- |
| Purpose | Graphics processing | General processing |
| Speed | Faster | Slower |
| Latency | Lower | Higher |
| Bandwidth | Higher | Lower |
| Cost | More expensive | Less expensive |
| Availability | Less widely available | More widely available |
![RAM-VRAM](concepts-images/RAM-VRAM.png)
## How to Connect GPU to the Motherboard via PCIe
Connecting hardware components to a motherboard is often likened to assembling LEGO pieces. If the parts fit together seamlessly, you're on the right track. Experienced PC builders find this process straightforward. However, for first-time builders, identifying where each hardware component belongs on the motherboard can be a bit perplexing.
**So follow the below 5 steps to Connect your GPU to the Motherboard:**
1. First, make sure your computer is powered off and unplugged from the electrical outlet to ensure safety.
2. Open your computer case if necessary to access the motherboard. Locate the PCIe x16 on the motherboard where you'll install the GPU. These slots are typically longer than other expansion slots and are used for graphics cards.
Remove Slot Covers (if applicable): Some PCIe slots may have protective covers or brackets covering them. Remove these covers by unscrewing them from the case using a Phillips-head screwdriver. And PCIe x16 will have plastic lock on one side only. There may be more than one PCIe x16 slot depending on the motherboard. You can use any of the slots according to your choice.
![PCIe x16](concepts-images/PCIex16.png)
3. Now Insert the Graphics Card slowly:
- Unlock the plastic lock on one side of the PCIe x16 slot by pulling it outwards.
![slot](concepts-images/slot.png)
- Align the PCIe slot with your graphics card, making sure that the HDMI port side of the GPU faces the rear side of the CPU case.
- Gently press on the card until you hear it securely snap in place.
![GPU](concepts-images/GPU.png)
4. Insert the Power Connector: If your GPU requires additional power (most modern GPUs do), connect the necessary power cables from your power supply to the GPU's power connectors. These connectors are usually located on the top or side of the GPU.
![Power](concepts-images/Power.png)
5. Power on the System: After turning on the PC see if the fans on your graphics card spin. If it does not spin, remove the power cable from the GPU, reconnect it, and power on the PC again.
> :memo: Note: To better understand you can also watch YouTube tutorials on how to Connect the GPU to the Motherboard via PCIe
## How to Choose a Graphics Card for your AI works
Selecting the optimal GPU for running Large Language Models (LLMs) on your home PC is a decision influenced by your budget and the specific LLMs you intend to work with. Your choice should strike a balance between performance, efficiency, and cost-effectiveness.
In general, the following GPU features are important for running LLMs:
- **High VRAM:** LLMs are typically very large and complex models, so they require a GPU with a high amount of VRAM. This will allow the model to be loaded into memory and processed efficiently.
- **CUDA Compatibility:** When running LLMs on a GPU, CUDA compatibility is paramount. CUDA is NVIDIA's parallel computing platform, and it plays a vital role in accelerating deep learning tasks. LLMs, with their extensive matrix calculations, heavily rely on parallel processing. Ensuring your GPU supports CUDA is like having the right tool for the job. It allows the LLM to leverage the GPU's parallel processing capabilities, significantly speeding up model training and inference.
- **Number of CUDA, Tensor, and RT Cores:** High-performance NVIDIA GPUs have both CUDA and Tensor cores. These cores are responsible for executing the neural network computations that underpin LLMs' language understanding and generation. The more CUDA cores your GPU has, the better equipped it is to handle the massive computational load that LLMs impose. Tensor cores in your GPU, further enhance LLM performance by accelerating the critical matrix operations integral to language modeling tasks.
- **Generation (Series)**: When selecting a GPU for LLMs, consider its generation or series (e.g., RTX 30 series). Newer GPU generations often come with improved architectures and features. For LLM tasks, opting for the latest generation can mean better performance, energy efficiency, and support for emerging AI technologies. Avoid purchasing, RTX-2000 series GPUs which are much outdated nowadays.
### Here are some of the best GPU options for this purpose:
1. **NVIDIA RTX 3090**: The NVIDIA RTX 3090 is a high-end GPU with a substantial 24GB of VRAM. This copious VRAM capacity makes it exceptionally well-suited for handling large LLMs. Moreover, it's known for its relative efficiency, meaning it won't overheat or strain your home PC's cooling system excessively. The RTX 3090's robust capabilities are a boon for those who need to work with hefty language models.
2. **NVIDIA RTX 4090**: If you're looking for peak performance and can afford the investment, the NVIDIA RTX 4090 represents the pinnacle of GPU power. Boasting 24GB of VRAM and featuring a cutting-edge Tensor Core architecture tailored for AI workloads, it outshines the RTX 3090 in terms of sheer capability. However, it's important to note that the RTX 4090 is also pricier and more power-hungry than its predecessor, the RTX 3090.
3. **AMD Radeon RX 6900 XT**: On the AMD side, the Radeon RX 6900 XT stands out as a high-end GPU with 16GB of VRAM. While it may not quite match the raw power of the RTX 3090 or RTX 4090, it strikes a balance between performance and affordability. Additionally, it tends to be more power-efficient, which could translate to a more sustainable and quieter setup in your home PC.
If budget constraints are a consideration, there are more cost-effective GPU options available:
- **NVIDIA RTX 3070**: The RTX 3070 is a solid mid-range GPU that can handle LLMs effectively. While it may not excel with the most massive or complex language models, it's a reliable choice for users looking for a balance between price and performance.
- **AMD Radeon RX 6800 XT**: Similarly, the RX 6800 XT from AMD offers commendable performance without breaking the bank. It's well-suited for running mid-sized LLMs and provides a competitive option in terms of both power and cost.
When selecting a GPU for LLMs, remember that it's not just about the GPU itself. Consider the synergy with other components in your PC:
- **CPU**: To ensure efficient processing, pair your GPU with a powerful CPU. LLMs benefit from fast processors, so having a capable CPU is essential.
- **RAM**: Sufficient RAM is crucial for LLMs. They can be memory-intensive, and having enough RAM ensures smooth operation.
- **Cooling System**: LLMs can push your PC's hardware to the limit. A robust cooling system helps maintain optimal temperatures, preventing overheating and performance throttling.
By taking all of these factors into account, you can build a home PC setup that's well-equipped to handle the demands of running LLMs effectively and efficiently.

3
docs/docs/hardware/examples/3090x1-@dan-jan.md
View File

@ -1,3 +0,0 @@
---
title: "@dan-jan: 3090 Desktop"
---

23
docs/docs/hardware/examples/4090x2-@dan-jan.md
View File

@ -1,3 +1,24 @@
---
title: "@janhq: 2x4090 Workstation"
title: "2 x 4090 Workstation"
---
![](/img/2x4090-workstation.png)
Jan uses a 2 x 4090 Workstation to run Codellama for internal use.[^1]
## Component List
| Type | Item | Unit Price | Total Price |
| :------------------- | :------------------------------------------------------------- | :--------- | ----------- |
| **CPU** | [Ryzen Threadripper Pro 5965WX 280W SP3 WOF](AMAZON-LINK-HERE) | $2,229 | |
| **Motherboard** | [Asus Pro WS WRX80E Sage SE WiFi](AMAZON-LINK-HERE) | $933 | |
| **RAM** | 4 x [G.Skill Ripjaw S5 2x32 6000C32](AMAZON-LINK-HERE) | $92.99 | |
| **GPU** | 2 x [Asus Strix RTX 4090 24GB OC](AMAZON-LINK-HERE) | $4,345 | |
| **Storage PCIe-SSD** | [Samsung 990 Pro 2TB NVME 2.0](AMAZON-LINK-HERE) | $134.99 | |
| **Cooler** | [BeQuiet Dark Rock 4 Pro TR4](AMAZON-LINK-HERE) | $89.90 | |
| **Power Supply** | [FSP Cannon 2000W Pro 92+ Full Modular PSU](AMAZON-LINK-HERE) | $449.99 | |
| **Case** | [Veddha 6GPUs Frame Black](AMAZON-LINK-HERE) | $59.99 | |
| **Total cost** | | $8,334 | |
[^1]: https://www.reddit.com/r/LocalLLaMA/comments/16lxt6a/case_for_dual_4090s/. ideb

4
docs/docs/hardware/hardware.md
View File

@ -1,6 +1,4 @@
---
sidebar_position: 1
title: Hardware
title: Introduction
---
TODO

3
docs/docs/hardware/overview/cloud-vs-buy.md
View File

@ -1,3 +0,0 @@
---
title: Cloud vs. Buy
---

62
docs/docs/hardware/overview/cloud-vs-self-hosting.md Normal file
View File

@ -0,0 +1,62 @@
---
title: Cloud vs. Self-hosting Your AI
---
The choice of how to run your AI - on GPU cloud services, on-prem, or just using an API provider - involves various trade-offs. The following is a naive exploration of the pros and cons of renting vs self-hosting.
## Cost Comparison
The following estimations use these general assumptions:
| | Self-Hosted | GPT 4.0 | GPU Rental |
| ---------- | ---------------------------------------- | -------------- | ------------------ |
| Unit Costs | $10k upfront for 2x4090s (5 year amort.) | $0.00012/token | $4.42 for 1xH100/h |
- 800 average tokens (input & output) in a single request
- Inference speed is at 24 tokens per second
### Low Usage
When operating at low capacity:
| | Self-Hosted | GPT 4.0 | GPU Rental |
| ---------------- | ----------- | ------- | ---------- |
| Cost per Request | $2.33 | $0.10 | $0.04 |
### High Usage
When operating at high capacity, i.e. 24 hours in a day, ~77.8k requests per month:
| | Self-Hosted | GPT 4.0 | GPU Rental |
| -------------- | ------------ | ------- | ---------- |
| Cost per Month | $166 (fixed) | $7465 | $3182 |
### Incremental Costs
Large context use cases are also interesting to evaluate. For example, if you had to write a 500 word essay summarizing Tolstoy's "War and Peace":
| | Self-Hosted | GPT 4.0 | GPU Rental |
| ----------------------- | -------------------- | ------- | ---------- |
| Cost of "War and Peace" | (upfront fixed cost) | $94 | $40 |
> **Takeaway**: Renting on cloud or using an API is great for initially scaling. However, it can quickly become expensive when dealing with large datasets and context windows. For predictable costs, self-hosting is an attractive option.
## Business Considerations
Other business level considerations may include:
| | Self-Hosted | GPT 4.0 | GPU Rental |
| ----------------------- | ----------- | ------- | ---------- |
| Data Privacy | ✅ | ❌ | ❌ |
| Offline Mode | ✅ | ❌ | ❌ |
| Customization & Control | ✅ | ❌ | ✅ |
| Auditing | ✅ | ❌ | ✅ |
| Setup Complexity | ❌ | ✅ | ✅ |
| Setup Cost | ❌ | ✅ | ✅ |
| Maintenance | ❌ | ✅ | ❌ |
## Conclusion
The decision to run LLMs in the cloud or on in-house servers is not one-size-fits-all. It depends on your business's specific needs, budget, and security considerations. Cloud-based LLMs offer scalability and cost-efficiency but come with potential security concerns, while in-house servers provide greater control, customization, and cost predictability.
In some situations, using a mix of cloud and in-house resources can be the best way to go. Businesses need to assess their needs and assets carefully to pick the right method for using LLMs in the ever-changing world of AI technology.

13
docs/docs/hardware/overview/cpu-vs-gpu.md
View File

@ -1,3 +1,14 @@
---
title: CPU vs. GPU
title: GPU vs CPU What's the Difference?
---
## CPU vs. GPU
| | CPU | GPU |
| ------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------- |
| **Function** | Generalized component that handles main processing functions of a server | Specialized component that excels at parallel computing |
| **Processing** | Designed for serial instruction processing | Designed for parallel instruction processing |
| **Design** | Fewer, more powerful cores | More cores than CPUs, but less powerful than CPU cores |
| **Best suited for** | General-purpose computing applications | High-performance computing applications |
![CPU VS GPU](https://media.discordapp.net/attachments/964896173401976932/1157998193741660222/CPU-vs-GPU-rendering.png?ex=651aa55b&is=651953db&hm=a22c80ed108a0d25106a20aa25236f7d0fa74167a50788194470f57ce7f4a6ca&=&width=807&height=426)

59
docs/docs/hardware/recommendations/by-budget.md
View File

@ -2,12 +2,61 @@
title: Recommended AI Hardware by Budget
---
## $1,000
> :warning: **Warning:** Do your own research before any purchase. Jan is not liable for compatibility, performance or other issues. Products can become outdated quickly.
## $2,500
## Entry-level PC Build at $1000
## $5,000
| Type | Item | Price |
| :------------------- | :--------------------------------------------------------- | :------- |
| **CPU** | [Intel Core i5 12400 2.5GHz 6-Core Processor](#) | $170.99 |
| **CPU Cooler** | [Intel Boxed Cooler (Included with CPU)](#) | Included |
| **Motherboard** | [ASUS Prime B660-PLUS DDR4 ATX LGA1700](#) | $169.95 |
| **GPU** | [Nvidia RTX 3050 8GB - ZOTAC Gaming Twin Edge](#) | $250 |
| **Memory** | [16GB (2 x 8GB) G.Skill Ripjaws V DDR4-3200 C16](#) | $49.99 |
| **Storage PCIe-SSD** | [ADATA XPG SX8200 Pro 512GB NVMe M.2 Solid State Drive](#) | $46.50 |
| **Power Supply** | [Corsair CX-M Series CX450M 450W ATX 2.4 Power Supply](#) | $89.99 |
| **Case** | [be quiet! Pure Base 600 Black ATX Mid Tower Case](#) | $97.00 |
| **Total cost** | | $870 |
## $7,500
## Entry-level PC Build at $1,500
## $10,000
| Type | Item | Price |
| :------------------- | :------------------------------------------------------- | :------ |
| **CPU** | [Intel Core i5 12600K 3.7GHz 6-Core Processor](#) | $269.99 |
| **CPU Cooler** | [be quiet! Dark Rock Pro 4](#) | $99.99 |
| **Motherboard** | [ASUS ProArt B660-Creator DDR4 ATX LGA1700](#) | $229.99 |
| **GPU** | [Nvidia RTX 3050 8GB - ZOTAC Gaming Twin Edge](#) | $349.99 |
| **Memory** | [32GB (2 x 16GB) G.Skill Ripjaws V DDR4-3200 C16](#) | $129.99 |
| **Storage PCIe-SSD** | [ADATA XPG SX8200 Pro 1TB NVMe M.2 Solid State Drive](#) | $109.99 |
| **Power Supply** | [Corsair RMx Series RM650x 650W ATX 2.4 Power Supply](#) | $119.99 |
| **Case** | [Corsair Carbide Series 200R ATX Mid Tower Case](#) | $59.99 |
| **Total cost** | | $1371 |
## Mid-range PC Build at $3000
| Type | Item | Price |
| :--------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :-------- |
| **CPU** | [AMD Ryzen 9 7950X 4.5 GHz 16-Core Processor](https://de.pcpartpicker.com/product/22XJ7P/amd-ryzen-9-7950x-45-ghz-16-core-processor-100-100000514wof) | $556 |
| **CPU Cooler** | [Thermalright Peerless Assassin 120 White 66.17 CFM CPU Cooler](https://de.pcpartpicker.com/product/476p99/thermalright-peerless-assassin-120-white-6617-cfm-cpu-cooler-pa120-white) | $59.99 |
| **Motherboard** | [Gigabyte B650 GAMING X AX ATX AM5 Motherboard](https://de.pcpartpicker.com/product/YZgFf7/gigabyte-b650-gaming-x-ax-atx-am5-motherboard-b650-gaming-x-ax) | $199.99 |
| **Memory** | [G.Skill Ripjaws S5 64 GB (2 x 32 GB) DDR5-6000 CL32 Memory](https://de.pcpartpicker.com/product/BJcG3C/gskill-ripjaws-s5-64-gb-2-x-32-gb-ddr5-6000-cl32-memory-f5-6000j3238g32gx2-rs5k) | $194 |
| **Storage** | [Crucial P5 Plus 2 TB M.2-2280 PCIe 4.0 X4 NVME Solid ](https://de.pcpartpicker.com/product/VZWzK8/crucial-p5-plus-2-tb-m2-2280-pcie-40-x4-nvme-solid-state-drive-ct2000p5pssd8) | $165.99 |
| **GPU** | [PNY XLR8 Gaming VERTO EPIC-X RGB OC GeForce RTX 4090 24 GB](https://de.pcpartpicker.com/product/TvpzK8/pny-xlr8-gaming-verto-epic-x-rgb-oc-geforce-rtx-4090-24-gb-video-card-vcg409024tfxxpb1-o) | $1,599.99 |
| **Case** | [Fractal Design Pop Air ATX Mid Tower Case](https://de.pcpartpicker.com/product/QnD7YJ/fractal-design-pop-air-atx-mid-tower-case-fd-c-poa1a-02) | $89.99 |
| **Power Supply** | [Thermaltake Toughpower GF A3 - TT Premium Edition 1050 W 80+ Gold](https://de.pcpartpicker.com/product/4v3NnQ/thermaltake-toughpower-gf-a3-1050-w-80-gold-certified-fully-modular-atx-power-supply-ps-tpd-1050fnfagu-l) | $139.99 |
| |
| **Total cost** | **$3000** |
## High-End PC Build at $6,000
| Type | Item | Price |
| :--------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------- |
| **CPU** | [AMD Ryzen 9 3900X 3.8 GHz 12-Core Processor](https://pcpartpicker.com/product/tLCD4D/amd-ryzen-9-3900x-36-ghz-12-core-processor-100-100000023box) | $365.00 |
| **CPU Cooler** | [Noctua NH-U12S chromax.black 55 CFM CPU Cooler](https://pcpartpicker.com/product/dMVG3C/noctua-nh-u12s-chromaxblack-55-cfm-cpu-cooler-nh-u12s-chromaxblack) | $89.95 |
| **Motherboard** | [Asus ProArt X570-CREATOR WIFI ATX AM4 Motherboard](https://pcpartpicker.com/product/8y8bt6/asus-proart-x570-creator-wifi-atx-am4-motherboard-proart-x570-creator-wifi) | $599.99 |
| **Memory** | [Corsair Vengeance LPX 128 GB (4 x 32 GB) DDR4-3200 CL16 Memory](https://pcpartpicker.com/product/tRH8TW/corsair-vengeance-lpx-128-gb-4-x-32-gb-ddr4-3200-memory-cmk128gx4m4e3200c16) | $249.99 |
| **Storage** | [Sabrent Rocket 4 Plus 2 TB M.2-2280 PCIe 4.0 X4 NVME Solid State Drive](https://pcpartpicker.com/product/PMBhP6/sabrent-rocket-4-plus-2-tb-m2-2280-nvme-solid-state-drive-sb-rkt4p-2tb) | $129.99 |
| **GPU** | [PNY RTX A-Series RTX A6000 48 GB Video Card](https://pcpartpicker.com/product/HWt9TW/pny-rtx-a-series-rtx-a6000-48-gb-video-card-vcnrtxa6000-pb) | $4269.00 |
| **Power Supply** | [EVGA SuperNOVA 850 G2 850 W 80+ Gold ](https://pcpartpicker.com/product/LCfp99/evga-supernova-850-g2-850-w-80-gold-certified-fully-modular-atx-power-supply-220-g2-0850-xr) | $322.42 |
| |
| **Total cost** | **$6026.34** |

179
docs/docs/hardware/recommendations/by-hardware.md
View File

@ -1,7 +1,182 @@
---
title: Recommended AI Models by Hardware
title: Selecting AI Hardware
---
When selecting a GPU for LLMs, remember that it's not just about the GPU itself. Consider the synergy with other components in your PC:
- **CPU**: To ensure efficient processing, pair your GPU with a powerful CPU. LLMs benefit from fast processors, so having a capable CPU is essential.
- **RAM**: Sufficient RAM is crucial for LLMs. They can be memory-intensive, and having enough RAM ensures smooth operation.
- **Cooling System**: LLMs can push your PC's hardware to the limit. A robust cooling system helps maintain optimal temperatures, preventing overheating and performance throttling.
By taking all of these factors into account, you can build a home PC setup that's well-equipped to handle the demands of running LLMs effectively and efficiently.
## GPU Selection
Selecting the optimal GPU for running Large Language Models (LLMs) on your home PC is a decision influenced by your budget and the specific LLMs you intend to work with. Your choice should strike a balance between performance, efficiency, and cost-effectiveness.
### GPU Comparison
| GPU | Price | Cores | VRAM (GB) | Bandwth (T/s) | Power |
| --------------------- | ----- | ----- | --------- | ------------- | ----- |
| Nvidia H100 | 40000 | 18432 | 80 | 2 | |
| Nvidia A100 | 15000 | 6912 | 80 | | |
| Nvidia A100 | 7015 | 6912 | 40 | | |
| Nvidia A10 | 2799 | 9216 | 24 | | |
| Nvidia RTX A6000 | 4100 | 10752 | 48 | 0.768 | |
| Nvidia RTX 6000 | 6800 | 4608 | 46 | | |
| Nvidia RTX 4090 Ti | 2000 | 18176 | 24 | | |
| Nvidia RTX 4090 | 1800 | 16384 | 24 | 1.008 | |
| Nvidia RTX 3090 | 1450 | 10496 | 24 | | |
| Nvidia RTX 3080 | 700 | 8704 | 12 | | |
| Nvidia RTX 3070 | 900 | 6144 | 8 | | |
| Nvidia L4 | 2711 | 7424 | 24 | | |
| Nvidia T4 | 2299 | 2560 | 16 | | |
| AMD Radeon RX 6900 XT | 1000 | 5120 | 16 | | |
| AMD Radeon RX 6800 XT | 420 | 4608 | 16 | | |
\*Market prices as of Oct 2023 via Amazon/PCMag
### Other Considerations
In general, the following GPU features are important for running LLMs:
- **High VRAM:** LLMs are typically very large and complex models, so they require a GPU with a high amount of VRAM. This will allow the model to be loaded into memory and processed efficiently.
- **CUDA Compatibility:** When running LLMs on a GPU, CUDA compatibility is paramount. CUDA is NVIDIA's parallel computing platform, and it plays a vital role in accelerating deep learning tasks. LLMs, with their extensive matrix calculations, heavily rely on parallel processing. Ensuring your GPU supports CUDA is like having the right tool for the job. It allows the LLM to leverage the GPU's parallel processing capabilities, significantly speeding up model training and inference.
- **Number of CUDA, Tensor, and RT Cores:** High-performance NVIDIA GPUs have both CUDA and Tensor cores. These cores are responsible for executing the neural network computations that underpin LLMs' language understanding and generation. The more CUDA cores your GPU has, the better equipped it is to handle the massive computational load that LLMs impose. Tensor cores in your GPU, further enhance LLM performance by accelerating the critical matrix operations integral to language modeling tasks.
- **Generation (Series)**: When selecting a GPU for LLMs, consider its generation or series (e.g., RTX 30 series). Newer GPU generations often come with improved architectures and features. For LLM tasks, opting for the latest generation can mean better performance, energy efficiency, and support for emerging AI technologies. Avoid purchasing, RTX-2000 series GPUs which are much outdated nowadays.
## CPU Selection
Selecting the right CPU for running Large Language Models (LLMs) on your home PC is contingent on your budget and the specific LLMs you intend to work with. It's a decision that warrants careful consideration, as the CPU plays a pivotal role in determining the overall performance of your system.
In general, the following CPU features are important for running LLMs:
- **Number of Cores and Threads:** the number of CPU cores and threads influences parallel processing. More cores and threads help handle the complex computations involved in language models. For tasks like training and inference, a higher core/thread count can significantly improve processing speed and efficiency, enabling quicker results.
- **High clock speed:** The base clock speed, or base frequency, represents the CPU's default operating speed. So having a CPU with a high clock speed. This will allow the model to process instructions more quickly, which can further improve performance.
- **Base Power (TDP):** LLMs often involve long training sessions and demanding computations. Therefore, a lower Thermal Design Power (TDP) is desirable. A CPU with a lower TDP consumes less power and generates less heat during prolonged LLM operations. This not only contributes to energy efficiency but also helps maintain stable temperatures in your system, preventing overheating and potential performance throttling.
- **Generation (Series):** Consider its generation or series (e.g., 9th Gen, 11th Gen Intel Core). Newer CPU generations often come with architectural improvements that enhance performance and efficiency. For LLM tasks, opting for a more recent generation can lead to faster and more efficient language model training and inference.
- **Support for AVX512:** AVX512 is a set of vector instruction extensions that can be used to accelerate machine learning workloads. Many LLMs are optimized to take advantage of AVX512, so it is important to make sure that your CPU supports this instruction set.
### Here are some CPU options for running LLMs:
1. **Intel Core i7-12700K**: Slightly less potent than the Core i9-12900K, the Intel Core i7-12700K is still a powerful CPU. With 12 cores and 20 threads, it strikes a balance between performance and cost-effectiveness. This CPU is well-suited for running mid-sized and large LLMs, making it a compelling option.
2. **Intel Core i9-12900K**: Positioned as a high-end CPU, the Intel Core i9-12900K packs a formidable punch with its 16 cores and 24 threads. It's one of the fastest CPUs available, making it an excellent choice for handling large and intricate LLMs. The abundance of cores and threads translates to exceptional parallel processing capabilities, which is crucial for tasks involving massive language models.
3. **AMD Ryzen 9 5950X**: Representing AMD's high-end CPU offering, the Ryzen 9 5950X boasts 16 cores and 32 threads. While it may not quite match the speed of the Core i9-12900K, it remains a robust and cost-effective choice. Its multicore prowess enables smooth handling of LLM workloads, and its affordability makes it an attractive alternative.
4. **AMD Ryzen 7 5800X**: Slightly less potent than the Ryzen 9 5950X, the Ryzen 7 5800X is still a formidable CPU with 8 cores and 16 threads. It's well-suited for running mid-sized and smaller LLMs, providing a compelling blend of performance and value.
For those operating within budget constraints, there are more budget-friendly CPU options:
- **Intel Core i5-12600K**: The Core i5-12600K is a capable mid-range CPU that can still handle LLMs effectively, though it may not be optimized for the largest or most complex models.
- **AMD Ryzen 5 5600X**: The Ryzen 5 5600X offers a balance of performance and affordability. It's suitable for running smaller to mid-sized LLMs without breaking the bank.
**When selecting a CPU for LLMs, consider the synergy with other components in your PC:**
- **GPU**: Pair your CPU with a powerful GPU to ensure smooth processing of LLMs. Some language models, particularly those used for AI, rely on GPU acceleration for optimal performance.
- **RAM**: Adequate RAM is essential for LLMs, as these models can be memory-intensive. Having enough RAM ensures that your CPU can operate efficiently without bottlenecks.
- **Cooling System**: Given the resource-intensive nature of LLMs, a robust cooling system is crucial to maintain optimal temperatures and prevent performance throttling.
By carefully weighing your budget and performance requirements and considering the interplay of components in your PC, you can assemble a well-rounded system that's up to the task of running LLMs efficiently.
> :memo: **Note:** It is important to note that these are just general recommendations. The specific CPU requirements for your LLM will vary depending on the specific model you are using and the tasks that you want to perform with it. If you are unsure what CPU to get, it is best to consult with an expert.
## RAM Selection
The amount of RAM you need to run an LLM depends on the size and complexity of the model, as well as the tasks you want to perform with it. For example, if you are simply running inference on a pre-trained LLM, you may be able to get away with using a relatively modest amount of RAM. However, if you are training a new LLM from scratch, or if you are running complex tasks like fine-tuning or code generation, you will need more RAM.
### Here is a general guide to RAM selection for running LLMs:
- **Capacity:** The amount of RAM you need will depend on the size and complexity of the LLM model you want to run. For inference, you will need at least 16GB of RAM, but 32GB or more is ideal for larger models and more complex tasks. For training, you will need at least 64GB of RAM, but 128GB or more is ideal for larger models and more complex tasks.
- **Speed:** LLMs can benefit from having fast RAM, so it is recommended to use DDR4 or DDR5 RAM with a speed of at least 3200MHz.
- **Latency:** RAM latency is the amount of time it takes for the CPU to access data in memory. Lower latency is better for performance, so it is recommended to look for RAM with a low latency rating.
- **Timing:** RAM timing is a set of parameters that control how the RAM operates. It is important to make sure that the RAM timing is compatible with your motherboard and CPU.
R**ecommended RAM** **options for running LLMs:**
- **Inference:** For inference on pre-trained LLMs, you will need at least 16GB of RAM. However, 32GB or more is ideal for larger models and more complex tasks.
- **Training:** For training LLMs from scratch, you will need at least 64GB of RAM. However, 128GB or more is ideal for larger models and more complex tasks.
In addition to the amount of RAM, it is also important to consider the speed of the RAM. LLMs can benefit from having fast RAM, so it is recommended to use DDR4 or DDR5 RAM with a speed of at least 3200MHz.
## Motherboard Selection
When picking a motherboard to run advanced language models, you need to think about a few things. First, consider the specific language model you want to use, the type of CPU and GPU in your computer, and your budget. Here are some suggestions:
1. **ASUS ROG Maximus Z790 Hero:** This is a top-notch motherboard with lots of great features. It works well with Intel's latest CPUs, fast DDR5 memory, and PCIe 5.0 devices. It's also good at keeping things cool, which is important for running demanding language models.
2. **MSI MEG Z790 Ace:** Similar to the ASUS ROG Maximus, this motherboard is high-end and has similar features. It's good for running language models too.
3. **Gigabyte Z790 Aorus Master:** This one is more budget-friendly but still works great with Intel's latest CPUs, DDR5 memory, and fast PCIe 5.0 devices. It's got a strong power system, which helps with running language models.
If you're on a tighter budget, you might want to check out mid-range options like the **ASUS TUF Gaming Z790-Plus WiFi** or the **MSI MPG Z790 Edge WiFi DDR5**. They offer good performance without breaking the bank.
No matter which motherboard you pick, make sure it works with your CPU and GPU. Also, check that it has the features you need, like enough slots for your GPU and storage drives.
Other things to think about when choosing a motherboard for language models:
- **Cooling:** Language models can make your CPU work hard, so a motherboard with good cooling is a must. This keeps your CPU from getting too hot.
- **Memory:** Language models need lots of memory, so make sure your motherboard supports a good amount of it. Check if it works with the type of memory you want to use, like DDR5 or DDR4.
- **Storage:** Language models can create and store a ton of data. So, look for a motherboard with enough slots for your storage drives.
- **BIOS:** The BIOS controls your motherboard. Make sure it's up-to-date and has the latest features, especially if you plan to overclock or undervolt your system.
## Cooling System Selection
Modern computers have two critical components, the CPU and GPU, which can heat up during high-performance tasks. To prevent overheating, they come with built-in temperature controls that automatically reduce performance when temperatures rise. To keep them cool and maintain optimal performance, you need a reliable cooling system.
For laptops, the only choice is a fan-based cooling system. Laptops have built-in fans and copper pipes to dissipate heat. Many gaming laptops even have two separate fans: one for the CPU and another for the GPU.
For desktop computers, you have the option to install more efficient water cooling systems. These are highly effective but can be expensive. Or you can install more cooling fans to keep you components cool.
Keep in mind that dust can accumulate in fan-based cooling systems, leading to malfunctions. So periodically clean the dust to keep your cooling system running smoothly.
## Use MacBook to run LLMs
An Apple MacBook equipped with either the M1 or the newer M2 Pro/Max processor. These cutting-edge chips leverage Apple's innovative Unified Memory Architecture (UMA), which revolutionizes the way the CPU and GPU interact with memory resources. This advancement plays a pivotal role in enhancing the performance and capabilities of LLMs.
Unified Memory Architecture, as implemented in Apple's M1 and M2 series processors, facilitates seamless and efficient data access for both the CPU and GPU. Unlike traditional systems where data needs to be shuttled between various memory pools, UMA offers a unified and expansive memory pool that can be accessed by both processing units without unnecessary data transfers. This transformative approach significantly minimizes latency while concurrently boosting data access bandwidth, resulting in substantial improvements in both the speed and quality of outputs.
![UMA](https://media.discordapp.net/attachments/1148534242104574012/1156600109967089714/IMG_3722.webp?ex=6516380a&is=6514e68a&hm=ebe3b6ecb1edb44cde58bd8d3fdd46cef66b60aa41ea6c03b51325fa65f8517e&=&width=807&height=426)
The M1 and M2 Pro/Max chips offer varying levels of unified memory bandwidth, further underscoring their prowess in handling data-intensive tasks like AI processing. The M1/M2 Pro chip boasts an impressive capacity of up to 200 GB/s of unified memory bandwidth, while the M1/M2 Max takes it a step further, supporting up to a staggering 400 GB/s of unified memory bandwidth. This means that regardless of the complexity and demands of the AI tasks at hand, these Apple laptops armed with M1 or M2 processors are well-equipped to handle them with unparalleled efficiency and speed.
## Calculating vRAM Requirements for an LLM
**For example:** Calculating the VRAM required to run a 13-billion-parameter Large Language Model (LLM) involves considering the model size, batch size, sequence length, token size, and any additional overhead. Here's how you can estimate the VRAM required for a 13B LLM:
1. **Model Size**: Find out the size of the 13B LLM in terms of the number of parameters. This information is typically provided in the model's documentation. A 13-billion-parameter model has 13,000,000,000 parameters.
2. **Batch Size**: Decide on the batch size you want to use during inference. The batch size represents how many input samples you process simultaneously. Smaller batch sizes require less VRAM.
3. **Sequence Length**: Determine the average length of the input text sequences you'll be working with. Sequence length can impact VRAM requirements; longer sequences need more memory.
4. **Token Size**: Understand the memory required to store one token in bytes. Most LLMs use 4 bytes per token.
5. **Overhead**: Consider any additional memory overhead for intermediate computations and framework requirements. Overhead can vary but should be estimated based on your specific setup.
Use the following formula to estimate the VRAM required:
**VRAM Required (in gigabytes)** = `Model Parameters x Token Size x Batch Size x Sequence Length + Overhead`
- **Model Parameters**: 13,000,000,000 parameters for a 13B LLM.
- **Token Size**: Usually 4 bytes per token.
- **Batch Size**: Choose your batch size.
- **Sequence Length**: The average length of input sequences.
- **Overhead**: Any additional VRAM required based on your setup.
Here's an example:
Suppose you want to run a 13B LLM with the following parameters:
- **Batch Size**: 4
- **Sequence Length**: 512 tokens
- **Token Size**: 4 bytes
- **Estimated Overhead**: 2 GB
VRAM Required (in gigabytes) = `(13,000,000,000 x 4 x 4 x 512) + 2`
VRAM Required (in gigabytes) = `(8,388,608,000) + 2,000`
VRAM Required (in gigabytes) ≈ `8,390,608,000 bytes`
To convert this to gigabytes, divide by `1,073,741,824 (1 GB)`
VRAM Required (in gigabytes) ≈ `8,390,608,000 / 1,073,741,824 ≈ 7.8 GB`
So, to run a 13-billion-parameter LLM with the specified parameters and overhead, you would need approximately 7.8 gigabytes of VRAM on your GPU. Make sure to have some additional VRAM for stable operation and consider testing the setup in practice to monitor VRAM usage accurately.
<!--
## Macbook 8GB RAM
## Macbook 16GB RAM
## Macbook 16GB RAM -->

62
docs/docs/hardware/recommendations/by-model.md
View File

@ -4,5 +4,65 @@ title: Recommended AI Hardware by Model
## Codellama 34b
## Falcon 180b
### System Requirements:
**For example**: If you want to use [Codellama 7B](https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GPTQ/tree/main) models on your own computer, you can take advantage of your GPU and run this with GPTQ file models.
GPTQ is a format that compresses the model parameters to 4-bit, which reduces the VRAM requirements significantly. You can use the [oobabooga webui](https://github.com/oobabooga/text-generation-webui) or [JanAI](https://jan.ai/), which are simple interfaces that let you interact with different LLMS on your browser. It is pretty easy to set up and run. You can install it on Windows or Linux. (linked it to our installation page)
**For 7B Parameter Models (4-bit Quantization)**
| Format | RAM Requirements | VRAM Requirements | Minimum recommended GPU |
| ------------------------------------------------ | -------------------- | ----------------- | ----------------------------------------- |
| GPTQ (GPU inference) | 6GB (Swap to Load\*) | 6GB | GTX 1660, 2060,RTX 3050, 3060 AMD 5700 XT |
| GGML / GGUF (CPU inference) | 4GB | 300MB | |
| Combination of GPTQ and GGML / GGUF (offloading) | 2GB | 2GB | |
**For 13B Parameter Models (4-bit Quantization)**
| Format | RAM Requirements | VRAM Requirements | Minimum recommended GPU |
| ------------------------------------------------ | --------------------- | ----------------- | -------------------------------------------------- |
| GPTQ (GPU inference) | 12GB (Swap to Load\*) | 10GB | |
| GGML / GGUF (CPU inference) | 8GB | 500MB | AMD 6900 XT, RTX 2060 12GB, 3060 12GB, 3080, A2000 |
| Combination of GPTQ and GGML / GGUF (offloading) | 10GB | 10GB | |
**For 34B Parameter Models (4-bit Quantization)**
| Format | RAM Requirements | VRAM Requirements | Minimum recommended GPU |
| ------------------------------------------------ | --------------------- | ----------------- | -------------------------------------------------------------------- |
| GPTQ (GPU inference) | 32GB (Swap to Load\*) | 20GB | |
| GGML / GGUF (CPU inference) | 20GB | 500MB | RTX 3080 20GB, A4500, A5000, 3090, 4090, 6000, Tesla V100, Tesla P40 |
| Combination of GPTQ and GGML / GGUF (offloading) | 10GB | 4GB | |
**For 7B Parameter Models (8-bit Quantization)**
| Format | RAM Requirements | VRAM Requirements | Minimum recommended GPU |
| ------------------------------------------------ | --------------------- | ----------------- | -------------------------------------- |
| GPTQ (GPU inference) | 24GB (Swap to Load\*) | 12GB | RTX 3080, RTX 3080 Ti, RTX 3090, A5000 |
| GGML / GGUF (CPU inference) | 16GB | 1GB | RTX 3060 12GB, RTX 3070, A2000 |
| Combination of GPTQ and GGML / GGUF (offloading) | 12GB | 4GB | RTX 3060, RTX 3060 Ti, A2000 |
**For 13B Parameter Models (8-bit Quantization)**
| Format | RAM Requirements | VRAM Requirements | Minimum recommended GPU |
| ------------------------------------------------ | --------------------- | ----------------- | --------------------------------- |
| GPTQ (GPU inference) | 36GB (Swap to Load\*) | 20GB | RTX 4090, A6000, A6000 Ti, A8000 |
| GGML / GGUF (CPU inference) | 24GB | 2GB | RTX 3080 20GB, RTX 3080 Ti, A5000 |
| Combination of GPTQ and GGML / GGUF (offloading) | 20GB | 8GB | RTX 3080, RTX 3080 Ti, A5000 |
**For 34B Parameter Models (8-bit Quantization)**
| Format | RAM Requirements | VRAM Requirements | Minimum recommended GPU |
| ------------------------------------------------ | --------------------- | ----------------- | -------------------------------- |
| GPTQ (GPU inference) | 64GB (Swap to Load\*) | 40GB | A8000, A8000 Ti, A9000 |
| GGML / GGUF (CPU inference) | 40GB | 2GB | RTX 4090, A6000, A6000 Ti, A8000 |
| Combination of GPTQ and GGML / GGUF (offloading) | 48GB | 20GB | RTX 4090, A6000, A6000 Ti, A8000 |
> :memo: **Note**: System RAM, not VRAM, required to load the model, in addition to having enough VRAM. Not required to run the model. You can use swap space if you do not have enough RAM.
### Performance Recommendations:
1. **Optimal Performance**: To achieve the best performance when working with CodeLlama models, consider investing in a high-end GPU such as NVIDIA's latest RTX 3090 or RTX 4090. For the largest models like the 65B and 70B, a dual GPU setup is recommended. Additionally, ensure your system boasts sufficient RAM, with a minimum of 16 GB, although 64 GB is ideal for seamless operation.
2. **Budget-Friendly Approach**: If budget constraints are a concern, focus on utilizing CodeLlama GGML/GGUF models that can comfortably fit within your system's available RAM. Keep in mind that while you can allocate some model weights to the system RAM to save GPU memory, this may result in a performance trade-off.
> :memo: **Note**: It's essential to note that these recommendations are guidelines, and the actual performance you experience will be influenced by various factors. These factors include the specific task you're performing, the implementation of the model, and the concurrent system processes. To optimize your setup, consider these recommendations as a starting point and adapt them to your unique requirements and constraints.

18
docs/docs/hardware/recommendations/by-usecase.md
View File

@ -2,19 +2,21 @@
title: Recommended AI Hardware by Use Case
---
## Personal Use
## Which AI Hardware to Choose Based on Your Use Case
### Entry-level Experimentation
Artificial intelligence (AI) is rapidly changing the world, and AI hardware is becoming increasingly important for businesses and individuals alike. Choosing the right hardware for your AI needs is crucial to get the best performance and results. Here are some tips for selecting AI hardware based on your specific use case and requirements.
### Personal Use
### Entry-level Experimentation:
- Macbook (16gb)
- 3090
**Personal Use:**
When venturing into the world of AI as an individual, your choice of hardware can significantly impact your experience. Here's a more detailed breakdown:
### Prosumer Use
- **Macbook (16GB):** A Macbook equipped with 16GB of RAM and either the M1 or the newer M2 Pro/Max processor is an excellent starting point for AI enthusiasts. These cutting-edge chips leverage Apple's innovative Unified Memory Architecture (UMA), which revolutionizes the way the CPU and GPU interact with memory resources. This advancement plays a pivotal role in enhancing the performance and capabilities of LLMs.
- **Nvidia GeForce RTX 3090:** This powerful graphics card is a solid alternative for AI beginners, offering exceptional performance for basic experiments.
- Apple Silicon
- 2 x 3090 (48gb RAM)
2. **Serious AI Work:**
- **2 x 3090 RTX Card (48GB RAM):** For those committed to more advanced AI projects, this configuration provides the necessary muscle. Its dual Nvidia GeForce RTX 3090 GPUs and ample RAM make it suitable for complex AI tasks and model training.
## Business Use

2
docs/docs/solutions/self-hosted.md
View File

@ -1,3 +1,3 @@
---
title: Self-hosted ChatGPT
title: Self-Hosted ChatGPT
---

4
docs/docusaurus.config.js
View File

@ -127,7 +127,7 @@ const config = {
type: "docSidebar",
sidebarId: "solutionsSidebar",
position: "left",
label: "Use Cases",
label: "Solutions",
},
{
type: "docSidebar",
@ -175,7 +175,7 @@ const config = {
to: "/platform",
},
{
label: "Use Cases",
label: "Solutions",
to: "/solutions",
},
],

983
docs/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

8
docs/package.json
View File

@ -14,14 +14,16 @@
"write-heading-ids": "docusaurus write-heading-ids"
},
"dependencies": {
"@docusaurus/core": "2.4.1",
"@docusaurus/preset-classic": "2.4.1",
"@docusaurus/theme-live-codeblock": "^2.4.1",
"@docusaurus/core": "^2.4.3",
"@docusaurus/preset-classic": "^2.4.3",
"@docusaurus/theme-live-codeblock": "^2.4.3",
"@headlessui/react": "^1.7.17",
"@heroicons/react": "^2.0.18",
"@mdx-js/react": "^1.6.22",
"autoprefixer": "^10.4.16",
"axios": "^1.5.1",
"clsx": "^1.2.1",
"js-yaml": "^4.1.0",
"postcss": "^8.4.30",
"prism-react-renderer": "^1.3.5",
"react": "^17.0.2",

238
docs/sidebars.js
View File

@ -43,28 +43,28 @@ const sidebars = {
// Note: Tab name is "Use Cases"
solutionsSidebar: [
"solutions/solutions",
// "solutions/solutions",
{
type: "category",
label: "Use cases",
label: "Solutions",
collapsible: true,
collapsed: false,
items: ["solutions/personal-ai", "solutions/self-hosted"],
},
{
type: "category",
label: "Industries",
collapsible: true,
collapsed: false,
items: [
"solutions/industries/software",
"solutions/industries/education",
"solutions/industries/law",
"solutions/industries/public-sector",
"solutions/industries/finance",
"solutions/industries/healthcare",
],
items: ["solutions/self-hosted", "solutions/personal-ai"],
},
// {
// type: "category",
// label: "Industries",
// collapsible: true,
// collapsed: false,
// items: [
// "solutions/industries/software",
// "solutions/industries/education",
// "solutions/industries/law",
// "solutions/industries/public-sector",
// "solutions/industries/finance",
// "solutions/industries/healthcare",
// ],
// },
],
docsSidebar: [
@ -83,101 +83,101 @@ const sidebars = {
],
hardwareSidebar: [
{
type: "category",
label: "Overview",
collapsible: true,
collapsed: true,
link: { type: "doc", id: "hardware/hardware" },
items: [
{
type: "doc",
label: "Cloud vs. Buy",
id: "hardware/overview/cloud-vs-buy",
},
{
type: "doc",
label: "CPUs vs. GPUs",
id: "hardware/overview/cpu-vs-gpu",
},
],
},
{
type: "category",
label: "Recommendations",
collapsible: true,
collapsed: false,
items: [
{
type: "doc",
label: "By Hardware",
id: "hardware/recommendations/by-hardware",
},
{
type: "doc",
label: "By Budget",
id: "hardware/recommendations/by-budget",
},
{
type: "doc",
label: "By Model",
id: "hardware/recommendations/by-model",
},
{
type: "doc",
label: "By Use Case",
id: "hardware/recommendations/by-usecase",
},
],
},
{
type: "category",
label: "Anatomy of a Thinking Machine",
collapsible: true,
collapsed: true,
link: { type: "doc", id: "hardware/concepts/concepts" },
items: [
{
type: "doc",
label: "Chassis",
id: "hardware/concepts/chassis",
},
{
type: "doc",
label: "Motherboard",
id: "hardware/concepts/motherboard",
},
{
type: "doc",
label: "CPU and RAM",
id: "hardware/concepts/cpu-and-ram",
},
{
type: "doc",
label: "GPU and VRAM",
id: "hardware/concepts/gpu-and-vram",
},
{
type: "doc",
label: "Storage",
id: "hardware/concepts/storage",
},
{
type: "doc",
label: "Network",
id: "hardware/concepts/network",
},
// {
// type: "category",
// label: "Overview",
// collapsible: true,
// collapsed: true,
// link: { type: "doc", id: "hardware/hardware" },
// items: [
// {
// type: "doc",
// label: "Cloud vs. Self-Hosting",
// id: "hardware/overview/cloud-vs-self-hosting",
// },
// {
// type: "doc",
// label: "CPUs vs. GPUs",
// id: "hardware/overview/cpu-vs-gpu",
// },
// ],
// },
// {
// type: "category",
// label: "Recommendations",
// collapsible: true,
// collapsed: false,
// items: [
// {
// type: "doc",
// label: "By Hardware",
// id: "hardware/recommendations/by-hardware",
// },
// {
// type: "doc",
// label: "By Budget",
// id: "hardware/recommendations/by-budget",
// },
// {
// type: "doc",
// label: "By Model",
// id: "hardware/recommendations/by-model",
// },
// {
// type: "doc",
// label: "By Use Case",
// id: "hardware/recommendations/by-usecase",
// },
// ],
// },
// {
// type: "category",
// label: "Anatomy of a Thinking Machine",
// collapsible: true,
// collapsed: true,
// link: { type: "doc", id: "hardware/concepts/concepts" },
// items: [
// {
// type: "doc",
// label: "Chassis",
// id: "hardware/concepts/chassis",
// },
// {
// type: "doc",
// label: "Motherboard",
// id: "hardware/concepts/motherboard",
// },
// {
// type: "doc",
// label: "CPU and RAM",
// id: "hardware/concepts/cpu-and-ram",
// },
// {
// type: "doc",
// label: "GPU and VRAM",
// id: "hardware/concepts/gpu-and-vram",
// },
// {
// type: "doc",
// label: "Storage",
// id: "hardware/concepts/storage",
// },
// {
// type: "doc",
// label: "Network",
// id: "hardware/concepts/network",
// },
{
type: "doc",
label: "Power Supply",
id: "hardware/concepts/power",
},
],
},
// {
// type: "doc",
// label: "Power Supply",
// id: "hardware/concepts/power",
// },
// ],
// },
{
type: "category",
label: "Community Examples",
label: "Hardware Examples",
collapsible: true,
collapsed: true,
link: { type: "doc", id: "hardware/community" },
@ -194,23 +194,35 @@ const sidebars = {
type: "category",
label: "About Jan",
collapsible: true,
collapsed: false,
collapsed: true,
link: { type: "doc", id: "about/about" },
items: [
"about/roadmap",
"about/team",
{
type: "link",
label: "Careers",
href: "https://janai.bamboohr.com/careers",
},
"about/brand-assets",
],
},
{
type: "category",
label: "Events",
collapsible: true,
collapsed: true,
items: [
{
type: "doc",
label: "Ho Chi Minh City (Oct 2023)",
id: "events/hcmc-oct23",
},
],
},
{
type: "category",
label: "Company Handbook",
collapsible: true,
collapsed: false,
collapsed: true,
link: { type: "doc", id: "handbook/handbook" },
items: ["handbook/remote-work"],
},

96
docs/src/components/Elements/dropdown.js
View File

@ -1,28 +1,29 @@
import React from "react";
import React, { useState, useEffect } from "react";
import { Fragment } from "react";
import { Menu, Transition } from "@headlessui/react";
import { ChevronDownIcon } from "@heroicons/react/20/solid";
import axios from "axios";
const items = [
const systemsTemplate = [
{
name: "Download for Mac (M1/M2)",
href: "https://github.com/janhq/jan/releases/download/v0.1.2/Jan-0.1.2-arm64.dmg",
logo: require("@site/static/img/apple-logo-white.png").default,
fileFormat: "{appname}-mac-arm64-{tag}.dmg",
},
{
name: "Download for Mac (Intel)",
href: "https://github.com/janhq/jan/releases/download/v0.1.2/Jan-0.1.2-arm64.dmg",
logo: require("@site/static/img/apple-logo-white.png").default,
fileFormat: "{appname}-mac-x64-{tag}.dmg",
},
{
name: "Download for Windows",
href: "https://static.vecteezy.com/system/resources/previews/004/243/615/non_2x/creative-coming-soon-teaser-background-free-vector.jpg",
logo: require("@site/static/img/windows-logo-white.png").default,
fileFormat: "{appname}-win-x64-{tag}.exe",
},
{
name: "Download for Linux",
href: "https://static.vecteezy.com/system/resources/previews/004/243/615/non_2x/creative-coming-soon-teaser-background-free-vector.jpg",
logo: require("@site/static/img/linux-logo-white.png").default,
fileFormat: "{appname}-linux-amd64-{tag}.deb",
},
];
@ -31,22 +32,81 @@ function classNames(...classes) {
}
export default function Dropdown() {
const [systems, setSystems] = useState(systemsTemplate);
const [defaultSystem, setDefaultSystem] = useState(systems[0]);
const getLatestReleaseInfo = async (repoOwner, repoName) => {
const url = `https://api.github.com/repos/${repoOwner}/${repoName}/releases/latest`;
try {
const response = await axios.get(url);
return response.data;
} catch (error) {
console.error(error);
return null;
}
};
const extractAppName = (fileName) => {
// Extract appname using a regex that matches the provided file formats
const regex = /^(.*?)-(?:mac|win|linux)-(?:arm64|x64|amd64)-.*$/;
const match = fileName.match(regex);
return match ? match[1] : null;
};
useEffect(() => {
const updateDownloadLinks = async () => {
try {
const releaseInfo = await getLatestReleaseInfo("janhq", "jan");
// Extract appname from the first asset name
const firstAssetName = releaseInfo.assets[0].name;
const appname = extractAppName(firstAssetName);
if (!appname) {
console.error("Failed to extract appname from file name:", firstAssetName);
return;
}
// Remove 'v' at the start of the tag_name
const tag = releaseInfo.tag_name.startsWith("v")
? releaseInfo.tag_name.substring(1)
: releaseInfo.tag_name;
const updatedSystems = systems.map((system) => {
const downloadUrl = system.fileFormat
.replace("{appname}", appname)
.replace("{tag}", tag);
return {
...system,
href: `https://github.com/janhq/jan/releases/download/${releaseInfo.tag_name}/${downloadUrl}`,
};
});
setSystems(updatedSystems);
setDefaultSystem(updatedSystems[0]);
} catch (error) {
console.error("Failed to update download links:", error);
}
};
updateDownloadLinks();
}, []);
return (
<div className="inline-flex align-items-stretch">
{/* TODO dynamically detect users OS through browser */}
<a
className="cursor-pointer relative inline-flex items-center rounded-l-md border-0 px-3.5 py-2.5 text-base font-semibold text-white bg-indigo-600 dark:bg-indigo-500 hover:bg-indigo-500 dark:hover:bg-indigo-400 hover:text-white"
href={items[0].href}
className="cursor-pointer relative inline-flex items-center rounded-l-md border-0 px-3.5 py-2.5 text-base font-semibold text-white bg-blue-600 dark:bg-blue-500 hover:bg-blue-500 dark:hover:bg-blue-400 hover:text-white"
href={defaultSystem.href}
>
<img
src={require("@site/static/img/apple-logo-white.png").default}
alt="Logo"
className="h-5 mr-3 -mt-1"
/>
Download for Mac (Silicon)
{defaultSystem.name}
</a>
<Menu as="div" className="relative -ml-px block">
<Menu.Button className="cursor-pointer relative inline-flex items-center rounded-r-md border-0 border-l border-gray-300 active:border-l active:border-white h-full text-white bg-indigo-600 dark:bg-indigo-500 hover:bg-indigo-500 dark:hover:bg-indigo-400">
<Menu.Button className="cursor-pointer relative inline-flex items-center rounded-r-md border-0 border-l border-gray-300 active:border-l active:border-white h-full text-white bg-blue-600 dark:bg-blue-500 hover:bg-blue-500 dark:hover:bg-blue-400">
<span className="sr-only">Open OS options</span>
<ChevronDownIcon className="h-5 w-5" aria-hidden="true" />
</Menu.Button>
@ -59,26 +119,26 @@ export default function Dropdown() {
leaveFrom="transform opacity-100 scale-100"
leaveTo="transform opacity-0 scale-95"
>
<Menu.Items className="absolute right-0 z-10 mt-2 w-72 text-left origin-top-right rounded-md bg-indigo-600 dark:bg-indigo-500 shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none">
<Menu.Items className="absolute right-0 z-10 mt-2 w-72 text-left origin-top-right rounded-md bg-blue-600 dark:bg-blue-500 shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none">
<div className="py-1">
{items.map((item) => (
<Menu.Item key={item.name}>
{systems.map((system) => (
<Menu.Item key={system.name}>
{({ active }) => (
<a
href={item.href}
href={system.href}
className={classNames(
active
? "bg-indigo-500 dark:hover:bg-indigo-400 hover:text-white"
? "bg-blue-500 dark:hover:bg-blue-400 hover:text-white"
: "text-white",
"block px-4 py-2"
)}
>
<img
src={item.logo}
src={system.logo}
alt="Logo"
className="w-3 mr-3 -mt-1"
/>
{item.name}
{system.name}
</a>
)}
</Menu.Item>

14
docs/src/components/Homepage/downloads.js
View File

@ -5,7 +5,7 @@ import {
LockClosedIcon,
} from "@heroicons/react/20/solid";
const features = [
const systems = [
{
name: "Mac",
description:
@ -47,20 +47,20 @@ export default function HomepageDownloads() {
</div>
<div className="mx-auto mt-16 max-w-2xl sm:mt-20 lg:mt-24 lg:max-w-none">
<dl className="grid max-w-xl grid-cols-1 gap-x-8 gap-y-16 lg:max-w-none lg:grid-cols-3">
{features.map((feature) => (
<div key={feature.name} className="flex flex-col">
{systems.map((system) => (
<div key={system.name} className="flex flex-col">
<dt className="flex items-center gap-x-3 text-base font-semibold leading-7 text-gray-900 dark: text-white">
<feature.icon
<system.icon
className="h-5 w-5 flex-none text-indigo-600 dark:text-indigo-400"
aria-hidden="true"
/>
{feature.name}
{system.name}
</dt>
<dd className="mt-4 flex flex-auto flex-col text-base leading-7 text-gray-600 dark:text-gray-300">
<p className="flex-auto">{feature.description}</p>
<p className="flex-auto">{system.description}</p>
<p className="mt-6">
<a
href={feature.href}
href={system.href}
className="text-sm font-semibold leading-6 text-indigo-600 dark:text-indigo-400"
>
Learn more <span aria-hidden="true">→</span>

15
docs/src/components/Homepage/hero.js
View File

@ -8,7 +8,7 @@ export default function HomepageHero() {
return (
<div className="bg-white dark:bg-gray-900">
<div className="relative isolate pt-14">
<div className="relative isolate md:pt-14 pt-0">
{/* Background top gradient styling */}
{colorMode === "dark" ? (
<div
@ -39,7 +39,7 @@ export default function HomepageHero() {
)}
{/* Main hero block */}
<div className="py-24 sm:py-32 lg:pb-40 animate-in fade-in zoom-in-50 duration-1000 ">
<div className="py-24 lg:pb-40 animate-in fade-in zoom-in-50 duration-1000 ">
<div className="mx-auto max-w-7xl px-6 lg:px-8">
{/* Hero text and buttons */}
<div className="mx-auto max-w-2xl text-center">
@ -47,7 +47,7 @@ export default function HomepageHero() {
Run your own AI
</h1>
<p className="mt-6 text-lg leading-8 text-gray-600 dark:text-gray-300">
Jan lets you run AI on your own hardware. 1-click to install the
Jan lets you run AI on your own hardware. 1-click to install the
latest open-source models. Monitor and manage software-hardware
performance.
<br></br>
@ -60,7 +60,7 @@ export default function HomepageHero() {
<Dropdown />
<button
type="button"
className="cursor-pointer relative inline-flex items-center rounded px-3.5 py-2 dark:py-2.5 text-base font-semibold text-indigo-600 bg-white border-indigo-600 dark:border-0 hover:bg-indigo-600 dark:hover:bg-indigo-500 hover:text-white"
className="cursor-pointer relative inline-flex items-center rounded px-3.5 py-2 dark:py-2.5 text-base font-semibold text-blue-600 bg-white border-blue-600 dark:border-0 hover:bg-blue-600 dark:hover:bg-blue-500 hover:text-white"
onClick={() =>
window.open(
"https://github.com/janhq/jan",
@ -79,14 +79,15 @@ export default function HomepageHero() {
src={
colorMode === "dark"
? // TODO replace with darkmode image
require("@site/static/img/desktop-llm-chat-dark.png").default
: require("@site/static/img/desktop-llm-chat-light.png").default
require("@site/static/img/desktop-llm-chat-dark.png")
.default
: require("@site/static/img/desktop-llm-chat-light.png")
.default
}
alt="App screenshot"
width={2432}
className="mt-16 rounded-lg md:rounded-2xl lg:rounded-3xl bg-white/5 shadow-2xl ring-1 ring-white/10 sm:mt-24"
/>
</div>
</div>
{/* Background top gradient styling */}

12
docs/src/css/custom.css
View File

@ -25,7 +25,7 @@
/* For readability concerns, you should choose a lighter palette in dark mode. */
[data-theme="dark"] {
--ifm-color-primary: #2563EB; /* New Primary Blue */
--ifm-color-primary: #ffffff; /* New Primary Blue */
--ifm-color-primary-dark: #204FCF; /* Darker Blue */
--ifm-color-primary-darker: #1B45B7; /* Even Darker Blue */
--ifm-color-primary-darkest: #163C9D; /* Darkest Blue */
@ -34,3 +34,13 @@
--ifm-color-primary-lightest: #3A8BFF; /* Lightest Blue */
--docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
}
.footer.footer--dark {
--ifm-footer-background-color: #1a212f;
--ifm-footer-color: var(--ifm-footer-link-color);
--ifm-footer-link-color: var(--ifm-color-secondary);
--ifm-footer-title-color: var(--ifm-color-white);
background-color: var(--ifm-footer-background-color);
color: var(--ifm-footer-color)
}

BIN
docs/static/img/2x4090-workstation.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.1 MiB

BIN
docs/static/img/github-readme-banner.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 184 KiB

BIN
docs/static/img/hcmc-villa-1.jpeg vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 228 KiB

BIN
docs/static/img/hcmc-villa-2.jpeg vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 332 KiB

1178
docs/yarn.lock
View File

File diff suppressed because it is too large Load Diff

8
electron/.eslintrc.js
View File

@ -34,5 +34,11 @@ module.exports = {
{ name: "Link", linkAttribute: "to" },
],
},
ignorePatterns: ["renderer/*", "node_modules/*", "core/plugins"],
ignorePatterns: [
"build",
"renderer",
"node_modules",
"core/plugins",
"core/**/*.test.js",
],
};

5
electron/auto-sign.sh Executable file
View File

@ -0,0 +1,5 @@
#!/bin/bash
DEVELOPER_ID="Developer ID Application: Eigenvector Pte Ltd"
find electron -type f -perm +111 -exec codesign -s "Developer ID Application: Eigenvector Pte Ltd (YT49P7GXG4)" --options=runtime {} \;

14
electron/core/plugin-manager/execution/Plugin.js
View File

@ -16,11 +16,23 @@ class Plugin {
/** @type {boolean} Whether this plugin should be activated when its activation points are triggered. */
active
constructor(name, url, activationPoints, active) {
/** @type {string} Plugin's description. */
description
/** @type {string} Plugin's version. */
version
/** @type {string} Plugin's logo. */
icon
constructor(name, url, activationPoints, active, description, version, icon) {
this.name = name
this.url = url
this.activationPoints = activationPoints
this.active = active
this.description = description
this.version = version
this.icon = icon
}
/**

12
electron/core/plugin-manager/execution/facade.js
View File

@ -25,6 +25,7 @@ export async function install(plugins) {
if (typeof window === "undefined") {
return;
}
// eslint-disable-next-line no-undef
const plgList = await window.pluggableElectronIpc.install(plugins);
if (plgList.cancelled) return false;
return plgList.map((plg) => {
@ -50,6 +51,7 @@ export function uninstall(plugins, reload = true) {
if (typeof window === "undefined") {
return;
}
// eslint-disable-next-line no-undef
return window.pluggableElectronIpc.uninstall(plugins, reload);
}
@ -62,6 +64,7 @@ export async function getActive() {
if (typeof window === "undefined") {
return;
}
// eslint-disable-next-line no-undef
const plgList = await window.pluggableElectronIpc.getActive();
return plgList.map(
(plugin) =>
@ -69,7 +72,10 @@ export async function getActive() {
plugin.name,
plugin.url,
plugin.activationPoints,
plugin.active
plugin.active,
plugin.description,
plugin.version,
plugin.icon
)
);
}
@ -83,6 +89,7 @@ export async function registerActive() {
if (typeof window === "undefined") {
return;
}
// eslint-disable-next-line no-undef
const plgList = await window.pluggableElectronIpc.getActive();
plgList.forEach((plugin) =>
register(
@ -107,6 +114,7 @@ export async function update(plugins, reload = true) {
if (typeof window === "undefined") {
return;
}
// eslint-disable-next-line no-undef
const plgList = await window.pluggableElectronIpc.update(plugins, reload);
return plgList.map(
(plugin) =>
@ -129,6 +137,7 @@ export function updatesAvailable(plugin) {
if (typeof window === "undefined") {
return;
}
// eslint-disable-next-line no-undef
return window.pluggableElectronIpc.updatesAvailable(plugin);
}
@ -143,6 +152,7 @@ export async function toggleActive(plugin, active) {
if (typeof window === "undefined") {
return;
}
// eslint-disable-next-line no-undef
const plg = await window.pluggableElectronIpc.toggleActive(plugin, active);
return new Plugin(plg.name, plg.url, plg.activationPoints, plg.active);
}

1
electron/core/plugin-manager/execution/index.js
View File

@ -5,6 +5,7 @@ export * as activationPoints from "./activation-manager.js";
export * as plugins from "./facade.js";
export { default as ExtensionPoint } from "./ExtensionPoint.js";
// eslint-disable-next-line no-undef
if (typeof window !== "undefined" && !window.pluggableElectronIpc)
console.warn(
"Facade is not registered in preload. Facade functions will throw an error if used."

24
electron/core/plugin-manager/facade/index.js
View File

@ -1,30 +1,32 @@
import { ipcRenderer, contextBridge } from "electron"
const { ipcRenderer, contextBridge } = require("electron");
export default function useFacade() {
function useFacade() {
const interfaces = {
install(plugins) {
return ipcRenderer.invoke('pluggable:install', plugins)
return ipcRenderer.invoke("pluggable:install", plugins);
},
uninstall(plugins, reload) {
return ipcRenderer.invoke('pluggable:uninstall', plugins, reload)
return ipcRenderer.invoke("pluggable:uninstall", plugins, reload);
},
getActive() {
return ipcRenderer.invoke('pluggable:getActivePlugins')
return ipcRenderer.invoke("pluggable:getActivePlugins");
},
update(plugins, reload) {
return ipcRenderer.invoke('pluggable:update', plugins, reload)
return ipcRenderer.invoke("pluggable:update", plugins, reload);
},
updatesAvailable(plugin) {
return ipcRenderer.invoke('pluggable:updatesAvailable', plugin)
return ipcRenderer.invoke("pluggable:updatesAvailable", plugin);
},
toggleActive(plugin, active) {
return ipcRenderer.invoke('pluggable:togglePluginActive', plugin, active)
return ipcRenderer.invoke("pluggable:togglePluginActive", plugin, active);
},
}
};
if (contextBridge) {
contextBridge.exposeInMainWorld('pluggableElectronIpc', interfaces)
contextBridge.exposeInMainWorld("pluggableElectronIpc", interfaces);
}
return interfaces
return interfaces;
}
module.exports = useFacade;

4
electron/core/plugin-manager/pluginMgr/Plugin.js
View File

@ -18,6 +18,8 @@ class Plugin {
* @property {string} version Version of the package as defined in the manifest.
* @property {Array<string>} activationPoints List of {@link ./Execution-API#activationPoints|activation points}.
* @property {string} main The entry point as defined in the main entry of the manifest.
* @property {string} description The description of plugin as defined in the manifest.
* @property {string} icon The icon of plugin as defined in the manifest.
*/
/** @private */
@ -75,6 +77,8 @@ class Plugin {
this.version = mnf.version
this.activationPoints = mnf.activationPoints || null
this.main = mnf.main
this.description = mnf.description
this.icon = mnf.icon
} catch (error) {
throw new Error(`Package ${this.origin} does not contain a valid manifest: ${error}`)

8
electron/core/plugins/data-plugin/config/tsconfig.cjs.json Normal file
View File

@ -0,0 +1,8 @@
{
"extends": "./../tsconfig.json",
"compilerOptions": {
"outDir": "./../dist/cjs",
"module": "commonjs"
},
"files": ["../module.ts"]
}

8
electron/core/plugins/data-plugin/config/tsconfig.esm.json Normal file
View File

@ -0,0 +1,8 @@
{
"extends": "./../tsconfig.json",
"compilerOptions": {
"outDir": "./../dist/esm",
"module": "esnext"
},
"files": ["../index.ts"]
}

327
electron/core/plugins/data-plugin/index.ts
View File

@ -1,168 +1,187 @@
import { core, store, RegisterExtensionPoint, StoreService, DataService } from "@janhq/plugin-core";
// Provide an async method to manipulate the price provided by the extension point
const MODULE_PATH = "data-plugin/dist/module.js";
const MODULE_PATH = "data-plugin/dist/cjs/module.js";
const storeModel = (model: any) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "storeModel", model)
.then((res: any) => resolve(res));
}
});
/**
* Create a collection on data store
*
* @param name name of the collection to create
* @param schema schema of the collection to create, include fields and their types
* @returns Promise<void>
*
*/
function createCollection({ name, schema }: { name: string; schema?: { [key: string]: any } }): Promise<void> {
console.log("renderer: creating collection:", name, schema);
return core.invokePluginFunc(MODULE_PATH, "createCollection", name, schema);
}
const getFinishedDownloadModels = () =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "getFinishedDownloadModels")
.then((res: any) => resolve(res));
}
});
/**
* Delete a collection
*
* @param name name of the collection to delete
* @returns Promise<void>
*
*/
function deleteCollection(name: string): Promise<void> {
return core.invokePluginFunc(MODULE_PATH, "deleteCollection", name);
}
const getModelById = (modelId: string) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "getModelById", modelId)
.then((res: any) => resolve(res));
}
});
/**
* Insert a value to a collection
*
* @param collectionName name of the collection
* @param value value to insert
* @returns Promise<any>
*
*/
function insertOne({ collectionName, value }: { collectionName: string; value: any }): Promise<any> {
return core.invokePluginFunc(MODULE_PATH, "insertOne", collectionName, value);
}
const updateFinishedDownloadAt = (fileName: string) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(
MODULE_PATH,
"updateFinishedDownloadAt",
fileName,
Date.now()
)
.then((res: any) => resolve(res));
}
});
/**
* Update value of a collection's record
*
* @param collectionName name of the collection
* @param key key of the record to update
* @param value value to update
* @returns Promise<void>
*
*/
function updateOne({ collectionName, key, value }: { collectionName: string; key: string; value: any }): Promise<void> {
return core.invokePluginFunc(MODULE_PATH, "updateOne", collectionName, key, value);
}
const getUnfinishedDownloadModels = () =>
new Promise<any>((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "getUnfinishedDownloadModels")
.then((res: any[]) => resolve(res));
} else {
resolve([]);
}
});
/**
* Updates all records that match a selector in a collection in the data store.
* @param collectionName - The name of the collection containing the records to update.
* @param selector - The selector to use to get the records to update.
* @param value - The new value for the records.
* @returns {Promise<void>} A promise that resolves when the records are updated.
*/
function updateMany({
collectionName,
value,
selector,
}: {
collectionName: string;
value: any;
selector?: { [key: string]: any };
}): Promise<void> {
return core.invokePluginFunc(MODULE_PATH, "updateMany", collectionName, value, selector);
}
const deleteDownloadModel = (modelId: string) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "deleteDownloadModel", modelId)
.then((res: any) => resolve(res));
}
});
/**
* Delete a collection's record
*
* @param collectionName name of the collection
* @param key key of the record to delete
* @returns Promise<void>
*
*/
function deleteOne({ collectionName, key }: { collectionName: string; key: string }): Promise<void> {
return core.invokePluginFunc(MODULE_PATH, "deleteOne", collectionName, key);
}
const getConversations = () =>
new Promise<any>((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "getConversations")
.then((res: any[]) => resolve(res));
} else {
resolve([]);
}
});
const getConversationMessages = (id: any) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "getConversationMessages", id)
.then((res: any[]) => resolve(res));
} else {
resolve([]);
}
});
/**
* Deletes all records with a matching key from a collection in the data store.
*
* @param collectionName name of the collection
* @param selector selector to use to get the records to delete.
* @returns {Promise<void>}
*
*/
function deleteMany({
collectionName,
selector,
}: {
collectionName: string;
selector?: { [key: string]: any };
}): Promise<void> {
return core.invokePluginFunc(MODULE_PATH, "deleteMany", collectionName, selector);
}
const createConversation = (conversation: any) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "storeConversation", conversation)
.then((res: any) => {
resolve(res);
});
} else {
resolve(undefined);
}
});
const createMessage = (message: any) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "storeMessage", message)
.then((res: any) => {
resolve(res);
});
} else {
resolve(undefined);
}
});
/**
* Retrieve a record from a collection in the data store.
* @param {string} collectionName - The name of the collection containing the record to retrieve.
* @param {string} key - The key of the record to retrieve.
* @returns {Promise<any>} A promise that resolves when the record is retrieved.
*/
function findOne({ collectionName, key }: { collectionName: string; key: string }): Promise<any> {
return core.invokePluginFunc(MODULE_PATH, "findOne", collectionName, key);
}
const updateMessage = (message: any) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "updateMessage", message)
.then((res: any) => {
resolve(res);
});
} else {
resolve(undefined);
}
});
/**
* Gets records in a collection in the data store using a selector.
* @param {string} collectionName - The name of the collection containing the record to get the value from.
* @param {{ [key: string]: any }} selector - The selector to use to get the value from the record.
* @param {[{ [key: string]: any }]} sort - The sort options to use to retrieve records.
* @returns {Promise<any>} A promise that resolves with the selected value.
*/
function findMany({
collectionName,
selector,
sort,
}: {
collectionName: string;
selector: { [key: string]: any };
sort?: [{ [key: string]: any }];
}): Promise<any> {
return core.invokePluginFunc(MODULE_PATH, "findMany", collectionName, selector, sort);
}
const deleteConversation = (id: any) =>
new Promise((resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "deleteConversation", id)
.then((res: any) => {
resolve(res);
});
} else {
resolve("-");
}
});
const setupDb = () => {
window.electronAPI.invokePluginFunc(MODULE_PATH, "init");
};
function onStart() {
createCollection({ name: "conversations", schema: {} });
createCollection({ name: "messages", schema: {} });
}
// Register all the above functions and objects with the relevant extension points
export function init({ register }: { register: any }) {
setupDb();
register("getConversations", "getConv", getConversations, 1);
register("createConversation", "insertConv", createConversation);
register("updateMessage", "updateMessage", updateMessage);
register("deleteConversation", "deleteConv", deleteConversation);
register("createMessage", "insertMessage", createMessage);
register("getConversationMessages", "getMessages", getConversationMessages);
register("storeModel", "storeModel", storeModel);
register(
"updateFinishedDownloadAt",
"updateFinishedDownloadAt",
updateFinishedDownloadAt
);
register(
"getUnfinishedDownloadModels",
"getUnfinishedDownloadModels",
getUnfinishedDownloadModels
);
register("deleteDownloadModel", "deleteDownloadModel", deleteDownloadModel);
register("getModelById", "getModelById", getModelById);
register(
"getFinishedDownloadModels",
"getFinishedDownloadModels",
getFinishedDownloadModels
);
export function init({ register }: { register: RegisterExtensionPoint }) {
onStart();
register(StoreService.CreateCollection, createCollection.name, createCollection);
register(StoreService.DeleteCollection, deleteCollection.name, deleteCollection);
register(StoreService.InsertOne, insertOne.name, insertOne);
register(StoreService.UpdateOne, updateOne.name, updateOne);
register(StoreService.UpdateMany, updateMany.name, updateMany);
register(StoreService.DeleteOne, deleteOne.name, deleteOne);
register(StoreService.DeleteMany, deleteMany.name, deleteMany);
register(StoreService.FindOne, findOne.name, findOne);
register(StoreService.FindMany, findMany.name, findMany);
register(DataService.GetConversations, getConversations.name, getConversations);
register(DataService.CreateConversation, createConversation.name, createConversation);
register(DataService.UpdateConversation, updateConversation.name, updateConversation);
register(DataService.UpdateMessage, updateMessage.name, updateMessage);
register(DataService.DeleteConversation, deleteConversation.name, deleteConversation);
register(DataService.CreateMessage, createMessage.name, createMessage);
register(DataService.GetConversationMessages, getConversationMessages.name, getConversationMessages);
}
function getConversations(): Promise<any> {
return store.findMany("conversations", {}, [{ updatedAt: "desc" }]);
}
function createConversation(conversation: any): Promise<number | undefined> {
return store.insertOne("conversations", conversation);
}
function updateConversation(conversation: any): Promise<void> {
return store.updateOne("conversations", conversation._id, conversation);
}
function createMessage(message: any): Promise<number | undefined> {
return store.insertOne("messages", message);
}
function updateMessage(message: any): Promise<void> {
return store.updateOne("messages", message._id, message);
}
function deleteConversation(id: any) {
return store.deleteOne("conversations", id).then(() => store.deleteMany("messages", { conversationId: id }));
}
function getConversationMessages(conversationId: any) {
return store.findMany("messages", { conversationId }, [{ createdAt: "desc" }]);
}

593
electron/core/plugins/data-plugin/module.ts
View File

@ -1,416 +1,239 @@
const sqlite3 = require("sqlite3").verbose();
const path = require("path");
const { app } = require("electron");
var PouchDB = require("pouchdb-node");
PouchDB.plugin(require("pouchdb-find"));
var path = require("path");
var { app } = require("electron");
var fs = require("fs");
const MODEL_TABLE_CREATION = `
CREATE TABLE IF NOT EXISTS models (
id TEXT PRIMARY KEY,
slug TEXT NOT NULL,
name TEXT NOT NULL,
description TEXT NOT NULL,
avatar_url TEXT,
long_description TEXT NOT NULL,
technical_description TEXT NOT NULL,
author TEXT NOT NULL,
version TEXT NOT NULL,
model_url TEXT NOT NULL,
nsfw INTEGER NOT NULL,
greeting TEXT NOT NULL,
type TEXT NOT NULL,
file_name TEXT NOT NULL,
download_url TEXT NOT NULL,
start_download_at INTEGER DEFAULT -1,
finish_download_at INTEGER DEFAULT -1,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);`;
const MODEL_TABLE_INSERTION = `
INSERT INTO models (
id,
slug,
name,
description,
avatar_url,
long_description,
technical_description,
author,
version,
model_url,
nsfw,
greeting,
type,
file_name,
download_url,
start_download_at
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`;
function init() {
const db = new sqlite3.Database(path.join(app.getPath("userData"), "jan.db"));
console.log(
`Database located at ${path.join(app.getPath("userData"), "jan.db")}`
);
db.serialize(() => {
db.run(MODEL_TABLE_CREATION);
db.run(
"CREATE TABLE IF NOT EXISTS conversations ( id INTEGER PRIMARY KEY, name TEXT, model_id TEXT, image TEXT, message TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP);"
);
db.run(
"CREATE TABLE IF NOT EXISTS messages ( id INTEGER PRIMARY KEY, name TEXT, conversation_id INTEGER, user TEXT, message TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP);"
);
});
const stmt = db.prepare(
"INSERT INTO conversations (name, model_id, image, message) VALUES (?, ?, ?, ?)"
);
stmt.finalize();
db.close();
}
const dbs: Record<string, any> = {};
/**
* Store a model in the database when user start downloading it
* Create a collection on data store
*
* @param model Product
*/
function storeModel(model: any) {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
console.debug("Inserting", JSON.stringify(model));
db.serialize(() => {
const stmt = db.prepare(MODEL_TABLE_INSERTION);
stmt.run(
model.id,
model.slug,
model.name,
model.description,
model.avatarUrl,
model.longDescription,
model.technicalDescription,
model.author,
model.version,
model.modelUrl,
model.nsfw,
model.greeting,
model.type,
model.fileName,
model.downloadUrl,
Date.now(),
function (err: any) {
if (err) {
// Handle the insertion error here
console.error(err.message);
res(undefined);
return;
}
// @ts-ignoreF
const id = this.lastID;
res(id);
return;
}
);
stmt.finalize();
});
db.close();
});
}
/**
* Update the finished download time of a model
* @param name name of the collection to create
* @param schema schema of the collection to create, include fields and their types
* @returns Promise<void>
*
* @param model Product
*/
function updateFinishedDownloadAt(fileName: string, time: number) {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
console.debug(`Updating fileName ${fileName} to ${time}`);
const stmt = `UPDATE models SET finish_download_at = ? WHERE file_name = ?`;
db.run(stmt, [time, fileName], (err: any) => {
if (err) {
console.log(err);
} else {
console.log("Updated 1 row");
res("Updated");
}
});
db.close();
function createCollection(name: string, schema?: { [key: string]: any }): Promise<void> {
return new Promise<void>((resolve) => {
const dbPath = path.join(app.getPath("userData"), "databases");
if (!fs.existsSync(dbPath)) fs.mkdirSync(dbPath);
const db = new PouchDB(`${path.join(dbPath, name)}`);
dbs[name] = db;
resolve();
});
}
/**
* Get all unfinished models from the database
* Delete a collection
*
* @param name name of the collection to delete
* @returns Promise<void>
*
*/
function getUnfinishedDownloadModels() {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
function deleteCollection(name: string): Promise<void> {
// Do nothing with Unstructured Database
return dbs[name].destroy();
}
const query = `SELECT * FROM models WHERE finish_download_at = -1 ORDER BY start_download_at DESC`;
db.all(query, (err: Error, row: any) => {
res(row);
/**
* Insert a value to a collection
*
* @param collectionName name of the collection
* @param value value to insert
* @returns Promise<any>
*
*/
function insertOne(collectionName: string, value: any): Promise<any> {
if (!value._id) return dbs[collectionName].post(value).then((doc) => doc.id);
return dbs[collectionName].put(value).then((doc) => doc.id);
}
/**
* Update value of a collection's record
*
* @param collectionName name of the collection
* @param key key of the record to update
* @param value value to update
* @returns Promise<void>
*
*/
function updateOne(collectionName: string, key: string, value: any): Promise<void> {
console.debug(`updateOne ${collectionName}: ${key} - ${JSON.stringify(value)}`);
return dbs[collectionName].get(key).then((doc) => {
return dbs[collectionName].put({
_id: key,
_rev: doc._rev,
...value,
},
{ force: true });
}).then((res: any) => {
console.info(`updateOne ${collectionName} result: ${JSON.stringify(res)}`);
}).catch((err: any) => {
console.error(`updateOne ${collectionName} error: ${err}`);
});
}
/**
* Update value of a collection's records
*
* @param collectionName name of the collection
* @param selector selector of records to update
* @param value value to update
* @returns Promise<void>
*
*/
function updateMany(collectionName: string, value: any, selector?: { [key: string]: any }): Promise<any> {
// Creates keys from selector for indexing
const keys = selector ? Object.keys(selector) : [];
// At a basic level, there are two steps to running a query: createIndex()
// (to define which fields to index) and find() (to query the index).
return (
keys.length > 0
? dbs[collectionName].createIndex({
// There is selector so we need to create index
index: { fields: keys },
})
: Promise.resolve()
) // No selector, so no need to create index
.then(() =>
dbs[collectionName].find({
// Find documents using Mango queries
selector,
})
)
.then((data) => {
const docs = data.docs.map((doc) => {
// Update doc with new value
return (doc = {
...doc,
...value,
});
});
return dbs[collectionName].bulkDocs(docs);
});
db.close();
});
}
function getFinishedDownloadModels() {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
const query = `SELECT * FROM models WHERE finish_download_at != -1 ORDER BY finish_download_at DESC`;
db.all(query, (err: Error, row: any) => {
res(row.map((item: any) => parseToProduct(item)));
});
db.close();
});
/**
* Delete a collection's record
*
* @param collectionName name of the collection
* @param key key of the record to delete
* @returns Promise<void>
*
*/
function deleteOne(collectionName: string, key: string): Promise<void> {
return findOne(collectionName, key).then((doc) => dbs[collectionName].remove(doc));
}
function deleteDownloadModel(modelId: string) {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
console.log(`Deleting ${modelId}`);
db.serialize(() => {
const stmt = db.prepare("DELETE FROM models WHERE id = ?");
stmt.run(modelId);
stmt.finalize();
res(modelId);
});
/**
* Delete a collection records by selector
*
* @param {string} collectionName name of the collection
* @param {{ [key: string]: any }} selector selector for retrieving records.
* @returns Promise<void>
*
*/
function deleteMany(collectionName: string, selector?: { [key: string]: any }): Promise<void> {
// Creates keys from selector for indexing
const keys = selector ? Object.keys(selector) : [];
db.close();
});
}
function getModelById(modelId: string) {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
console.debug("Get model by id", modelId);
db.get(
`SELECT * FROM models WHERE id = ?`,
[modelId],
(err: any, row: any) => {
console.debug("Get model by id result", row);
if (row) {
const product = {
id: row.id,
slug: row.slug,
name: row.name,
description: row.description,
avatarUrl: row.avatar_url,
longDescription: row.long_description,
technicalDescription: row.technical_description,
author: row.author,
version: row.version,
modelUrl: row.model_url,
nsfw: row.nsfw,
greeting: row.greeting,
type: row.type,
inputs: row.inputs,
outputs: row.outputs,
createdAt: new Date(row.created_at),
updatedAt: new Date(row.updated_at),
fileName: row.file_name,
downloadUrl: row.download_url,
};
res(product);
}
}
);
db.close();
});
}
function getConversations() {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
db.all(
"SELECT * FROM conversations ORDER BY updated_at DESC",
(err: any, row: any) => {
res(row);
}
);
db.close();
});
}
function storeConversation(conversation: any): Promise<number | undefined> {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
db.serialize(() => {
const stmt = db.prepare(
"INSERT INTO conversations (name, model_id, image, message) VALUES (?, ?, ?, ?)"
// At a basic level, there are two steps to running a query: createIndex()
// (to define which fields to index) and find() (to query the index).
return (
keys.length > 0
? dbs[collectionName].createIndex({
// There is selector so we need to create index
index: { fields: keys },
})
: Promise.resolve()
) // No selector, so no need to create index
.then(() =>
dbs[collectionName].find({
// Find documents using Mango queries
selector,
})
)
.then((data) => {
return Promise.all(
// Remove documents
data.docs.map((doc) => {
return dbs[collectionName].remove(doc);
})
);
stmt.run(
conversation.name,
conversation.model_id,
conversation.image,
conversation.message,
function (err: any) {
if (err) {
// Handle the insertion error here
console.error(err.message);
res(undefined);
return;
}
// @ts-ignoreF
const id = this.lastID;
res(id);
return;
}
);
stmt.finalize();
});
db.close();
});
}
function storeMessage(message: any): Promise<number | undefined> {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
db.serialize(() => {
const stmt = db.prepare(
"INSERT INTO messages (name, conversation_id, user, message) VALUES (?, ?, ?, ?)"
);
stmt.run(
message.name,
message.conversation_id,
message.user,
message.message,
function (err: any) {
if (err) {
// Handle the insertion error here
console.error(err.message);
res(undefined);
return;
}
//@ts-ignore
const id = this.lastID;
res(id);
return;
}
);
stmt.finalize();
});
db.close();
});
}
function updateMessage(message: any): Promise<number | undefined> {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
db.serialize(() => {
const stmt = db.prepare(
"UPDATE messages SET message = ?, updated_at = ? WHERE id = ?"
);
stmt.run(message.message, message.updated_at, message.id);
stmt.finalize();
res(message.id);
});
db.close();
});
/**
* Retrieve a record from a collection in the data store.
* @param {string} collectionName - The name of the collection containing the record to retrieve.
* @param {string} key - The key of the record to retrieve.
* @returns {Promise<any>} A promise that resolves when the record is retrieved.
*/
function findOne(collectionName: string, key: string): Promise<any> {
return dbs[collectionName].get(key).catch(() => undefined);
}
function deleteConversation(id: any) {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
/**
* Gets records in a collection in the data store using a selector.
* @param {string} collectionName - The name of the collection containing records to retrieve.
* @param {{ [key: string]: any }} selector - The selector to use to retrieve records.
* @param {[{ [key: string]: any }]} sort - The sort options to use to retrieve records.
* @returns {Promise<any>} A promise that resolves with the selected records.
*/
function findMany(
collectionName: string,
selector?: { [key: string]: any },
sort?: [{ [key: string]: any }]
): Promise<any> {
const keys = selector ? Object.keys(selector) : [];
const sortKeys = sort ? sort.flatMap((e) => (e ? Object.keys(e) : undefined)) : [];
db.serialize(() => {
const deleteConv = db.prepare("DELETE FROM conversations WHERE id = ?");
deleteConv.run(id);
deleteConv.finalize();
const deleteMessages = db.prepare(
"DELETE FROM messages WHERE conversation_id = ?"
);
deleteMessages.run(id);
deleteMessages.finalize();
res(id);
});
db.close();
// Note that we are specifying that the field must be greater than or equal to null
// which is a workaround for the fact that the Mango query language requires us to have a selector.
// In CouchDB collation order, null is the "lowest" value, and so this will return all documents regardless of their field value.
sortKeys.forEach((key) => {
if (!keys.includes(key)) {
selector = { ...selector, [key]: { $gt: null } };
}
});
}
function getConversationMessages(conversation_id: any) {
return new Promise((res) => {
const db = new sqlite3.Database(
path.join(app.getPath("userData"), "jan.db")
);
const query = `SELECT * FROM messages WHERE conversation_id = ${conversation_id} ORDER BY id DESC`;
db.all(query, (err: Error, row: any) => {
res(row);
});
db.close();
});
}
function parseToProduct(row: any) {
const product = {
id: row.id,
slug: row.slug,
name: row.name,
description: row.description,
avatarUrl: row.avatar_url,
longDescription: row.long_description,
technicalDescription: row.technical_description,
author: row.author,
version: row.version,
modelUrl: row.model_url,
nsfw: row.nsfw,
greeting: row.greeting,
type: row.type,
inputs: row.inputs,
outputs: row.outputs,
createdAt: new Date(row.created_at),
updatedAt: new Date(row.updated_at),
fileName: row.file_name,
downloadUrl: row.download_url,
};
return product;
// There is no selector & sort, so we can just use allDocs() to get all the documents.
if (sortKeys.concat(keys).length === 0) {
return dbs[collectionName]
.allDocs({
include_docs: true,
endkey: "_design",
inclusive_end: false,
})
.then((data) => data.rows.map((row) => row.doc));
}
// At a basic level, there are two steps to running a query: createIndex()
// (to define which fields to index) and find() (to query the index).
return dbs[collectionName]
.createIndex({
// Create index for selector & sort
index: { fields: sortKeys.concat(keys) },
})
.then(() => {
// Find documents using Mango queries
return dbs[collectionName].find({
selector,
sort,
});
})
.then((data) => data.docs); // Return documents
}
module.exports = {
init,
getConversations,
deleteConversation,
storeConversation,
storeMessage,
updateMessage,
getConversationMessages,
storeModel,
updateFinishedDownloadAt,
getUnfinishedDownloadModels,
getFinishedDownloadModels,
deleteDownloadModel,
getModelById,
createCollection,
deleteCollection,
insertOne,
findOne,
findMany,
updateOne,
updateMany,
deleteOne,
deleteMany,
};

5030
electron/core/plugins/data-plugin/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

27
electron/core/plugins/data-plugin/package.json
View File

@ -1,24 +1,27 @@
{
"name": "data-plugin",
"version": "2.1.0",
"description": "",
"main": "dist/index.js",
"version": "1.0.4",
"description": "The Data Connector provides easy access to a data API using the PouchDB engine. It offers accessible data management capabilities.",
"icon": "https://raw.githubusercontent.com/tailwindlabs/heroicons/88e98b0c2b458553fbadccddc2d2f878edc0387b/src/20/solid/circle-stack.svg",
"main": "dist/esm/index.js",
"author": "Jan",
"license": "MIT",
"activationPoints": [
"init"
],
"scripts": {
"build": "tsc -b . && webpack --config webpack.config.js",
"build:package": "rimraf ./data-plugin*.tgz && npm run build && npm pack",
"build:publish": "npm run build:package && cpx *.tgz ../../pre-install"
"build": "tsc --project ./config/tsconfig.esm.json && tsc --project ./config/tsconfig.cjs.json && webpack --config webpack.config.js",
"postinstall": "rimraf ./data-plugin*.tgz && npm run build",
"build:publish": "npm pack && cpx *.tgz ../../pre-install"
},
"exports": {
".": "./dist/index.js",
"./main": "./dist/module.js"
"import": "./dist/esm/index.js",
"require": "./dist/cjs/module.js",
"default": "./dist/esm/index.js"
},
"devDependencies": {
"cpx": "^1.5.0",
"node-pre-gyp": "^0.17.0",
"rimraf": "^3.0.2",
"ts-loader": "^9.4.4",
"ts-node": "^10.9.1",
@ -27,8 +30,8 @@
"webpack-cli": "^5.1.4"
},
"bundledDependencies": [
"sql.js",
"sqlite3"
"pouchdb-node",
"pouchdb-find"
],
"files": [
"dist/**",
@ -36,6 +39,8 @@
"node_modules"
],
"dependencies": {
"sqlite3": "^5.1.6"
"@janhq/plugin-core": "file:../../../../plugin-core",
"pouchdb-find": "^8.0.1",
"pouchdb-node": "^8.0.1"
}
}

26
electron/core/plugins/data-plugin/tsconfig.json
View File

@ -1,22 +1,12 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Language and Environment */
"target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
/* Modules */
"module": "ES6" /* Specify what module code is generated. */,
// "rootDir": "./", /* Specify the root folder within your source files. */
// "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "." /* Specify the base directory to resolve non-relative module names. */,
// "paths": {} /* Specify a set of entries that re-map imports to additional lookup locations. */,
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "resolveJsonModule": true, /* Enable importing .json files. */
"outDir": "./dist" /* Specify an output folder for all emitted files. */,
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
/* Type Checking */
"strict": false /* Enable all strict type-checking options. */,
"skipLibCheck": true /* Skip type checking all .d.ts files. */
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true
}
}

5
electron/core/plugins/data-plugin/webpack.config.js
View File

@ -14,12 +14,15 @@ module.exports = {
],
},
output: {
filename: "index.js", // Adjust the output file name as needed
filename: "esm/index.js", // Adjust the output file name as needed
path: path.resolve(__dirname, "dist"),
library: { type: "module" }, // Specify ESM output format
},
resolve: {
extensions: [".ts", ".js"],
},
optimization: {
minimize: false
},
// Add loaders and other configuration as needed for your project
};

16
electron/core/plugins/inference-plugin/index.js → electron/core/plugins/inference-plugin/index.ts
View File

@ -9,19 +9,15 @@ const initModel = async (product) =>
}
});
const dispose = async () =>
new Promise(async (resolve) => {
if (window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "killSubprocess")
.then((res) => resolve(res));
}
});
const inferenceUrl = () => "http://localhost:8080/llama/chat_completion";
const inferenceUrl = () => "http://localhost:3928/llama/chat_completion";
const stopModel = () => {
window.electronAPI.invokePluginFunc(MODULE_PATH, "killSubprocess");
};
// Register all the above functions and objects with the relevant extension points
export function init({ register }) {
register("initModel", "initModel", initModel);
register("inferenceUrl", "inferenceUrl", inferenceUrl);
register("dispose", "dispose", dispose);
register("stopModel", "stopModel", stopModel);
}

87
electron/core/plugins/inference-plugin/module.js
View File

@ -1,87 +0,0 @@
const path = require("path");
const { app, dialog } = require("electron");
const { spawn } = require("child_process");
const fs = require("fs");
let subprocess = null;
async function initModel(product) {
// fileName fallback
if (!product.fileName) {
product.fileName = product.file_name;
}
if (!product.fileName) {
await dialog.showMessageBox({
message: "Selected model does not have file name..",
});
return;
}
if (subprocess) {
console.error(
"A subprocess is already running. Attempt to kill then reinit."
);
killSubprocess();
}
let binaryFolder = path.join(__dirname, "nitro"); // Current directory by default
// Read the existing config
const configFilePath = path.join(binaryFolder, "config", "config.json");
let config = {};
if (fs.existsSync(configFilePath)) {
const rawData = fs.readFileSync(configFilePath, "utf-8");
config = JSON.parse(rawData);
}
// Update the llama_model_path
if (!config.custom_config) {
config.custom_config = {};
}
const modelPath = path.join(app.getPath("userData"), product.fileName);
config.custom_config.llama_model_path = modelPath;
// Write the updated config back to the file
fs.writeFileSync(configFilePath, JSON.stringify(config, null, 4));
const binaryPath =
process.platform === "win32"
? path.join(binaryFolder, "nitro.exe")
: path.join(binaryFolder, "nitro");
// Execute the binary
subprocess = spawn(binaryPath, [configFilePath], { cwd: binaryFolder });
// Handle subprocess output
subprocess.stdout.on("data", (data) => {
console.log(`stdout: ${data}`);
});
subprocess.stderr.on("data", (data) => {
console.error(`stderr: ${data}`);
});
subprocess.on("close", (code) => {
console.log(`child process exited with code ${code}`);
subprocess = null;
});
}
function dispose() {
if (subprocess) {
subprocess.kill();
subprocess = null;
console.log("Subprocess terminated.");
} else {
console.error("No subprocess is currently running.");
}
}
module.exports = {
initModel,
dispose,
};

119
electron/core/plugins/inference-plugin/module.ts Normal file
View File

@ -0,0 +1,119 @@
const path = require("path");
const { app } = require("electron");
const { spawn } = require("child_process");
const fs = require("fs");
const tcpPortUsed = require("tcp-port-used");
const { killPortProcess } = require("kill-port-process");
let subprocess = null;
const PORT = 3928;
const initModel = (fileName) => {
return (
new Promise<void>(async (resolve, reject) => {
if (!fileName) {
reject("Model not found, please download again.");
}
if (subprocess) {
console.error(
"A subprocess is already running. Attempt to kill then reinit."
);
killSubprocess();
}
resolve(fileName);
})
// Kill port process if it is already in use
.then((fileName) =>
tcpPortUsed
.waitUntilFree(PORT, 200, 3000)
.catch(() => killPortProcess(PORT))
.then(() => fileName)
)
// Spawn Nitro subprocess to load model
.then(() => {
let binaryFolder = path.join(__dirname, "nitro"); // Current directory by default
// Read the existing config
const configFilePath = path.join(binaryFolder, "config", "config.json");
let config: any = {};
if (fs.existsSync(configFilePath)) {
const rawData = fs.readFileSync(configFilePath, "utf-8");
config = JSON.parse(rawData);
}
// Update the llama_model_path
if (!config.custom_config) {
config.custom_config = {};
}
const modelPath = path.join(app.getPath("userData"), fileName);
config.custom_config.llama_model_path = modelPath;
// Write the updated config back to the file
fs.writeFileSync(configFilePath, JSON.stringify(config, null, 4));
let binaryName;
if (process.platform === "win32") {
binaryName = "nitro_windows_amd64.exe";
} else if (process.platform === "darwin") {
// Mac OS platform
binaryName =
process.arch === "arm64" ? "nitro_mac_arm64" : "nitro_mac_amd64";
} else {
// Linux
binaryName = "nitro_linux_amd64_cuda"; // For other platforms
}
const binaryPath = path.join(binaryFolder, binaryName);
// Execute the binary
subprocess = spawn(binaryPath, [configFilePath], { cwd: binaryFolder });
// Handle subprocess output
subprocess.stdout.on("data", (data) => {
console.log(`stdout: ${data}`);
});
subprocess.stderr.on("data", (data) => {
console.error(`stderr: ${data}`);
});
subprocess.on("close", (code) => {
console.log(`child process exited with code ${code}`);
subprocess = null;
});
})
.then(() => tcpPortUsed.waitUntilUsed(PORT, 300, 30000))
.then(() => {
return {};
})
.catch((err) => {
return { error: err };
})
);
};
function dispose() {
killSubprocess();
// clean other registered resources here
}
function killSubprocess() {
if (subprocess) {
subprocess.kill();
subprocess = null;
console.log("Subprocess terminated.");
} else {
killPortProcess(PORT);
console.error("No subprocess is currently running.");
}
}
module.exports = {
initModel,
killSubprocess,
dispose,
};

14
electron/core/plugins/inference-plugin/nitro/config/config.json
View File

@ -1 +1,13 @@
{"custom_config": {"llama_model_path":"","ctx_len":2048,"ngl":100}}
{
"listeners": [
{
"address": "0.0.0.0",
"port": 3928
}
],
"custom_config": {
"llama_model_path": "",
"ctx_len": 2048,
"ngl": 100
}
}

170
electron/core/plugins/inference-plugin/nitro/ggml-metal.metal
View File

@ -24,12 +24,59 @@ typedef struct {
int8_t qs[QK8_0]; // quants
} block_q8_0;
// general-purpose kernel for addition of two tensors
// pros: works for non-contiguous tensors, supports broadcast across dims 1, 2 and 3
// cons: not very efficient
kernel void kernel_add(
device const float4 * src0,
device const float4 * src1,
device float4 * dst,
uint tpig[[thread_position_in_grid]]) {
dst[tpig] = src0[tpig] + src1[tpig];
device const char * src0,
device const char * src1,
device char * dst,
constant int64_t & ne00,
constant int64_t & ne01,
constant int64_t & ne02,
constant int64_t & ne03,
constant int64_t & nb00,
constant int64_t & nb01,
constant int64_t & nb02,
constant int64_t & nb03,
constant int64_t & ne10,
constant int64_t & ne11,
constant int64_t & ne12,
constant int64_t & ne13,
constant int64_t & nb10,
constant int64_t & nb11,
constant int64_t & nb12,
constant int64_t & nb13,
constant int64_t & ne0,
constant int64_t & ne1,
constant int64_t & ne2,
constant int64_t & ne3,
constant int64_t & nb0,
constant int64_t & nb1,
constant int64_t & nb2,
constant int64_t & nb3,
uint3 tgpig[[threadgroup_position_in_grid]],
uint3 tpitg[[thread_position_in_threadgroup]],
uint3 ntg[[threads_per_threadgroup]]) {
const int64_t i03 = tgpig.z;
const int64_t i02 = tgpig.y;
const int64_t i01 = tgpig.x;
const int64_t i13 = i03 % ne13;
const int64_t i12 = i02 % ne12;
const int64_t i11 = i01 % ne11;
device const char * src0_ptr = src0 + i03*nb03 + i02*nb02 + i01*nb01 + tpitg.x*nb00;
device const char * src1_ptr = src1 + i13*nb13 + i12*nb12 + i11*nb11 + tpitg.x*nb10;
device char * dst_ptr = dst + i03*nb3 + i02*nb2 + i01*nb1 + tpitg.x*nb0;
for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) {
((device float *)dst_ptr)[0] = ((device float *)src0_ptr)[0] + ((device float *)src1_ptr)[0];
src0_ptr += ntg.x*nb00;
src1_ptr += ntg.x*nb10;
dst_ptr += ntg.x*nb0;
}
}
// assumption: src1 is a row
@ -38,7 +85,7 @@ kernel void kernel_add_row(
device const float4 * src0,
device const float4 * src1,
device float4 * dst,
constant int64_t & nb,
constant int64_t & nb [[buffer(27)]],
uint tpig[[thread_position_in_grid]]) {
dst[tpig] = src0[tpig] + src1[tpig % nb];
}
@ -783,7 +830,9 @@ kernel void kernel_alibi_f32(
constant uint64_t & nb1,
constant uint64_t & nb2,
constant uint64_t & nb3,
constant float & m0,
constant float & m0,
constant float & m1,
constant int & n_heads_log2_floor,
uint3 tgpig[[threadgroup_position_in_grid]],
uint3 tpitg[[thread_position_in_threadgroup]],
uint3 ntg[[threads_per_threadgroup]]) {
@ -799,37 +848,73 @@ kernel void kernel_alibi_f32(
const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);
device float * dst_data = (device float *) ((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
float m_k = pow(m0, i2 + 1);
float m_k;
if (i2 < n_heads_log2_floor) {
m_k = pow(m0, i2 + 1);
} else {
m_k = pow(m1, 2 * (i2 - n_heads_log2_floor) + 1);
}
for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) {
device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00);
dst_data[i00] = src[0] + m_k * (i00 - ne00 + 1);
}
}
typedef void (rope_t)(
device const void * src0,
device const int32_t * src1,
device float * dst,
constant int64_t & ne00,
constant int64_t & ne01,
constant int64_t & ne02,
constant int64_t & ne03,
constant uint64_t & nb00,
constant uint64_t & nb01,
constant uint64_t & nb02,
constant uint64_t & nb03,
constant int64_t & ne0,
constant int64_t & ne1,
constant int64_t & ne2,
constant int64_t & ne3,
constant uint64_t & nb0,
constant uint64_t & nb1,
constant uint64_t & nb2,
constant uint64_t & nb3,
constant int & n_past,
constant int & n_dims,
constant int & mode,
constant float & freq_base,
constant float & freq_scale,
uint tiitg[[thread_index_in_threadgroup]],
uint3 tptg[[threads_per_threadgroup]],
uint3 tgpig[[threadgroup_position_in_grid]]);
template<typename T>
kernel void kernel_rope(
device const void * src0,
device float * dst,
constant int64_t & ne00,
constant int64_t & ne01,
constant int64_t & ne02,
constant int64_t & ne03,
constant uint64_t & nb00,
constant uint64_t & nb01,
constant uint64_t & nb02,
constant uint64_t & nb03,
constant int64_t & ne0,
constant int64_t & ne1,
constant int64_t & ne2,
constant int64_t & ne3,
constant uint64_t & nb0,
constant uint64_t & nb1,
constant uint64_t & nb2,
constant uint64_t & nb3,
constant int & n_past,
constant int & n_dims,
constant int & mode,
constant float & freq_base,
constant float & freq_scale,
device const void * src0,
device const int32_t * src1,
device float * dst,
constant int64_t & ne00,
constant int64_t & ne01,
constant int64_t & ne02,
constant int64_t & ne03,
constant uint64_t & nb00,
constant uint64_t & nb01,
constant uint64_t & nb02,
constant uint64_t & nb03,
constant int64_t & ne0,
constant int64_t & ne1,
constant int64_t & ne2,
constant int64_t & ne3,
constant uint64_t & nb0,
constant uint64_t & nb1,
constant uint64_t & nb2,
constant uint64_t & nb3,
constant int & n_past,
constant int & n_dims,
constant int & mode,
constant float & freq_base,
constant float & freq_scale,
uint tiitg[[thread_index_in_threadgroup]],
uint3 tptg[[threads_per_threadgroup]],
uint3 tgpig[[threadgroup_position_in_grid]]) {
@ -839,7 +924,9 @@ kernel void kernel_rope(
const bool is_neox = mode & 2;
const int64_t p = ((mode & 1) == 0 ? n_past + i2 : i2);
device const int32_t * pos = src1;
const int64_t p = pos[i2];
const float theta_0 = freq_scale * (float)p;
const float inv_ndims = -1.f/n_dims;
@ -851,11 +938,11 @@ kernel void kernel_rope(
const float cos_theta = cos(theta);
const float sin_theta = sin(theta);
device const float * const src = (device float *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
device float * dst_data = (device float *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
const float x0 = src[0];
const float x1 = src[1];
const T x0 = src[0];
const T x1 = src[1];
dst_data[0] = x0*cos_theta - x1*sin_theta;
dst_data[1] = x0*sin_theta + x1*cos_theta;
@ -870,8 +957,8 @@ kernel void kernel_rope(
const int64_t i0 = ib*n_dims + ic/2;
device const float * const src = (device float *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
device float * dst_data = (device float *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
device const T * const src = (device T *)((device char *) src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
device T * dst_data = (device T *)((device char *) dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
const float x0 = src[0];
const float x1 = src[n_dims/2];
@ -883,6 +970,9 @@ kernel void kernel_rope(
}
}
template [[host_name("kernel_rope_f32")]] kernel rope_t kernel_rope<float>;
template [[host_name("kernel_rope_f16")]] kernel rope_t kernel_rope<half>;
kernel void kernel_cpy_f16_f16(
device const half * src0,
device half * dst,
@ -1273,8 +1363,8 @@ kernel void kernel_mul_mat_q3_K_f32(
float yl[32];
const uint16_t kmask1 = 0x3030;
const uint16_t kmask2 = 0x0f0f;
//const uint16_t kmask1 = 0x3030;
//const uint16_t kmask2 = 0x0f0f;
const int tid = tiisg/4;
const int ix = tiisg%4;

BIN
electron/core/plugins/inference-plugin/nitro/nitro_linux_amd64_cuda Executable file
View File

Binary file not shown.

BIN
electron/core/plugins/inference-plugin/nitro/nitro_mac_amd64 Executable file
View File

Binary file not shown.

BIN
electron/core/plugins/inference-plugin/nitro/nitro → electron/core/plugins/inference-plugin/nitro/nitro_mac_arm64
View File

Binary file not shown.

BIN
electron/core/plugins/inference-plugin/nitro/nitro.exe → electron/core/plugins/inference-plugin/nitro/nitro_windows_amd64.exe
View File

Binary file not shown.

4029
electron/core/plugins/inference-plugin/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

30
electron/core/plugins/inference-plugin/package.json
View File

@ -1,17 +1,22 @@
{
"name": "inference-plugin",
"version": "0.0.1",
"description": "",
"version": "1.0.0",
"description": "Inference Plugin, powered by @janhq/nitro, bring a high-performance Llama model inference in pure C++.",
"icon": "https://raw.githubusercontent.com/tailwindlabs/heroicons/88e98b0c2b458553fbadccddc2d2f878edc0387b/src/20/solid/command-line.svg",
"main": "dist/index.js",
"author": "James",
"author": "Jan",
"license": "MIT",
"activationPoints": [
"init"
],
"scripts": {
"build": "webpack --config webpack.config.js",
"build:package": "rimraf ./*.tgz && npm run build && cpx \"module.js\" \"dist\" && rm -rf dist/nitro && cp -r nitro dist/nitro && npm pack",
"build:publish": "yarn build:package && cpx *.tgz ../../pre-install"
"build": "tsc -b . && webpack --config webpack.config.js",
"postinstall": "rimraf ./*.tgz && npm run build && rimraf dist/nitro/* && cpx \"nitro/**\" \"dist/nitro\"",
"build:publish": "npm pack && cpx *.tgz ../../pre-install"
},
"exports": {
".": "./dist/index.js",
"./main": "./dist/module.js"
},
"devDependencies": {
"cpx": "^1.5.0",
@ -19,14 +24,15 @@
"webpack": "^5.88.2",
"webpack-cli": "^5.1.4"
},
"bundledDependencies": [
"electron-is-dev",
"node-llama-cpp"
],
"dependencies": {
"electron-is-dev": "^2.0.0",
"node-llama-cpp": "^2.4.1"
"kill-port-process": "^3.2.0",
"tcp-port-used": "^1.0.2",
"ts-loader": "^9.5.0"
},
"bundledDependencies": [
"tcp-port-used",
"kill-port-process"
],
"engines": {
"node": ">=18.0.0"
},

22
electron/core/plugins/inference-plugin/tsconfig.json Normal file
View File

@ -0,0 +1,22 @@
{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Language and Environment */
"target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
/* Modules */
"module": "ES6" /* Specify what module code is generated. */,
// "rootDir": "./", /* Specify the root folder within your source files. */
// "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "." /* Specify the base directory to resolve non-relative module names. */,
// "paths": {} /* Specify a set of entries that re-map imports to additional lookup locations. */,
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "resolveJsonModule": true, /* Enable importing .json files. */
"outDir": "./dist" /* Specify an output folder for all emitted files. */,
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
/* Type Checking */
"strict": false /* Enable all strict type-checking options. */,
"skipLibCheck": true /* Skip type checking all .d.ts files. */
}
}

0
electron/core/plugins/data-plugin/types/index.d.ts → electron/core/plugins/inference-plugin/types/index.d.ts vendored
View File

4
electron/core/plugins/inference-plugin/webpack.config.js
View File

@ -2,7 +2,7 @@ const path = require("path");
module.exports = {
experiments: { outputModule: true },
entry: "./index.js", // Adjust the entry point to match your project's main file
entry: "./index.ts", // Adjust the entry point to match your project's main file
mode: "production",
module: {
rules: [
@ -19,7 +19,7 @@ module.exports = {
library: { type: "module" }, // Specify ESM output format
},
resolve: {
extensions: [".js"],
extensions: [".ts", ".js"],
},
// Add loaders and other configuration as needed for your project
};

47
electron/core/plugins/model-management-plugin/index.js
View File

@ -1,47 +0,0 @@
const MODULE_PATH = "model-management-plugin/dist/module.js";
const getDownloadedModels = async () =>
new Promise(async (resolve) => {
if (window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "getDownloadedModels")
.then((res) => resolve(res));
}
});
const getAvailableModels = async () =>
new Promise(async (resolve) => {
if (window.electronAPI) {
window.electronAPI
.invokePluginFunc(MODULE_PATH, "getAvailableModels")
.then((res) => resolve(res));
}
});
const downloadModel = async (product) =>
new Promise(async (resolve) => {
if (window && window.electronAPI) {
window.electronAPI
.downloadFile(product.downloadUrl, product.fileName)
.then((res) => resolve(res));
} else {
resolve("-");
}
});
const deleteModel = async (path) =>
new Promise(async (resolve) => {
if (window.electronAPI) {
console.debug(`Delete model model management plugin: ${path}`);
const response = await window.electronAPI.deleteFile(path);
resolve(response);
}
});
// Register all the above functions and objects with the relevant extension points
export function init({ register }) {
register("getDownloadedModels", "getDownloadedModels", getDownloadedModels);
register("getAvailableModels", "getAvailableModels", getAvailableModels);
register("downloadModel", "downloadModel", downloadModel);
register("deleteModel", "deleteModel", deleteModel);
}

103
electron/core/plugins/model-management-plugin/index.ts Normal file
View File

@ -0,0 +1,103 @@
import { ModelManagementService, RegisterExtensionPoint, core, store } from "@janhq/plugin-core";
const MODULE_PATH = "model-management-plugin/dist/module.js";
const getDownloadedModels = () => core.invokePluginFunc(MODULE_PATH, "getDownloadedModels");
const getAvailableModels = () => core.invokePluginFunc(MODULE_PATH, "getAvailableModels");
const downloadModel = (product) => core.downloadFile(product.downloadUrl, product.fileName);
const deleteModel = (path) => core.deleteFile(path);
const searchModels = (params) => core.invokePluginFunc(MODULE_PATH, "searchModels", params);
const getConfiguredModels = () => core.invokePluginFunc(MODULE_PATH, "getConfiguredModels");
/**
* Store a model in the database when user start downloading it
*
* @param model Product
*/
function storeModel(model: any) {
return store.findOne("models", model._id).then((doc) => {
if (doc) {
return store.updateOne("models", model._id, model);
} else {
return store.insertOne("models", model);
}
});
}
/**
* Update the finished download time of a model
*
* @param model Product
*/
function updateFinishedDownloadAt(_id: string): Promise<any> {
return store.updateMany("models", { _id }, { time: Date.now(), finishDownloadAt: 1 });
}
/**
* Retrieves all unfinished models from the database.
*
* @returns A promise that resolves with an array of unfinished models.
*/
function getUnfinishedDownloadModels(): Promise<any> {
return store.findMany("models", { finishDownloadAt: -1 }, [{ startDownloadAt: "desc" }]);
}
/**
* Retrieves all finished models from the database.
*
* @returns A promise that resolves with an array of finished models.
*/
function getFinishedDownloadModels(): Promise<any> {
return store.findMany("models", { finishDownloadAt: 1 });
}
/**
* Deletes a model from the database.
*
* @param modelId The ID of the model to delete.
* @returns A promise that resolves when the model is deleted.
*/
function deleteDownloadModel(modelId: string): Promise<any> {
return store.deleteOne("models", modelId);
}
/**
* Retrieves a model from the database by ID.
*
* @param modelId The ID of the model to retrieve.
* @returns A promise that resolves with the model.
*/
function getModelById(modelId: string): Promise<any> {
return store.findOne("models", modelId);
}
function onStart() {
store.createCollection("models", {});
}
// Register all the above functions and objects with the relevant extension points
export function init({ register }: { register: RegisterExtensionPoint }) {
onStart();
register(ModelManagementService.GetDownloadedModels, getDownloadedModels.name, getDownloadedModels);
register(ModelManagementService.GetAvailableModels, getAvailableModels.name, getAvailableModels);
register(ModelManagementService.DownloadModel, downloadModel.name, downloadModel);
register(ModelManagementService.DeleteModel, deleteModel.name, deleteModel);
register(ModelManagementService.SearchModels, searchModels.name, searchModels);
register(ModelManagementService.GetConfiguredModels, getConfiguredModels.name, getConfiguredModels);
register(ModelManagementService.StoreModel, storeModel.name, storeModel);
register(ModelManagementService.UpdateFinishedDownloadAt, updateFinishedDownloadAt.name, updateFinishedDownloadAt);
register(
ModelManagementService.GetUnfinishedDownloadModels,
getUnfinishedDownloadModels.name,
getUnfinishedDownloadModels
);
register(ModelManagementService.DeleteDownloadModel, deleteDownloadModel.name, deleteDownloadModel);
register(ModelManagementService.GetModelById, getModelById.name, getModelById);
register(ModelManagementService.GetFinishedDownloadModels, getFinishedDownloadModels.name, getFinishedDownloadModels);
}

102
electron/core/plugins/model-management-plugin/module.js
View File

@ -1,102 +0,0 @@
const path = require("path");
const { readdirSync, lstatSync } = require("fs");
const { app } = require("electron");
const ALL_MODELS = [
{
id: "llama-2-7b-chat.Q4_K_M.gguf.bin",
slug: "llama-2-7b-chat.Q4_K_M.gguf.bin",
name: "Llama 2 7B Chat - GGUF",
description: "medium, balanced quality - recommended",
avatarUrl:
"https://aeiljuispo.cloudimg.io/v7/https://cdn-uploads.huggingface.co/production/uploads/6426d3f3a7723d62b53c259b/tvPikpAzKTKGN5wrpadOJ.jpeg?w=200&h=200&f=face",
longDescription:
"GGUF is a new format introduced by the llama.cpp team on August 21st 2023. It is a replacement for GGML, which is no longer supported by llama.cpp. GGUF offers numerous advantages over GGML, such as better tokenisation, and support for special tokens. It is also supports metadata, and is designed to be extensible.",
technicalDescription:
'GGML_TYPE_Q4_K - "type-1" 4-bit quantization in super-blocks containing 8 blocks, each block having 32 weights. Scales and mins are quantized with 6 bits. This ends up using 4.5 bpw.',
author: "The Bloke",
version: "1.0.0",
modelUrl: "https://google.com",
nsfw: false,
greeting: "Hello there",
type: "LLM",
inputs: undefined,
outputs: undefined,
createdAt: 0,
updatedAt: undefined,
fileName: "llama-2-7b-chat.Q4_K_M.gguf.bin",
downloadUrl:
"https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf",
},
{
id: "llama-2-13b-chat.Q4_K_M.gguf",
slug: "llama-2-13b-chat.Q4_K_M.gguf",
name: "Llama 2 13B Chat - GGUF",
description:
"medium, balanced quality - not recommended for RAM 16GB and below",
avatarUrl:
"https://aeiljuispo.cloudimg.io/v7/https://cdn-uploads.huggingface.co/production/uploads/6426d3f3a7723d62b53c259b/tvPikpAzKTKGN5wrpadOJ.jpeg?w=200&h=200&f=face",
longDescription:
"GGUF is a new format introduced by the llama.cpp team on August 21st 2023. It is a replacement for GGML, which is no longer supported by llama.cpp. GGUF offers numerous advantages over GGML, such as better tokenisation, and support for special tokens. It is also supports metadata, and is designed to be extensible.",
technicalDescription:
'GGML_TYPE_Q4_K - "type-1" 4-bit quantization in super-blocks containing 8 blocks, each block having 32 weights. Scales and mins are quantized with 6 bits. This ends up using 4.5 bpw.',
author: "The Bloke",
version: "1.0.0",
modelUrl: "https://google.com",
nsfw: false,
greeting: "Hello there",
type: "LLM",
inputs: undefined,
outputs: undefined,
createdAt: 0,
updatedAt: undefined,
fileName: "llama-2-13b-chat.Q4_K_M.gguf.bin",
downloadUrl:
"https://huggingface.co/TheBloke/Llama-2-13B-chat-GGUF/resolve/main/llama-2-13b-chat.Q4_K_M.gguf",
},
];
function getDownloadedModels() {
const userDataPath = app.getPath("userData");
const allBinariesName = [];
var files = readdirSync(userDataPath);
for (var i = 0; i < files.length; i++) {
var filename = path.join(userDataPath, files[i]);
var stat = lstatSync(filename);
if (stat.isDirectory()) {
// ignore
} else if (filename.endsWith(".bin")) {
var binaryName = path.basename(filename);
allBinariesName.push(binaryName);
}
}
const downloadedModels = ALL_MODELS.map((model) => {
if (
model.fileName &&
allBinariesName
.map((t) => t.toLowerCase())
.includes(model.fileName.toLowerCase())
) {
return model;
}
return undefined;
}).filter((m) => m !== undefined);
return downloadedModels;
}
function getAvailableModels() {
const downloadedModelIds = getDownloadedModels().map((model) => model.id);
return ALL_MODELS.filter((model) => {
if (!downloadedModelIds.includes(model.id)) {
return model;
}
});
}
module.exports = {
getDownloadedModels,
getAvailableModels,
};

212
electron/core/plugins/model-management-plugin/module.ts Normal file
View File

@ -0,0 +1,212 @@
const { listModels, listFiles, fileDownloadInfo } = require("@huggingface/hub");
const https = require("https");
let modelsIterator = undefined;
let currentSearchOwner = undefined;
// Github API
const githubHostName = "api.github.com";
const githubHeaders = {
"User-Agent": "node.js",
Accept: "application/vnd.github.v3+json",
};
const githubPath = "/repos/janhq/models/contents";
const getNextModels = async (count) => {
const models = [];
let hasMore = true;
while (models.length < count) {
const next = await modelsIterator.next();
// end if we reached the end
if (next.done) {
hasMore = false;
break;
}
const model = next.value;
const files = await listFilesByName(model.name);
models.push({
...model,
files,
});
}
const result = {
data: models,
hasMore,
};
return result;
};
const searchModels = async (params) => {
if (currentSearchOwner === params.search.owner && modelsIterator != null) {
// paginated search
console.debug(`Paginated search owner: ${params.search.owner}`);
const models = await getNextModels(params.limit);
return models;
} else {
// new search
console.debug(`Init new search owner: ${params.search.owner}`);
currentSearchOwner = params.search.owner;
modelsIterator = listModels({
search: params.search,
credentials: params.credentials,
});
const models = await getNextModels(params.limit);
return models;
}
};
const listFilesByName = async (modelName) => {
const repo = { type: "model", name: modelName };
const fileDownloadInfoMap = {};
for await (const file of listFiles({
repo: repo,
})) {
if (file.type === "file" && file.path.endsWith(".bin")) {
const downloadInfo = await fileDownloadInfo({
repo: repo,
path: file.path,
});
fileDownloadInfoMap[file.path] = {
...file,
...downloadInfo,
};
}
}
return fileDownloadInfoMap;
};
async function getConfiguredModels() {
const files: any = await getModelFiles();
const promises = files.map((file) => getContent(file));
const response = await Promise.all(promises);
const models = [];
response.forEach((model) => {
models.push(parseToModel(model));
});
return models;
}
const parseToModel = (model) => {
const modelVersions = [];
model.versions.forEach((v) => {
const version = {
_id: `${model.author}-${v.name}`,
name: v.name,
quantMethod: v.quantMethod,
bits: v.bits,
size: v.size,
maxRamRequired: v.maxRamRequired,
usecase: v.usecase,
downloadLink: v.downloadLink,
productId: model.id,
};
modelVersions.push(version);
});
const product = {
_id: model.id,
name: model.name,
shortDescription: model.shortDescription,
avatarUrl: model.avatarUrl,
author: model.author,
version: model.version,
modelUrl: model.modelUrl,
nsfw: model.nsfw,
tags: model.tags,
greeting: model.defaultGreeting,
type: model.type,
createdAt: model.createdAt,
longDescription: model.longDescription,
status: "Downloadable",
releaseDate: 0,
availableVersions: modelVersions,
};
return product;
};
async function getModelFiles() {
const options = {
hostname: githubHostName,
path: githubPath,
headers: githubHeaders,
};
const data = await new Promise((resolve, reject) => {
const req = https.request(options, (res) => {
let data = "";
res.on("data", (chunk) => {
data += chunk;
});
res.on("end", () => {
const files = JSON.parse(data);
if (files.filter == null) {
console.error(files.message);
reject(files.message ?? "No files found");
}
if (!files || files.length === 0) {
resolve([]);
}
const jsonFiles = files.filter((file) => file.name.endsWith(".json"));
resolve(jsonFiles);
});
});
req.on("error", (error) => {
console.error(error);
});
req.end();
});
return data;
}
async function getContent(file) {
const options = {
hostname: githubHostName,
path: `${githubPath}/${file.path}`,
headers: githubHeaders,
};
const data = await new Promise((resolve) => {
const req = https.request(options, (res) => {
let data = "";
res.on("data", (chunk) => {
data += chunk;
});
res.on("end", () => {
const fileData = JSON.parse(data);
const fileContent = Buffer.from(fileData.content, "base64").toString();
resolve(JSON.parse(fileContent));
});
});
req.on("error", (error) => {
console.error(error);
});
req.end();
});
return data;
}
module.exports = {
searchModels,
getConfiguredModels,
};

3734
electron/core/plugins/model-management-plugin/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

19
electron/core/plugins/model-management-plugin/package.json
View File

@ -1,7 +1,8 @@
{
"name": "model-management-plugin",
"version": "0.0.1",
"description": "",
"version": "1.0.0",
"description": "Model Management Plugin provides model exploration and seamless downloads",
"icon": "https://raw.githubusercontent.com/tailwindlabs/heroicons/88e98b0c2b458553fbadccddc2d2f878edc0387b/src/20/solid/queue-list.svg",
"main": "dist/index.js",
"author": "James",
"license": "MIT",
@ -9,9 +10,9 @@
"init"
],
"scripts": {
"build": "webpack --config webpack.config.js",
"build:package": "rimraf ./*.tgz && npm run build && cpx \"module.js\" \"dist\" && npm pack",
"build:publish": "yarn build:package && cpx *.tgz ../../pre-install"
"build": "tsc -b . && webpack --config webpack.config.js",
"postinstall": "rimraf ./*.tgz && npm run build",
"build:publish": "npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"cpx": "^1.5.0",
@ -23,5 +24,13 @@
"dist/*",
"package.json",
"README.md"
],
"dependencies": {
"@huggingface/hub": "^0.8.5",
"@janhq/plugin-core": "file:../../../../plugin-core",
"ts-loader": "^9.5.0"
},
"bundledDependencies": [
"@huggingface/hub"
]
}

12
electron/core/plugins/model-management-plugin/tsconfig.json Normal file
View File

@ -0,0 +1,12 @@
{
"compilerOptions": {
"target": "es2016",
"module": "ES6",
"moduleResolution": "node",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true
}
}

7
electron/core/plugins/model-management-plugin/webpack.config.js
View File

@ -2,7 +2,7 @@ const path = require("path");
module.exports = {
experiments: { outputModule: true },
entry: "./index.js", // Adjust the entry point to match your project's main file
entry: "./index.ts", // Adjust the entry point to match your project's main file
mode: "production",
module: {
rules: [
@ -19,7 +19,10 @@ module.exports = {
library: { type: "module" }, // Specify ESM output format
},
resolve: {
extensions: [".js"],
extensions: [".ts", ".js"],
},
optimization: {
minimize: false
},
// Add loaders and other configuration as needed for your project
};

1479
electron/core/plugins/monitoring-plugin/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

9
electron/core/plugins/monitoring-plugin/package.json
View File

@ -1,7 +1,8 @@
{
"name": "monitoring-plugin",
"version": "0.0.1",
"description": "",
"version": "1.0.0",
"description": "Utilizing systeminformation, it provides essential System and OS information retrieval",
"icon": "https://raw.githubusercontent.com/tailwindlabs/heroicons/88e98b0c2b458553fbadccddc2d2f878edc0387b/src/20/solid/cpu-chip.svg",
"main": "dist/bundle.js",
"author": "Jan",
"license": "MIT",
@ -10,8 +11,8 @@
],
"scripts": {
"build": "webpack --config webpack.config.js",
"build:package": "rimraf ./*.tgz && npm run build && cpx \"module.js\" \"dist\" && npm pack",
"build:publish": "yarn build:package && cpx *.tgz ../../pre-install"
"postinstall": "rimraf ./*.tgz && npm run build && cpx \"module.js\" \"dist\"",
"build:publish": "npm pack && cpx *.tgz ../../pre-install"
},
"devDependencies": {
"rimraf": "^3.0.2",

14
electron/entitlements.mac.plist Normal file
View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.cs.allow-jit</key>
<true/>
<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
<true/>
<key>com.apple.security.cs.allow-dyld-environment-variables</key>
<true/>
<key>com.apple.security.cs.disable-library-validation</key>
<true/>
</dict>
</plist>

0
electron/icon.png → electron/icons/icon.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 38 KiB

44
electron/main.ts
View File

@ -6,7 +6,6 @@ import { init } from "./core/plugin-manager/pluginMgr";
import { setupMenu } from "./utils/menu";
import { dispose } from "./utils/disposable";
const isDev = require("electron-is-dev");
const request = require("request");
const progress = require("request-progress");
const { autoUpdater } = require("electron-updater");
@ -49,9 +48,9 @@ function createMainWindow() {
},
});
const startURL = isDev
? "http://localhost:3000"
: `file://${join(__dirname, "../renderer/index.html")}`;
const startURL = app.isPackaged
? `file://${join(__dirname, "../renderer/index.html")}`
: "http://localhost:3000";
mainWindow.loadURL(startURL);
@ -60,7 +59,7 @@ function createMainWindow() {
if (process.platform !== "darwin") app.quit();
});
if (isDev) mainWindow.webContents.openDevTools();
if (!app.isPackaged) mainWindow.webContents.openDevTools();
}
function handleAppUpdates() {
@ -127,7 +126,9 @@ function handleIPCs() {
const basePluginPath = join(
__dirname,
"../",
isDev ? "/core/pre-install" : "../app.asar.unpacked/core/pre-install"
app.isPackaged
? "../app.asar.unpacked/core/pre-install"
: "/core/pre-install"
);
return readdirSync(basePluginPath)
.filter((file) => extname(file) === ".tgz")
@ -143,6 +144,37 @@ function handleIPCs() {
ipcMain.handle("openExternalUrl", async (_event, url) => {
shell.openExternal(url);
});
ipcMain.handle("relaunch", async (_event, url) => {
dispose(requiredModules);
app.relaunch();
app.exit();
});
ipcMain.handle("reloadPlugins", async (_event, url) => {
const userDataPath = app.getPath("userData");
const fullPath = join(userDataPath, "plugins");
rmdir(fullPath, { recursive: true }, function (err) {
if (err) console.log(err);
dispose(requiredModules);
// just relaunch if packaged, should launch manually in development mode
if (app.isPackaged) {
app.relaunch();
app.exit();
} else {
for (const modulePath in requiredModules) {
delete require.cache[
require.resolve(
join(app.getPath("userData"), "plugins", modulePath)
)
];
}
setupPlugins();
mainWindow?.reload();
}
});
});
/**
* Used to delete a file from the user data folder

60
electron/package.json
View File

@ -1,10 +1,11 @@
{
"name": "jan-electron",
"version": "0.1.1",
"name": "jan",
"version": "0.1.3",
"main": "./build/main.js",
"author": "Jan",
"author": "Jan <service@jan.ai>",
"license": "MIT",
"homepage": "./",
"homepage": "https://github.com/janhq/jan/tree/main/electron",
"description": "Use offline LLMs with your own data. Run open source models like Llama2 or Falcon on your internal computers/servers.",
"build": {
"appId": "jan.ai.app",
"productName": "Jan",
@ -12,7 +13,8 @@
"renderer/**/*",
"build/*.{js,map}",
"build/**/*.{js,map}",
"core/pre-install"
"core/pre-install",
"core/plugin-manager/facade"
],
"asarUnpack": [
"core/pre-install"
@ -26,33 +28,57 @@
],
"extends": null,
"mac": {
"type": "distribution"
}
"type": "distribution",
"entitlements": "./entitlements.mac.plist",
"entitlementsInherit": "./entitlements.mac.plist",
"notarize": {
"teamId": "YT49P7GXG4"
},
"icon": "icons/icon.png"
},
"linux": {
"target": ["deb"],
"category": "Utility",
"icon": "icons/"
},
"win": {
"icon": "icons/icon.png"
},
"artifactName": "jan-${os}-${arch}-${version}.${ext}"
},
"scripts": {
"lint": "eslint . --ext \".js,.jsx,.ts,.tsx\"",
"test:e2e": "playwright test --workers=1",
"dev": "tsc -p . && electron .",
"build": "tsc -p . && electron-builder -p never -mw",
"build:publish": "tsc -p . && electron-builder -p onTagOrDraft -mw",
"postinstall": "electron-builder install-app-deps"
"build": "tsc -p . && electron-builder -p never -m",
"build:darwin": "tsc -p . && electron-builder -p never -m --x64 --arm64",
"build:win32": "tsc -p . && electron-builder -p never -w",
"build:linux": "tsc -p . && electron-builder -p never --linux deb",
"build:publish": "tsc -p . && electron-builder -p onTagOrDraft -m",
"build:publish-darwin": "tsc -p . && electron-builder -p onTagOrDraft -m --x64 --arm64",
"build:publish-win32": "tsc -p . && electron-builder -p onTagOrDraft -w",
"build:publish-linux": "tsc -p . && electron-builder -p onTagOrDraft --linux deb "
},
"dependencies": {
"electron-is-dev": "^2.0.0",
"@npmcli/arborist": "^7.1.0",
"@uiball/loaders": "^1.3.0",
"electron-store": "^8.1.0",
"electron-updater": "^6.1.4",
"node-llama-cpp": "^2.4.1",
"pluggable-electron": "^0.6.0",
"pacote": "^17.0.4",
"react-intersection-observer": "^9.5.2",
"request": "^2.88.2",
"request-progress": "^3.0.0"
"request-progress": "^3.0.0",
"use-debounce": "^9.0.4"
},
"devDependencies": {
"@electron/notarize": "^2.1.0",
"@playwright/test": "^1.38.1",
"@typescript-eslint/eslint-plugin": "^6.7.3",
"@typescript-eslint/parser": "^6.7.3",
"concurrently": "^8.2.1",
"electron": "26.2.1",
"electron-builder": "^24.6.4",
"eslint-plugin-react": "^7.33.2",
"wait-on": "^7.0.1"
"electron-playwright-helpers": "^1.6.0",
"eslint-plugin-react": "^7.33.2"
},
"installConfig": {
"hoistingLimits": "workspaces"

10
electron/playwright.config.ts Normal file
View File

@ -0,0 +1,10 @@
import { PlaywrightTestConfig } from "@playwright/test";
const config: PlaywrightTestConfig = {
testDir: "./tests",
testIgnore: "./core/**",
retries: 0,
timeout: 120000,
};
export default config;

7
electron/preload.ts
View File

@ -1,7 +1,6 @@
/* eslint-disable react-hooks/rules-of-hooks */
// Make Pluggable Electron's facade available to the renderer on window.plugins
//@ts-ignore
const useFacade = require("pluggable-electron/facade");
const useFacade = require("../core/plugin-manager/facade");
useFacade();
//@ts-ignore
const { contextBridge, ipcRenderer } = require("electron");
@ -14,10 +13,14 @@ contextBridge.exposeInMainWorld("electronAPI", {
pluginPath: () => ipcRenderer.invoke("pluginPath"),
reloadPlugins: () => ipcRenderer.invoke("reloadPlugins"),
appVersion: () => ipcRenderer.invoke("appVersion"),
openExternalUrl: (url: string) => ipcRenderer.invoke("openExternalUrl", url),
relaunch: () => ipcRenderer.invoke("relaunch"),
deleteFile: (filePath: string) => ipcRenderer.invoke("deleteFile", filePath),
downloadFile: (url: string, path: string) =>

Some files were not shown because too many files have changed in this diff Show More