Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: Phi-3 doesn't display (#2928)

Browse Source 
* fix: params correction

* add phi

* version bump
This commit is contained in:
Hoang Ha 2024-05-20 23:45:06 +07:00 committed by GitHub
parent e78d057f0f
commit 65b8d8e66b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/inference-nitro-extension/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@janhq/inference-cortex-extension",
"productName": "Cortex Inference Engine",
"version": "1.0.7",
"version": "1.0.8",
"description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
"main": "dist/index.js",
"node": "dist/node/index.cjs.js",

12
extensions/inference-nitro-extension/resources/models/phi3-3.8b/model.json
View File

@ -8,17 +8,23 @@
"id": "phi3-3.8b",
"object": "model",
"name": "Phi-3 Mini",
"version": "1.0",
"version": "1.1",
"description": "Phi-3 Mini is Microsoft's newest, compact model designed for mobile use.",
"format": "gguf",
"settings": {
"ctx_len": 4096,
"prompt_template": "<|user|>\n{prompt}<|end|>\n<|assistant|>\n",
"llama_model_path": "Phi-3-mini-4k-instruct-q4.gguf"
"llama_model_path": "Phi-3-mini-4k-instruct-q4.gguf",
"ngl": 32
},
"parameters": {
"max_tokens": 4096,
"stop": ["<|end|>"]
"stop": ["<|end|>"],
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Microsoft",

2
extensions/inference-nitro-extension/rollup.config.ts
View File

@ -23,6 +23,7 @@ const mistralIns7bq4Json = require('./resources/models/mistral-ins-7b-q4/model.j
const mixtral8x7bInstructJson = require('./resources/models/mixtral-8x7b-instruct/model.json')
const noromaid7bJson = require('./resources/models/noromaid-7b/model.json')
const openchat357bJson = require('./resources/models/openchat-3.5-7b/model.json')
const phi3bJson = require('./resources/models/phi3-3.8b/model.json')
const phind34bJson = require('./resources/models/phind-34b/model.json')
const qwen7bJson = require('./resources/models/qwen-7b/model.json')
const stableZephyr3bJson = require('./resources/models/stable-zephyr-3b/model.json')
@ -64,6 +65,7 @@ export default [
mixtral8x7bInstructJson,
noromaid7bJson,
openchat357bJson,
phi3bJson,
phind34bJson,
qwen7bJson,
stableZephyr3bJson,