Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Chore: phi3 long-context update (#2936)

Browse Source 
* init

* init

* fix: correct version

* version bump

* correct url

* remove small

* correct size
This commit is contained in:
Hoang Ha 2024-05-22 21:20:42 +07:00 committed by GitHub
parent 5573ec9fdb
commit 385ebb7750
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 39 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/inference-nitro-extension/package.json
View File

@ -1,7 +1,7 @@
{ {
"name": "@janhq/inference-cortex-extension", "name": "@janhq/inference-cortex-extension",
"productName": "Cortex Inference Engine", "productName": "Cortex Inference Engine",
"version": "1.0.8", "version": "1.0.9",
"description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.", "description": "This extension embeds cortex.cpp, a lightweight inference engine written in C++. See https://nitro.jan.ai.\nAdditional dependencies could be installed to run without Cuda Toolkit installation.",
"main": "dist/index.js", "main": "dist/index.js",
"node": "dist/node/index.cjs.js", "node": "dist/node/index.cjs.js",

38
extensions/inference-nitro-extension/resources/models/phi3-medium/model.json Normal file
View File

@ -0,0 +1,38 @@
{
"sources": [
{
"url": "https://huggingface.co/bartowski/Phi-3-medium-128k-instruct-GGUF/resolve/main/Phi-3-medium-128k-instruct-Q4_K_M.gguf",
"filename": "Phi-3-medium-128k-instruct-Q4_K_M.gguf"
}
],
"id": "phi3-medium",
"object": "model",
"name": "Phi-3 Medium",
"version": "1.0",
"description": "Phi-3 Medium is Microsoft's latest SOTA model.",
"format": "gguf",
"settings": {
"ctx_len": 128000,
"prompt_template": "<|user|>\n{prompt}<|end|>\n<|assistant|>\n",
"llama_model_path": "Phi-3-medium-128k-instruct-Q4_K_M.gguf",
"ngl": 32
},
"parameters": {
"max_tokens": 128000,
"stop": ["<|end|>"],
"temperature": 0.7,
"top_p": 0.95,
"stream": true,
"frequency_penalty": 0,
"presence_penalty": 0
},
"metadata": {
"author": "Microsoft",
"tags": [
"7B",
"Finetuned"
],
"size": 8366000000
},
"engine": "nitro"
}