feat: Quick round 1st draft
This commit is contained in:
hiro
parent
52d56a8ae1
commit
d2b26d4806
@ -8,37 +8,99 @@ Models are AI models like Llama and Mistral
|
|||||||
|
|
||||||
## Model Object
|
## Model Object
|
||||||
|
|
||||||
- `model.json`
|
- LOCAL MODEL `model-zephyr-7B.json`
|
||||||
|
- Reference: https://huggingface.co/TheBloke/zephyr-7B-beta-GGUF/
|
||||||
|
|
||||||
> Equivalent to: https://platform.openai.com/docs/api-reference/models/object
|
> Equivalent to: https://platform.openai.com/docs/api-reference/models/object
|
||||||
|
|
||||||
```json
|
```sh=
|
||||||
{
|
# Required
|
||||||
// OpenAI model compatibility
|
|
||||||
// https://platform.openai.com/docs/api-reference/models)
|
|
||||||
"id": "llama-2-uuid",
|
|
||||||
"object": "model",
|
|
||||||
"created": 1686935002,
|
|
||||||
"owned_by": "you"
|
|
||||||
|
|
||||||
// Model settings (benchmark: Ollama)
|
"url": TheBloke/zephyr-7B-beta-GGUF
|
||||||
// https://github.com/jmorganca/ollama/blob/main/docs/modelfile.md#template
|
|
||||||
"model_name": "llama2",
|
|
||||||
"model_path": "ROOT/models/...",
|
|
||||||
"parameters": {
|
|
||||||
"temperature": "..",
|
|
||||||
"token-limit": "..",
|
|
||||||
"top-k": "..",
|
|
||||||
"top-p": ".."
|
|
||||||
},
|
|
||||||
"template": "This is a full prompt template",
|
|
||||||
"system": "This is a system prompt",
|
|
||||||
|
|
||||||
// Model metadata (benchmark: HuggingFace)
|
# Optional - by default use `default``
|
||||||
"version": "...",
|
import_format: thebloke
|
||||||
"author": "...",
|
# default # downloads the whole thing
|
||||||
"tags": "...",
|
# thebloke # custom importer (detects from URL)
|
||||||
...
|
# janhq # Custom importers
|
||||||
|
# openai
|
||||||
|
"default_download": zephyr-7b-beta.Q2_K.gguf # optional, by default download model with recommended hardware
|
||||||
|
|
||||||
|
# Optional: OpenAI format
|
||||||
|
"id": "/huggingface.co/TheBloke/zephyr-7B-beta-GGUF", # Autofilled by Jan with required URL above
|
||||||
|
"object": "model",
|
||||||
|
"created": 1686935002,
|
||||||
|
"owned_by": "TheBloke"
|
||||||
|
|
||||||
|
# Optional: params
|
||||||
|
"init_parameters": {
|
||||||
|
"ctx_len": 2048,
|
||||||
|
"ngl": 100,
|
||||||
|
"embedding": true,
|
||||||
|
"n_parallel": 4,
|
||||||
|
"pre_prompt": "A chat between a curious user and an artificial intelligence",
|
||||||
|
"user_prompt": "USER: ",
|
||||||
|
"ai_prompt": "ASSISTANT: "
|
||||||
|
},
|
||||||
|
|
||||||
|
"runtime_parameters": {
|
||||||
|
"temperature": "0.7",
|
||||||
|
"token_limit": "2048",
|
||||||
|
"top_k": "",
|
||||||
|
"top_p": "..",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Jan specific configs
|
||||||
|
"metadata": { // @Q: should we put all under "jan"
|
||||||
|
"engine": "api", // enum[llamacpp,api]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- REMOTE MODEL `model-azure-openai-gpt4-turbo.json`
|
||||||
|
- Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/quickstart?tabs=command-line%2Cpython&pivots=rest-api
|
||||||
|
|
||||||
|
> Equivalent to: https://platform.openai.com/docs/api-reference/models/object
|
||||||
|
|
||||||
|
```sh=
|
||||||
|
# Required
|
||||||
|
|
||||||
|
"url": https://docs-test-001.openai.azure.com/ # This is `api.openai.com` if it's OpenAI platform
|
||||||
|
|
||||||
|
# Optional - by default use `default``
|
||||||
|
import_format: azure_openai
|
||||||
|
# default # downloads the whole thing
|
||||||
|
# thebloke # custom importer (detects from URL)
|
||||||
|
# janhq # Custom importers
|
||||||
|
# azure_openai # Custom importers
|
||||||
|
# openai # Custom importers
|
||||||
|
"default_download": zephyr-7b-beta.Q2_K.gguf # optional, by default download model with recommended hardware
|
||||||
|
|
||||||
|
# Optional: OpenAI format
|
||||||
|
"id": "/openai.azure.com/docs-test-001/gpt4-turbo", # Autofilled by Jan with required URL above
|
||||||
|
"object": "model",
|
||||||
|
"created": 1686935002,
|
||||||
|
"owned_by": "OpenAI Azure"
|
||||||
|
|
||||||
|
# Optional: params
|
||||||
|
# This is the one model gets configured and cannot be changed by assistant
|
||||||
|
"init_parameters": {
|
||||||
|
"API-KEY": "",
|
||||||
|
"DEPLOYMENT-NAME": "",
|
||||||
|
"api-version": "2023-05-15"
|
||||||
|
},
|
||||||
|
|
||||||
|
# This is the one that assistant can override
|
||||||
|
"runtime_parameters": {
|
||||||
|
"temperature": "0.7",
|
||||||
|
"max_tokens": "2048",
|
||||||
|
"presence_penalty": "0",
|
||||||
|
"top_p": "1",
|
||||||
|
"stream": "true"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Jan specific configs
|
||||||
|
"metadata": { // @Q: should we put all under "jan"
|
||||||
|
"engine": "api", // enum[llamacpp,api]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -49,33 +111,58 @@ See [/model](/api/model)
|
|||||||
- Equivalent to: https://platform.openai.com/docs/api-reference/models
|
- Equivalent to: https://platform.openai.com/docs/api-reference/models
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
GET https://localhost:1337/v1/models # List models
|
# List models
|
||||||
GET https://localhost:1337/v1/models/{model} # Get model object
|
GET https://localhost:1337/v1/models?filter=[enum](all,running,downloaded,downloading)
|
||||||
DELETE https://localhost:1337/v1/models/{model} # Delete model
|
List[model_object]
|
||||||
|
|
||||||
|
# Get model object
|
||||||
|
GET https://localhost:1337/v1/models/{model_id} # json file name as {model_id} model-azure-openai-gpt4-turbo, model-zephyr-7B
|
||||||
|
model_object
|
||||||
|
|
||||||
|
# Delete model
|
||||||
|
DELETE https://localhost:1337/v1/models/{model_id} # json file name as {model_id} model-azure-openai-gpt4-turbo, model-zephyr-7B
|
||||||
|
|
||||||
TODO:
|
|
||||||
# Start model
|
|
||||||
# Stop model
|
# Stop model
|
||||||
|
PUT https://localhost:1337/v1/models/{model_id}/stop # json file name as {model_id} model-azure-openai-gpt4-turbo, model-zephyr-7B
|
||||||
|
|
||||||
|
# Start model
|
||||||
|
PUT https://localhost:1337/v1/models/{model_id}/start # json file name as {model_id} model-azure-openai-gpt4-turbo, model-zephyr-7B
|
||||||
|
{
|
||||||
|
"id": [string] # The model name to be used in `chat_completion` = model_id
|
||||||
|
"model_parameters": [jsonPayload],
|
||||||
|
"engine": [enum](llamacpp)
|
||||||
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
## Model Filesystem
|
## Model Filesystem
|
||||||
|
|
||||||
How `models` map onto your local filesystem
|
How `models` map onto your local filesystem
|
||||||
|
|
||||||
```sh
|
```shell=
|
||||||
/janroot
|
/janroot
|
||||||
/models
|
/models
|
||||||
/modelA
|
llama2-70b.json
|
||||||
model.json # Default model params
|
llama2-7b-gguf.json
|
||||||
modelA.gguf
|
|
||||||
modelA.bin
|
huggingface.co/ # Model registries (de-factor open source)
|
||||||
/modelB/*
|
meta-llama/
|
||||||
|
llama2-70b-chat-hf/
|
||||||
|
llama2-7b-chat/
|
||||||
|
thebloke/
|
||||||
|
llama2-70b-chat-hf-gguf/
|
||||||
|
llama2-7b-chat/
|
||||||
|
llama7b_q2_K_L.gguf
|
||||||
|
llama7b_q3_K_L.gguf
|
||||||
|
model.louis.ai/ # Private model registries
|
||||||
|
meta-llama/
|
||||||
|
llama2-70b-chat-hf-tensorrt-llm/
|
||||||
|
llama2-70b-chat-hf-awq/
|
||||||
model.json
|
model.json
|
||||||
modelB.gguf
|
thebloke/
|
||||||
/assistants
|
llava-1-5-gguf/ # Use case with multiple model
|
||||||
model.json # Defines model, default: looks in `/models`
|
mmproj.bin
|
||||||
/models # Optional /models folder that overrides root
|
model-q5.ggml
|
||||||
/modelA
|
|
||||||
model.json
|
llama-70b-finetune.bin
|
||||||
modelA.bin
|
llama-70b-finetune.json
|
||||||
```
|
```
|
||||||
Loading…
x
Reference in New Issue
Block a user