Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: add fallback as default endpoint for inference engine (#2669)

Browse Source 
Co-authored-by: James <james@jan.ai>
This commit is contained in:
NamH 2024-04-10 18:15:20 +07:00 committed by GitHub
parent 69f73b86c4
commit 8917be5ef3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 52 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/browser/extensions/engines/OAIEngine.ts
View File

@ -102,7 +102,7 @@ export abstract class OAIEngine extends AIEngine {
events.emit(MessageEvent.OnMessageUpdate, message)
},
error: async (err: any) => {
console.error(`Inference error:`, err)
console.error(`Inference error:`, JSON.stringify(err))
if (this.isCancelled || message.content.length) {
message.status = MessageStatus.Stopped
events.emit(MessageEvent.OnMessageUpdate, message)

2
extensions/inference-groq-extension/resources/settings.json
View File

@ -5,7 +5,7 @@
"description": "The endpoint to use for chat completions. See the [Groq documentation](https://console.groq.com/docs/openai) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Chat Completions Endpoint",
"placeholder": "https://api.groq.com/openai/v1/chat/completions",
"value": "https://api.groq.com/openai/v1/chat/completions"
}
},

14
extensions/inference-groq-extension/src/index.ts
View File

@ -6,7 +6,7 @@
* @module inference-groq-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core'
declare const SETTINGS: Array<any>
declare const MODELS: Array<any>
@ -43,7 +43,17 @@ export default class JanInferenceGroqExtension extends RemoteOAIEngine {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = value as string
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}

2
extensions/inference-mistral-extension/resources/settings.json
View File

@ -5,7 +5,7 @@
"description": "The endpoint to use for chat completions. See the [Mistral API documentation](https://docs.mistral.ai/api/#operation/createChatCompletion) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Chat Completions Endpoint",
"placeholder": "https://api.mistral.ai/v1/chat/completions",
"value": "https://api.mistral.ai/v1/chat/completions"
}
},

12
extensions/inference-mistral-extension/src/index.ts
View File

@ -42,7 +42,17 @@ export default class JanInferenceMistralExtension extends RemoteOAIEngine {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = value as string
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}

2
extensions/inference-openai-extension/resources/settings.json
View File

@ -5,7 +5,7 @@
"description": "The endpoint to use for chat completions. See the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/chat/create) for more information.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Chat Completions Endpoint",
"placeholder": "https://api.openai.com/v1/chat/completions",
"value": "https://api.openai.com/v1/chat/completions"
}
},

14
extensions/inference-openai-extension/src/index.ts
View File

@ -6,7 +6,7 @@
* @module inference-openai-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core'
declare const SETTINGS: Array<any>
declare const MODELS: Array<any>
@ -43,7 +43,17 @@ export default class JanInferenceOpenAIExtension extends RemoteOAIEngine {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = value as string
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}

2
extensions/inference-triton-trtllm-extension/resources/settings.json
View File

@ -5,7 +5,7 @@
"description": "The endpoint to use for chat completions.",
"controllerType": "input",
"controllerProps": {
"placeholder": "Chat Completions Endpoint",
"placeholder": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate",
"value": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate"
}
},

14
extensions/inference-triton-trtllm-extension/src/index.ts
View File

@ -6,7 +6,7 @@
* @module inference-nvidia-triton-trt-llm-extension/src/index
*/
import { RemoteOAIEngine } from '@janhq/core'
import { RemoteOAIEngine, SettingComponentProps } from '@janhq/core'
declare const SETTINGS: Array<any>
enum Settings {
@ -43,7 +43,17 @@ export default class JanInferenceTritonTrtLLMExtension extends RemoteOAIEngine {
if (key === Settings.apiKey) {
this.apiKey = value as string
} else if (key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = value as string
if (typeof value !== 'string') return
if (value.trim().length === 0) {
SETTINGS.forEach((setting) => {
if (setting.key === Settings.chatCompletionsEndPoint) {
this.inferenceUrl = setting.controllerProps.value as string
}
})
} else {
this.inferenceUrl = value
}
}
}
}