Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: wrong engine handling (#2363)

Browse Source
This commit is contained in:
Louis 2024-03-14 23:59:42 +07:00 committed by GitHub
parent e40d0481b7
commit 58e12f35c9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extensions/tensorrt-llm-extension/src/index.ts
View File

@ -129,6 +129,8 @@ export default class TensorRTLLMExtension extends LocalOAIEngine {
}
async onModelInit(model: Model): Promise<void> {
if (model.engine !== this.provider) return
if ((await this.installationState()) === 'Installed')
return super.onModelInit(model)
else {
@ -138,7 +140,6 @@ export default class TensorRTLLMExtension extends LocalOAIEngine {
message: 'EXTENSION_IS_NOT_INSTALLED::TensorRT-LLM extension',
},
})
return
}
}
@ -162,6 +163,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine {
}
inference(data: MessageRequest): void {
if (!this.isRunning) return
// TensorRT LLM Extension supports streaming only
if (data.model) data.model.parameters.stream = true
super.inference(data)