From 4f2a3b7b2786d6e2a4e277cffe74d0f5c3d5366d Mon Sep 17 00:00:00 2001 From: hiro Date: Tue, 5 Dec 2023 00:32:01 +0700 Subject: [PATCH] fix: Add dynamic values from engine settings and model params to inference request --- .../inference-nitro-extension/src/helpers/sse.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/extensions/inference-nitro-extension/src/helpers/sse.ts b/extensions/inference-nitro-extension/src/helpers/sse.ts index f427e443c..4da520fe3 100644 --- a/extensions/inference-nitro-extension/src/helpers/sse.ts +++ b/extensions/inference-nitro-extension/src/helpers/sse.ts @@ -1,3 +1,4 @@ +import { Model } from "@janhq/core"; import { Observable } from "rxjs"; /** * Sends a request to the inference server to generate a response based on the recent messages. @@ -6,21 +7,23 @@ import { Observable } from "rxjs"; */ export function requestInference( recentMessages: any[], + engine: EngineSettings, + model: Model, controller?: AbortController ): Observable { return new Observable((subscriber) => { const requestBody = JSON.stringify({ messages: recentMessages, - stream: true, - model: "gpt-3.5-turbo", - max_tokens: 2048, + model: model.id, + stream: model.parameters.stream || true, + max_tokens: model.parameters.max_tokens || 2048, }); fetch(INFERENCE_URL, { method: "POST", headers: { "Content-Type": "application/json", - Accept: "text/event-stream", "Access-Control-Allow-Origin": "*", + Accept: "text/event-stream", }, body: requestBody, signal: controller?.signal,