Nicholai/jan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: abort inference stream implementation

Browse Source
This commit is contained in:
Louis 2023-11-21 20:49:17 +07:00
parent b5717e03fe
commit 664806e7f1
2 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
plugins/inference-plugin/src/helpers/sse.ts
View File

@ -4,7 +4,7 @@ import { Observable } from "rxjs";
* @param recentMessages - An array of recent messages to use as context for the inference. * @param recentMessages - An array of recent messages to use as context for the inference.
* @returns An Observable that emits the generated response as a string. * @returns An Observable that emits the generated response as a string.
*/ */
export function requestInference(recentMessages: any[]): Observable<string> { export function requestInference(recentMessages: any[], controller?: AbortController): Observable<string> {
return new Observable((subscriber) => { return new Observable((subscriber) => {
const requestBody = JSON.stringify({ const requestBody = JSON.stringify({
messages: recentMessages, messages: recentMessages,
@ -20,6 +20,7 @@ export function requestInference(recentMessages: any[]): Observable<string> {
"Access-Control-Allow-Origin": "*", "Access-Control-Allow-Origin": "*",
}, },
body: requestBody, body: requestBody,
signal: controller?.signal
}) })
.then(async (response) => { .then(async (response) => {
const stream = response.body; const stream = response.body;

7
plugins/inference-plugin/src/index.ts
View File

@ -28,6 +28,7 @@ import { fs } from "@janhq/core";
* It also subscribes to events emitted by the @janhq/core package and handles new message requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests.
*/ */
export default class JanInferencePlugin implements InferencePlugin { export default class JanInferencePlugin implements InferencePlugin {
controller = new AbortController();
/** /**
* Returns the type of the plugin. * Returns the type of the plugin.
* @returns {PluginType} The type of the plugin. * @returns {PluginType} The type of the plugin.
@ -75,7 +76,7 @@ export default class JanInferencePlugin implements InferencePlugin {
* @returns {Promise<void>} A promise that resolves when the streaming is stopped. * @returns {Promise<void>} A promise that resolves when the streaming is stopped.
*/ */
async stopInference(): Promise<void> { async stopInference(): Promise<void> {
// TODO: Implementation this.controller.abort();
} }
/** /**
@ -121,7 +122,9 @@ export default class JanInferencePlugin implements InferencePlugin {
}; };
events.emit(EventName.OnNewMessageResponse, message); events.emit(EventName.OnNewMessageResponse, message);
requestInference(data.messages).subscribe({ this.controller = new AbortController();
requestInference(data.messages, this.controller).subscribe({
next: (content) => { next: (content) => {
message.content = content; message.content = content;
events.emit(EventName.OnMessageResponseUpdate, message); events.emit(EventName.OnMessageResponseUpdate, message);