From dbc4bed40fbc6ca4facca0830d045cf4909ddffb Mon Sep 17 00:00:00 2001
From: Louis <louis@jan.ai>
Date: Tue, 24 Sep 2024 20:26:06 +0700
Subject: [PATCH] fix: #3673 - API responds with Request body is too large
 (#3729)

---
 .../node/api/restful/helper/builder.test.ts   | 41 +++++++++++++++++++
 core/src/node/api/restful/helper/builder.ts   |  6 ++-
 server/index.ts                               |  5 +++
 3 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/core/src/node/api/restful/helper/builder.test.ts b/core/src/node/api/restful/helper/builder.test.ts
index fef40c70a..eb21e9401 100644
--- a/core/src/node/api/restful/helper/builder.test.ts
+++ b/core/src/node/api/restful/helper/builder.test.ts
@@ -236,6 +236,47 @@ describe('builder helper functions', () => {
       })
     })
 
+    it('should return the error on status not ok', async () => {
+      const request = { body: { model: 'model1' } }
+      const mockSend = jest.fn()
+      const reply = {
+        code: jest.fn().mockReturnThis(),
+        send: jest.fn(),
+        headers: jest.fn().mockReturnValue({
+          send: mockSend,
+        }),
+        raw: {
+          writeHead: jest.fn(),
+          pipe: jest.fn(),
+        },
+      }
+
+      ;(existsSync as jest.Mock).mockReturnValue(true)
+      ;(readdirSync as jest.Mock).mockReturnValue(['file1'])
+      ;(readFileSync as jest.Mock).mockReturnValue(
+        JSON.stringify({ id: 'model1', engine: 'openai' })
+      )
+
+      // Mock fetch
+      const fetch = require('node-fetch')
+      fetch.mockResolvedValue({
+        status: 400,
+        headers: new Map([
+          ['content-type', 'application/json'],
+          ['x-request-id', '123456'],
+        ]),
+        body: { pipe: jest.fn() },
+        text: jest.fn().mockResolvedValue({ error: 'Mock error response' }),
+      })
+      await chatCompletions(request, reply)
+      expect(reply.code).toHaveBeenCalledWith(400)
+      expect(mockSend).toHaveBeenCalledWith(
+        expect.objectContaining({
+          error: 'Mock error response',
+        })
+      )
+    })
+
     it('should return the chat completions', async () => {
       const request = { body: { model: 'model1' } }
       const reply = {
diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts
index 1a8120918..db2000d69 100644
--- a/core/src/node/api/restful/helper/builder.ts
+++ b/core/src/node/api/restful/helper/builder.ts
@@ -353,8 +353,10 @@ export const chatCompletions = async (request: any, reply: any) => {
     body: JSON.stringify(request.body),
   })
   if (response.status !== 200) {
-    console.error(response)
-    reply.code(400).send(response)
+    // Forward the error response to client via reply
+    const responseBody = await response.text()
+    const responseHeaders = Object.fromEntries(response.headers)
+    reply.code(response.status).headers(responseHeaders).send(responseBody)
   } else {
     reply.raw.writeHead(200, {
       'Content-Type': request.body.stream === true ? 'text/event-stream' : 'application/json',
diff --git a/server/index.ts b/server/index.ts
index f82c4f5bc..e8a6eea78 100644
--- a/server/index.ts
+++ b/server/index.ts
@@ -67,6 +67,11 @@ export const startServer = async (configs?: ServerConfig): Promise<boolean> => {
     // Initialize Fastify server with logging
     server = fastify({
       logger: new Logger(),
+      // Set body limit to 100MB - Default is 1MB
+      // According to OpenAI - a batch input file can be up to 100 MB in size
+      // Whisper endpoints accept up to 25MB
+      // Vision endpoints accept up to 4MB
+      bodyLimit: 104_857_600
     })
 
     // Register CORS if enabled