From cef3e122ff284e25edb7d5b339bc7456113a4dbc Mon Sep 17 00:00:00 2001
From: Faisal Amir <urmauur@gmail.com>
Date: Thu, 14 Aug 2025 13:39:38 +0700
Subject: [PATCH] chore: send attachment file when send message

---
 web-app/src/containers/ChatInput.tsx     |  17 ++-
 web-app/src/containers/ThreadContent.tsx | 152 ++++++++++++++++++++---
 web-app/src/hooks/useChat.ts             |  16 ++-
 web-app/src/lib/completion.ts            |  68 +++++++---
 web-app/src/lib/messages.ts              | 117 ++++++++++++++---
 5 files changed, 316 insertions(+), 54 deletions(-)

diff --git a/web-app/src/containers/ChatInput.tsx b/web-app/src/containers/ChatInput.tsx
index 0cd9c523e..e711804a8 100644
--- a/web-app/src/containers/ChatInput.tsx
+++ b/web-app/src/containers/ChatInput.tsx
@@ -100,11 +100,16 @@ const ChatInput = ({ model, className, initialMessage }: ChatInputProps) => {
       setMessage('Please select a model to start chatting.')
       return
     }
-    if (!prompt.trim()) {
+    if (!prompt.trim() && uploadedFiles.length === 0) {
       return
     }
     setMessage('')
-    sendMessage(prompt)
+    sendMessage(
+      prompt,
+      true,
+      uploadedFiles.length > 0 ? uploadedFiles : undefined
+    )
+    setUploadedFiles([])
   }
 
   useEffect(() => {
@@ -629,9 +634,13 @@ const ChatInput = ({ model, className, initialMessage }: ChatInputProps) => {
               </Button>
             ) : (
               <Button
-                variant={!prompt.trim() ? null : 'default'}
+                variant={
+                  !prompt.trim() && uploadedFiles.length === 0
+                    ? null
+                    : 'default'
+                }
                 size="icon"
-                disabled={!prompt.trim()}
+                disabled={!prompt.trim() && uploadedFiles.length === 0}
                 data-test-id="send-message-button"
                 onClick={() => handleSendMesage(prompt)}
               >
diff --git a/web-app/src/containers/ThreadContent.tsx b/web-app/src/containers/ThreadContent.tsx
index 54ba342cb..b64cd5310 100644
--- a/web-app/src/containers/ThreadContent.tsx
+++ b/web-app/src/containers/ThreadContent.tsx
@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import { ThreadMessage } from '@janhq/core'
 import { RenderMarkdown } from './RenderMarkdown'
 import React, { Fragment, memo, useCallback, useMemo, useState } from 'react'
@@ -144,7 +145,7 @@ export const ThreadContent = memo(
       isLastMessage?: boolean
       index?: number
       showAssistant?: boolean
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+
       streamTools?: any
       contextOverflowModal?: React.ReactNode | null
       updateMessage?: (item: ThreadMessage, message: string) => void
@@ -172,9 +173,12 @@ export const ThreadContent = memo(
     const { reasoningSegment, textSegment } = useMemo(() => {
       // Check for thinking formats
       const hasThinkTag = text.includes('<think>') && !text.includes('</think>')
-      const hasAnalysisChannel = text.includes('<|channel|>analysis<|message|>') && !text.includes('<|start|>assistant<|channel|>final<|message|>')
-      
-      if (hasThinkTag || hasAnalysisChannel) return { reasoningSegment: text, textSegment: '' }
+      const hasAnalysisChannel =
+        text.includes('<|channel|>analysis<|message|>') &&
+        !text.includes('<|start|>assistant<|channel|>final<|message|>')
+
+      if (hasThinkTag || hasAnalysisChannel)
+        return { reasoningSegment: text, textSegment: '' }
 
       // Check for completed think tag format
       const thinkMatch = text.match(/<think>([\s\S]*?)<\/think>/)
@@ -187,7 +191,9 @@ export const ThreadContent = memo(
       }
 
       // Check for completed analysis channel format
-      const analysisMatch = text.match(/<\|channel\|>analysis<\|message\|>([\s\S]*?)<\|start\|>assistant<\|channel\|>final<\|message\|>/)
+      const analysisMatch = text.match(
+        /<\|channel\|>analysis<\|message\|>([\s\S]*?)<\|start\|>assistant<\|channel\|>final<\|message\|>/
+      )
       if (analysisMatch?.index !== undefined) {
         const splitIndex = analysisMatch.index + analysisMatch[0].length
         return {
@@ -213,7 +219,49 @@ export const ThreadContent = memo(
       }
       if (toSendMessage) {
         deleteMessage(toSendMessage.thread_id, toSendMessage.id ?? '')
-        sendMessage(toSendMessage.content?.[0]?.text?.value || '')
+        // Extract text content and any attachments
+        const textContent =
+          toSendMessage.content?.find((c) => c.type === 'text')?.text?.value ||
+          ''
+        const attachments = toSendMessage.content
+          ?.filter(
+            (c) =>
+              (c.type === 'image_url' && c.image_url?.url) ||
+              ((c as any).type === 'file' && (c as any).file?.data)
+          )
+          .map((c) => {
+            if (c.type === 'image_url' && c.image_url?.url) {
+              const url = c.image_url.url
+              const [mimeType, base64] = url
+                .replace('data:', '')
+                .split(';base64,')
+              return {
+                name: 'image', // We don't have the original filename
+                type: mimeType,
+                size: 0, // We don't have the original size
+                base64: base64,
+                dataUrl: url,
+              }
+            } else if ((c as any).type === 'file' && (c as any).file?.data) {
+              const fileContent = (c as any).file
+              return {
+                name: fileContent.filename || 'file',
+                type: fileContent.media_type,
+                size: 0, // We don't have the original size
+                base64: fileContent.data,
+                dataUrl: `data:${fileContent.media_type};base64,${fileContent.data}`,
+              }
+            }
+            return null
+          })
+          .filter(Boolean) as Array<{
+          name: string
+          type: string
+          size: number
+          base64: string
+          dataUrl: string
+        }>
+        sendMessage(textContent, true, attachments)
       }
     }, [deleteMessage, getMessages, item, sendMessage])
 
@@ -255,22 +303,92 @@ export const ThreadContent = memo(
 
     return (
       <Fragment>
-        {item.content?.[0]?.text && item.role === 'user' && (
+        {item.role === 'user' && (
           <div className="w-full">
-            <div className="flex justify-end w-full h-full text-start break-words whitespace-normal">
-              <div className="bg-main-view-fg/4 relative text-main-view-fg p-2 rounded-md inline-block max-w-[80%] ">
-                <div className="select-text">
-                  <RenderMarkdown
-                    content={item.content?.[0].text.value}
-                    components={linkComponents}
-                    isUser
-                  />
+            {/* Render attachments above the message bubble */}
+            {item.content?.some(
+              (c) =>
+                (c.type === 'image_url' && c.image_url?.url) ||
+                ((c as any).type === 'file' && (c as any).file?.data)
+            ) && (
+              <div className="flex justify-end w-full mb-2">
+                <div className="flex flex-wrap gap-2 max-w-[80%] justify-end">
+                  {item.content
+                    ?.filter(
+                      (c) =>
+                        (c.type === 'image_url' && c.image_url?.url) ||
+                        ((c as any).type === 'file' && (c as any).file?.data)
+                    )
+                    .map((contentPart, index) => {
+                      // Handle images
+                      if (
+                        contentPart.type === 'image_url' &&
+                        contentPart.image_url?.url
+                      ) {
+                        return (
+                          <div key={index} className="relative">
+                            <img
+                              src={contentPart.image_url.url}
+                              alt="Uploaded attachment"
+                              className="size-40 rounded-md object-cover border border-main-view-fg/10"
+                            />
+                          </div>
+                        )
+                      }
+                      // Handle PDF files
+                      else if (
+                        (contentPart as any).type === 'file' &&
+                        (contentPart as any).file?.media_type ===
+                          'application/pdf'
+                      ) {
+                        const fileContent = (contentPart as any).file
+                        return (
+                          <div key={index} className="relative">
+                            <div className="w-40 h-40 bg-main-view-fg/5 border border-main-view-fg/10 rounded-md flex flex-col items-center justify-center p-4">
+                              <div className="text-2xl mb-2">📄</div>
+                              <div className="text-xs text-center text-main-view-fg/70 truncate w-full">
+                                {fileContent.filename || 'PDF Document'}
+                              </div>
+                              <div className="text-xs text-main-view-fg/50 mt-1">
+                                PDF
+                              </div>
+                            </div>
+                          </div>
+                        )
+                      }
+                      return null
+                    })}
                 </div>
               </div>
-            </div>
+            )}
+
+            {/* Render text content in the message bubble */}
+            {item.content?.some((c) => c.type === 'text' && c.text?.value) && (
+              <div className="flex justify-end w-full h-full text-start break-words whitespace-normal">
+                <div className="bg-main-view-fg/4 relative text-main-view-fg p-2 rounded-md inline-block max-w-[80%] ">
+                  <div className="select-text">
+                    {item.content
+                      ?.filter((c) => c.type === 'text' && c.text?.value)
+                      .map((contentPart, index) => (
+                        <div key={index}>
+                          <RenderMarkdown
+                            content={contentPart.text!.value}
+                            components={linkComponents}
+                            isUser
+                          />
+                        </div>
+                      ))}
+                  </div>
+                </div>
+              </div>
+            )}
+
             <div className="flex items-center justify-end gap-2 text-main-view-fg/60 text-xs mt-2">
               <EditDialog
-                message={item.content?.[0]?.text.value}
+                message={
+                  item.content?.find((c) => c.type === 'text')?.text?.value ||
+                  ''
+                }
                 setMessage={(message) => {
                   if (item.updateMessage) {
                     item.updateMessage(item, message)
diff --git a/web-app/src/hooks/useChat.ts b/web-app/src/hooks/useChat.ts
index e90d5b1c2..3ffd19318 100644
--- a/web-app/src/hooks/useChat.ts
+++ b/web-app/src/hooks/useChat.ts
@@ -203,7 +203,17 @@ export const useChat = () => {
   )
 
   const sendMessage = useCallback(
-    async (message: string, troubleshooting = true) => {
+    async (
+      message: string,
+      troubleshooting = true,
+      attachments?: Array<{
+        name: string
+        type: string
+        size: number
+        base64: string
+        dataUrl: string
+      }>
+    ) => {
       const activeThread = await getCurrentThread()
 
       resetTokenSpeed()
@@ -217,7 +227,7 @@ export const useChat = () => {
       updateStreamingContent(emptyThreadContent)
       // Do not add new message on retry
       if (troubleshooting)
-        addMessage(newUserThreadContent(activeThread.id, message))
+        addMessage(newUserThreadContent(activeThread.id, message, attachments))
       updateThreadTimestamp(activeThread.id)
       setPrompt('')
       try {
@@ -231,7 +241,7 @@ export const useChat = () => {
           messages,
           currentAssistant?.instructions
         )
-        if (troubleshooting) builder.addUserMessage(message)
+        if (troubleshooting) builder.addUserMessage(message, attachments)
 
         let isCompleted = false
 
diff --git a/web-app/src/lib/completion.ts b/web-app/src/lib/completion.ts
index 22ac724e9..94e4e07e5 100644
--- a/web-app/src/lib/completion.ts
+++ b/web-app/src/lib/completion.ts
@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import {
   ContentType,
   ChatCompletionRole,
@@ -50,11 +51,16 @@ export type ChatCompletionResponse =
  */
 export const newUserThreadContent = (
   threadId: string,
-  content: string
-): ThreadMessage => ({
-  type: 'text',
-  role: ChatCompletionRole.User,
-  content: [
+  content: string,
+  attachments?: Array<{
+    name: string
+    type: string
+    size: number
+    base64: string
+    dataUrl: string
+  }>
+): ThreadMessage => {
+  const contentParts = [
     {
       type: ContentType.Text,
       text: {
@@ -62,14 +68,46 @@ export const newUserThreadContent = (
         annotations: [],
       },
     },
-  ],
-  id: ulid(),
-  object: 'thread.message',
-  thread_id: threadId,
-  status: MessageStatus.Ready,
-  created_at: 0,
-  completed_at: 0,
-})
+  ]
+
+  // Add attachments to content array
+  if (attachments) {
+    attachments.forEach((attachment) => {
+      if (attachment.type.startsWith('image/')) {
+        contentParts.push({
+          type: ContentType.Image,
+          image_url: {
+            url: `data:${attachment.type};base64,${attachment.base64}`,
+            detail: 'auto',
+          },
+        } as any)
+      } else if (attachment.type === 'application/pdf') {
+        contentParts.push({
+          type: 'file' as any,
+          file: {
+            filename: attachment.name,
+            file_data: `data:${attachment.type};base64,${attachment.base64}`,
+            // Keep original data for local display purposes
+            data: attachment.base64,
+            media_type: attachment.type,
+          },
+        } as any)
+      }
+    })
+  }
+
+  return {
+    type: 'text',
+    role: ChatCompletionRole.User,
+    content: contentParts,
+    id: ulid(),
+    object: 'thread.message',
+    thread_id: threadId,
+    status: MessageStatus.Ready,
+    created_at: 0,
+    completed_at: 0,
+  }
+}
 /**
  * @fileoverview Helper functions for creating thread content.
  * These functions are used to create thread content objects
@@ -161,13 +199,11 @@ export const sendCompletion = async (
   if (
     thread.model.id &&
     !Object.values(models[providerName]).flat().includes(thread.model.id) &&
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     !tokenJS.extendedModelExist(providerName as any, thread.model.id) &&
     provider.provider !== 'llamacpp'
   ) {
     try {
       tokenJS.extendModelList(
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         providerName as any,
         thread.model.id,
         // This is to inherit the model capabilities from another built-in model
@@ -200,7 +236,7 @@ export const sendCompletion = async (
       ? await tokenJS.chat.completions.create(
           {
             stream: true,
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+
             provider: providerName as any,
             model: thread.model?.id,
             messages,
diff --git a/web-app/src/lib/messages.ts b/web-app/src/lib/messages.ts
index b187fb514..f26403ab8 100644
--- a/web-app/src/lib/messages.ts
+++ b/web-app/src/lib/messages.ts
@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import { ChatCompletionMessageParam } from 'token.js'
 import { ChatCompletionMessageToolCall } from 'openai/resources'
 import { ThreadMessage } from '@janhq/core'
@@ -19,32 +20,120 @@ export class CompletionMessagesBuilder {
     this.messages.push(
       ...messages
         .filter((e) => !e.metadata?.error)
-        .map<ChatCompletionMessageParam>(
-          (msg) =>
-            ({
+        .map<ChatCompletionMessageParam>((msg) => {
+          if (msg.role === 'assistant') {
+            return {
               role: msg.role,
-              content:
-                msg.role === 'assistant'
-                  ? this.normalizeContent(msg.content[0]?.text?.value || '.')
-                  : msg.content[0]?.text?.value || '.',
-            }) as ChatCompletionMessageParam
-        )
+              content: this.normalizeContent(
+                msg.content[0]?.text?.value || '.'
+              ),
+            } as ChatCompletionMessageParam
+          } else {
+            // For user messages, handle multimodal content
+            if (msg.content.length > 1) {
+              // Multiple content parts (text + images + files)
+              const content = msg.content.map((contentPart) => {
+                if (contentPart.type === 'text') {
+                  return {
+                    type: 'text',
+                    text: contentPart.text?.value || '',
+                  }
+                } else if (contentPart.type === 'image_url') {
+                  return {
+                    type: 'image_url',
+                    image_url: {
+                      url: contentPart.image_url?.url || '',
+                      detail: contentPart.image_url?.detail || 'auto',
+                    },
+                  }
+                } else if ((contentPart as any).type === 'file') {
+                  return {
+                    type: 'file',
+                    file: {
+                      filename: (contentPart as any).file?.filename || 'document.pdf',
+                      file_data: (contentPart as any).file?.file_data || (contentPart as any).file?.data ? `data:application/pdf;base64,${(contentPart as any).file.data}` : '',
+                    },
+                  }
+                }
+                return contentPart
+              })
+              return {
+                role: msg.role,
+                content,
+              } as ChatCompletionMessageParam
+            } else {
+              // Single text content
+              return {
+                role: msg.role,
+                content: msg.content[0]?.text?.value || '.',
+              } as ChatCompletionMessageParam
+            }
+          }
+        })
     )
   }
 
   /**
    * Add a user message to the messages array.
    * @param content - The content of the user message.
+   * @param attachments - Optional attachments for the message.
    */
-  addUserMessage(content: string) {
+  addUserMessage(
+    content: string,
+    attachments?: Array<{
+      name: string
+      type: string
+      size: number
+      base64: string
+      dataUrl: string
+    }>
+  ) {
     // Ensure no consecutive user messages
     if (this.messages[this.messages.length - 1]?.role === 'user') {
       this.messages.pop()
     }
-    this.messages.push({
-      role: 'user',
-      content: content,
-    })
+
+    // Handle multimodal content with attachments
+    if (attachments && attachments.length > 0) {
+      const messageContent: any[] = [
+        {
+          type: 'text',
+          text: content,
+        },
+      ]
+
+      // Add attachments (images and PDFs)
+      attachments.forEach((attachment) => {
+        if (attachment.type.startsWith('image/')) {
+          messageContent.push({
+            type: 'image_url',
+            image_url: {
+              url: `data:${attachment.type};base64,${attachment.base64}`,
+              detail: 'auto',
+            },
+          })
+        } else if (attachment.type === 'application/pdf') {
+          messageContent.push({
+            type: 'file',
+            file: {
+              filename: attachment.name,
+              file_data: `data:${attachment.type};base64,${attachment.base64}`,
+            },
+          })
+        }
+      })
+
+      this.messages.push({
+        role: 'user',
+        content: messageContent,
+      } as any)
+    } else {
+      // Text-only message
+      this.messages.push({
+        role: 'user',
+        content: content,
+      })
+    }
   }
 
   /**