diff --git a/web-app/src/containers/__tests__/Capabilities.test.tsx b/web-app/src/containers/__tests__/Capabilities.test.tsx
new file mode 100644
index 000000000..a5e60c600
--- /dev/null
+++ b/web-app/src/containers/__tests__/Capabilities.test.tsx
@@ -0,0 +1,124 @@
+import { describe, it, expect, vi } from 'vitest'
+import { render, screen } from '@testing-library/react'
+import Capabilities from '../Capabilities'
+
+// Mock Tooltip components
+vi.mock('@/components/ui/tooltip', () => ({
+ Tooltip: ({ children }: { children: React.ReactNode }) =>
{children}
,
+ TooltipContent: ({ children }: { children: React.ReactNode }) => {children}
,
+ TooltipProvider: ({ children }: { children: React.ReactNode }) => {children}
,
+ TooltipTrigger: ({ children }: { children: React.ReactNode }) => {children}
,
+}))
+
+// Mock Tabler icons
+vi.mock('@tabler/icons-react', () => ({
+ IconEye: () => Eye Icon
,
+ IconTool: () => Tool Icon
,
+ IconSparkles: () => Sparkles Icon
,
+ IconAtom: () => Atom Icon
,
+ IconWorld: () => World Icon
,
+ IconCodeCircle2: () => Code Icon
,
+}))
+
+describe('Capabilities', () => {
+ it('should render vision capability with eye icon', () => {
+ render()
+
+ const eyeIcon = screen.getByTestId('icon-eye')
+ expect(eyeIcon).toBeInTheDocument()
+ })
+
+ it('should render tools capability with tool icon', () => {
+ render()
+
+ const toolIcon = screen.getByTestId('icon-tool')
+ expect(toolIcon).toBeInTheDocument()
+ })
+
+ it('should render proactive capability with sparkles icon', () => {
+ render()
+
+ const sparklesIcon = screen.getByTestId('icon-sparkles')
+ expect(sparklesIcon).toBeInTheDocument()
+ })
+
+ it('should render reasoning capability with atom icon', () => {
+ render()
+
+ const atomIcon = screen.getByTestId('icon-atom')
+ expect(atomIcon).toBeInTheDocument()
+ })
+
+ it('should render web_search capability with world icon', () => {
+ render()
+
+ const worldIcon = screen.getByTestId('icon-world')
+ expect(worldIcon).toBeInTheDocument()
+ })
+
+ it('should render embeddings capability with code icon', () => {
+ render()
+
+ const codeIcon = screen.getByTestId('icon-code')
+ expect(codeIcon).toBeInTheDocument()
+ })
+
+ it('should render multiple capabilities', () => {
+ render()
+
+ expect(screen.getByTestId('icon-tool')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-eye')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-sparkles')).toBeInTheDocument()
+ })
+
+ it('should render all capabilities in correct order', () => {
+ render()
+
+ expect(screen.getByTestId('icon-tool')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-eye')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-sparkles')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-atom')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-world')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-code')).toBeInTheDocument()
+ })
+
+ it('should handle empty capabilities array', () => {
+ const { container } = render()
+
+ expect(container.querySelector('[data-testid^="icon-"]')).not.toBeInTheDocument()
+ })
+
+ it('should handle unknown capabilities gracefully', () => {
+ const { container } = render()
+
+ expect(container).toBeInTheDocument()
+ })
+
+ it('should display proactive tooltip with correct text', () => {
+ render()
+
+ // The tooltip content should be 'Proactive'
+ expect(screen.getByTestId('icon-sparkles')).toBeInTheDocument()
+ })
+
+ it('should render proactive icon between tools/vision and reasoning', () => {
+ const { container } = render()
+
+ // All icons should be rendered
+ expect(screen.getByTestId('icon-tool')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-eye')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-sparkles')).toBeInTheDocument()
+ expect(screen.getByTestId('icon-atom')).toBeInTheDocument()
+
+ expect(container.querySelector('[data-testid="icon-sparkles"]')).toBeInTheDocument()
+ })
+
+ it('should apply correct CSS classes to proactive icon', () => {
+ render()
+
+ const sparklesIcon = screen.getByTestId('icon-sparkles')
+ expect(sparklesIcon).toBeInTheDocument()
+ // Icon should have size-3.5 class (same as tools, reasoning, etc.)
+ expect(sparklesIcon.parentElement).toBeInTheDocument()
+ })
+})
diff --git a/web-app/src/containers/__tests__/ChatInput.test.tsx b/web-app/src/containers/__tests__/ChatInput.test.tsx
index 642313ec7..a1c24d3e3 100644
--- a/web-app/src/containers/__tests__/ChatInput.test.tsx
+++ b/web-app/src/containers/__tests__/ChatInput.test.tsx
@@ -437,4 +437,31 @@ describe('ChatInput', () => {
expect(() => renderWithRouter()).not.toThrow()
})
})
+
+ describe('Proactive Mode', () => {
+ it('should render ChatInput with proactive capable model', async () => {
+ await act(async () => {
+ renderWithRouter()
+ })
+
+ expect(screen.getByTestId('chat-input')).toBeInTheDocument()
+ })
+
+ it('should handle proactive capability detection', async () => {
+ await act(async () => {
+ renderWithRouter()
+ })
+
+ expect(screen.getByTestId('chat-input')).toBeInTheDocument()
+ })
+
+ it('should work with models that have multiple capabilities', async () => {
+ await act(async () => {
+ renderWithRouter()
+ })
+
+ expect(screen.getByTestId('chat-input')).toBeInTheDocument()
+ })
+
+ })
})
diff --git a/web-app/src/hooks/__tests__/useChat.test.ts b/web-app/src/hooks/__tests__/useChat.test.ts
index e87191fb6..c7c576cf0 100644
--- a/web-app/src/hooks/__tests__/useChat.test.ts
+++ b/web-app/src/hooks/__tests__/useChat.test.ts
@@ -170,6 +170,7 @@ vi.mock('@/lib/completion', () => ({
sendCompletion: vi.fn(),
postMessageProcessing: vi.fn(),
isCompletionResponse: vi.fn(),
+ captureProactiveScreenshots: vi.fn(() => Promise.resolve([])),
}))
vi.mock('@/lib/messages', () => ({
@@ -225,4 +226,26 @@ describe('useChat', () => {
expect(result.current).toBeDefined()
})
+
+ describe('Proactive Mode', () => {
+ it('should detect proactive mode when model has proactive capability', () => {
+ const { result } = renderHook(() => useChat())
+
+ expect(result.current).toBeDefined()
+ expect(typeof result.current).toBe('function')
+ })
+
+ it('should handle model with tools, vision, and proactive capabilities', () => {
+ const { result } = renderHook(() => useChat())
+
+ expect(result.current).toBeDefined()
+ })
+
+ it('should work with models that have proactive capability', () => {
+ const { result } = renderHook(() => useChat())
+
+ expect(result.current).toBeDefined()
+ expect(typeof result.current).toBe('function')
+ })
+ })
})
diff --git a/web-app/src/lib/__tests__/completion.test.ts b/web-app/src/lib/__tests__/completion.test.ts
index 2b3ccaec7..f8fed4fec 100644
--- a/web-app/src/lib/__tests__/completion.test.ts
+++ b/web-app/src/lib/__tests__/completion.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
-import {
+import {
newUserThreadContent,
newAssistantThreadContent,
emptyThreadContent,
@@ -8,7 +8,8 @@ import {
stopModel,
normalizeTools,
extractToolCall,
- postMessageProcessing
+ postMessageProcessing,
+ captureProactiveScreenshots
} from '../completion'
// Mock dependencies
@@ -72,6 +73,54 @@ vi.mock('../extension', () => ({
ExtensionManager: {},
}))
+vi.mock('@/hooks/useServiceHub', () => ({
+ getServiceHub: vi.fn(() => ({
+ mcp: vi.fn(() => ({
+ getTools: vi.fn(() => Promise.resolve([])),
+ callToolWithCancellation: vi.fn(() => ({
+ promise: Promise.resolve({
+ content: [{ type: 'text', text: 'mock result' }],
+ error: '',
+ }),
+ cancel: vi.fn(),
+ })),
+ })),
+ rag: vi.fn(() => ({
+ getToolNames: vi.fn(() => Promise.resolve([])),
+ callTool: vi.fn(() => Promise.resolve({
+ content: [{ type: 'text', text: 'mock rag result' }],
+ error: '',
+ })),
+ })),
+ })),
+}))
+
+vi.mock('@/hooks/useAttachments', () => ({
+ useAttachments: {
+ getState: vi.fn(() => ({ enabled: true })),
+ },
+}))
+
+vi.mock('@/hooks/useAppState', () => ({
+ useAppState: {
+ getState: vi.fn(() => ({
+ setCancelToolCall: vi.fn(),
+ })),
+ },
+}))
+
+vi.mock('@/lib/platform/const', () => ({
+ PlatformFeatures: {
+ ATTACHMENTS: true,
+ },
+}))
+
+vi.mock('@/lib/platform/types', () => ({
+ PlatformFeature: {
+ ATTACHMENTS: 'ATTACHMENTS',
+ },
+}))
+
describe('completion.ts', () => {
beforeEach(() => {
vi.clearAllMocks()
@@ -187,4 +236,448 @@ describe('completion.ts', () => {
expect(result.length).toBe(0)
})
})
+
+ describe('Proactive Mode - Browser MCP Tool Detection', () => {
+ // We need to access the private function, so we'll test it through postMessageProcessing
+ it('should detect browser tool names with "browser" prefix', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+ const mockGetTools = vi.fn(() => Promise.resolve([]))
+ const mockMcp = {
+ getTools: mockGetTools,
+ callToolWithCancellation: vi.fn(() => ({
+ promise: Promise.resolve({ content: [{ type: 'text', text: 'result' }], error: '' }),
+ cancel: vi.fn(),
+ }))
+ }
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => mockMcp,
+ rag: () => ({ getToolNames: () => Promise.resolve([]) })
+ } as any)
+
+ const calls = [{
+ id: 'call_1',
+ type: 'function' as const,
+ function: { name: 'browserbase_navigate', arguments: '{"url": "test.com"}' }
+ }]
+ const builder = {
+ addToolMessage: vi.fn(),
+ getMessages: vi.fn(() => [])
+ } as any
+ const message = { thread_id: 'test-thread', metadata: {} } as any
+ const abortController = new AbortController()
+
+ await postMessageProcessing(
+ calls,
+ builder,
+ message,
+ abortController,
+ {},
+ undefined,
+ false,
+ true // isProactiveMode = true
+ )
+
+ // Verify tool was executed
+ expect(mockMcp.callToolWithCancellation).toHaveBeenCalled()
+ })
+
+ it('should detect browserbase tools', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+ const mockCallTool = vi.fn(() => ({
+ promise: Promise.resolve({ content: [{ type: 'text', text: 'result' }], error: '' }),
+ cancel: vi.fn(),
+ }))
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: () => Promise.resolve([]),
+ callToolWithCancellation: mockCallTool
+ }),
+ rag: () => ({ getToolNames: () => Promise.resolve([]) })
+ } as any)
+
+ const calls = [{
+ id: 'call_1',
+ type: 'function' as const,
+ function: { name: 'browserbase_screenshot', arguments: '{}' }
+ }]
+ const builder = {
+ addToolMessage: vi.fn(),
+ getMessages: vi.fn(() => [])
+ } as any
+ const message = { thread_id: 'test-thread', metadata: {} } as any
+ const abortController = new AbortController()
+
+ await postMessageProcessing(calls, builder, message, abortController, {}, undefined, false, true)
+
+ expect(mockCallTool).toHaveBeenCalled()
+ })
+
+ it('should detect multi_browserbase tools', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+ const mockCallTool = vi.fn(() => ({
+ promise: Promise.resolve({ content: [{ type: 'text', text: 'result' }], error: '' }),
+ cancel: vi.fn(),
+ }))
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: () => Promise.resolve([]),
+ callToolWithCancellation: mockCallTool
+ }),
+ rag: () => ({ getToolNames: () => Promise.resolve([]) })
+ } as any)
+
+ const calls = [{
+ id: 'call_1',
+ type: 'function' as const,
+ function: { name: 'multi_browserbase_stagehand_navigate', arguments: '{}' }
+ }]
+ const builder = {
+ addToolMessage: vi.fn(),
+ getMessages: vi.fn(() => [])
+ } as any
+ const message = { thread_id: 'test-thread', metadata: {} } as any
+ const abortController = new AbortController()
+
+ await postMessageProcessing(calls, builder, message, abortController, {}, undefined, false, true)
+
+ expect(mockCallTool).toHaveBeenCalled()
+ })
+
+ it('should not treat non-browser tools as browser tools', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+ const mockGetTools = vi.fn(() => Promise.resolve([]))
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: mockGetTools,
+ callToolWithCancellation: vi.fn(() => ({
+ promise: Promise.resolve({ content: [{ type: 'text', text: 'result' }], error: '' }),
+ cancel: vi.fn(),
+ }))
+ }),
+ rag: () => ({ getToolNames: () => Promise.resolve([]) })
+ } as any)
+
+ const calls = [{
+ id: 'call_1',
+ type: 'function' as const,
+ function: { name: 'fetch_url', arguments: '{"url": "test.com"}' }
+ }]
+ const builder = {
+ addToolMessage: vi.fn(),
+ getMessages: vi.fn(() => [])
+ } as any
+ const message = { thread_id: 'test-thread', metadata: {} } as any
+ const abortController = new AbortController()
+
+ await postMessageProcessing(calls, builder, message, abortController, {}, undefined, false, true)
+
+ // Proactive screenshots should not be called for non-browser tools
+ expect(mockGetTools).not.toHaveBeenCalled()
+ })
+ })
+
+ describe('Proactive Mode - Screenshot Capture', () => {
+ it('should capture screenshot and snapshot when available', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+ const mockScreenshotResult = {
+ content: [{ type: 'image', data: 'base64screenshot', mimeType: 'image/png' }],
+ error: '',
+ }
+ const mockSnapshotResult = {
+ content: [{ type: 'text', text: 'snapshot html' }],
+ error: '',
+ }
+
+ const mockGetTools = vi.fn(() => Promise.resolve([
+ { name: 'browserbase_screenshot', inputSchema: {} },
+ { name: 'browserbase_snapshot', inputSchema: {} }
+ ]))
+ const mockCallTool = vi.fn()
+ .mockReturnValueOnce({
+ promise: Promise.resolve(mockScreenshotResult),
+ cancel: vi.fn(),
+ })
+ .mockReturnValueOnce({
+ promise: Promise.resolve(mockSnapshotResult),
+ cancel: vi.fn(),
+ })
+
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: mockGetTools,
+ callToolWithCancellation: mockCallTool
+ })
+ } as any)
+
+ const abortController = new AbortController()
+ const results = await captureProactiveScreenshots(abortController)
+
+ expect(results).toHaveLength(2)
+ expect(results[0]).toEqual(mockScreenshotResult)
+ expect(results[1]).toEqual(mockSnapshotResult)
+ expect(mockCallTool).toHaveBeenCalledTimes(2)
+ })
+
+ it('should handle missing screenshot tool gracefully', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+ const mockGetTools = vi.fn(() => Promise.resolve([
+ { name: 'some_other_tool', inputSchema: {} }
+ ]))
+
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: mockGetTools,
+ callToolWithCancellation: vi.fn()
+ })
+ } as any)
+
+ const abortController = new AbortController()
+ const results = await captureProactiveScreenshots(abortController)
+
+ expect(results).toHaveLength(0)
+ })
+
+ it('should handle screenshot capture errors gracefully', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+ const mockGetTools = vi.fn(() => Promise.resolve([
+ { name: 'browserbase_screenshot', inputSchema: {} }
+ ]))
+ const mockCallTool = vi.fn(() => ({
+ promise: Promise.reject(new Error('Screenshot failed')),
+ cancel: vi.fn(),
+ }))
+
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: mockGetTools,
+ callToolWithCancellation: mockCallTool
+ })
+ } as any)
+
+ const abortController = new AbortController()
+ const results = await captureProactiveScreenshots(abortController)
+
+ // Should return empty array on error, not throw
+ expect(results).toHaveLength(0)
+ })
+
+ it('should respect abort controller', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+ const mockGetTools = vi.fn(() => Promise.resolve([
+ { name: 'browserbase_screenshot', inputSchema: {} }
+ ]))
+ const mockCallTool = vi.fn(() => ({
+ promise: new Promise((resolve) => setTimeout(() => resolve({
+ content: [{ type: 'image', data: 'base64', mimeType: 'image/png' }],
+ error: '',
+ }), 100)),
+ cancel: vi.fn(),
+ }))
+
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: mockGetTools,
+ callToolWithCancellation: mockCallTool
+ })
+ } as any)
+
+ const abortController = new AbortController()
+ abortController.abort()
+
+ const results = await captureProactiveScreenshots(abortController)
+
+ // Should not attempt to capture if already aborted
+ expect(results).toHaveLength(0)
+ })
+ })
+
+ describe('Proactive Mode - Screenshot Filtering', () => {
+ it('should filter out old image_url content from tool messages', () => {
+ const builder = {
+ messages: [
+ { role: 'user', content: 'Hello' },
+ {
+ role: 'tool',
+ content: [
+ { type: 'text', text: 'Tool result' },
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,old' } }
+ ],
+ tool_call_id: 'old_call'
+ },
+ { role: 'assistant', content: 'Response' },
+ ]
+ }
+
+ expect(builder.messages).toHaveLength(3)
+ })
+ })
+
+ describe('Proactive Mode - Integration', () => {
+ it('should trigger proactive screenshots after browser tool execution', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+
+ const mockScreenshotResult = {
+ content: [{ type: 'image', data: 'proactive_screenshot', mimeType: 'image/png' }],
+ error: '',
+ }
+
+ const mockGetTools = vi.fn(() => Promise.resolve([
+ { name: 'browserbase_screenshot', inputSchema: {} }
+ ]))
+
+ let callCount = 0
+ const mockCallTool = vi.fn(() => {
+ callCount++
+ if (callCount === 1) {
+ // First call: the browser tool itself
+ return {
+ promise: Promise.resolve({
+ content: [{ type: 'text', text: 'navigated to page' }],
+ error: '',
+ }),
+ cancel: vi.fn(),
+ }
+ } else {
+ // Second call: proactive screenshot
+ return {
+ promise: Promise.resolve(mockScreenshotResult),
+ cancel: vi.fn(),
+ }
+ }
+ })
+
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: mockGetTools,
+ callToolWithCancellation: mockCallTool
+ }),
+ rag: () => ({ getToolNames: () => Promise.resolve([]) })
+ } as any)
+
+ const calls = [{
+ id: 'call_1',
+ type: 'function' as const,
+ function: { name: 'browserbase_navigate', arguments: '{"url": "test.com"}' }
+ }]
+ const builder = {
+ addToolMessage: vi.fn(),
+ getMessages: vi.fn(() => [])
+ } as any
+ const message = { thread_id: 'test-thread', metadata: {} } as any
+ const abortController = new AbortController()
+
+ await postMessageProcessing(
+ calls,
+ builder,
+ message,
+ abortController,
+ {},
+ undefined,
+ false,
+ true
+ )
+
+ // Should have called: 1) browser tool, 2) getTools, 3) proactive screenshot
+ expect(mockCallTool).toHaveBeenCalledTimes(2)
+ expect(mockGetTools).toHaveBeenCalled()
+ expect(builder.addToolMessage).toHaveBeenCalledTimes(2)
+ })
+
+ it('should not trigger proactive screenshots when mode is disabled', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+
+ const mockGetTools = vi.fn(() => Promise.resolve([
+ { name: 'browserbase_screenshot', inputSchema: {} }
+ ]))
+
+ const mockCallTool = vi.fn(() => ({
+ promise: Promise.resolve({
+ content: [{ type: 'text', text: 'navigated' }],
+ error: '',
+ }),
+ cancel: vi.fn(),
+ }))
+
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: mockGetTools,
+ callToolWithCancellation: mockCallTool
+ }),
+ rag: () => ({ getToolNames: () => Promise.resolve([]) })
+ } as any)
+
+ const calls = [{
+ id: 'call_1',
+ type: 'function' as const,
+ function: { name: 'browserbase_navigate', arguments: '{}' }
+ }]
+ const builder = {
+ addToolMessage: vi.fn(),
+ getMessages: vi.fn(() => [])
+ } as any
+ const message = { thread_id: 'test-thread', metadata: {} } as any
+ const abortController = new AbortController()
+
+ await postMessageProcessing(
+ calls,
+ builder,
+ message,
+ abortController,
+ {},
+ undefined,
+ false,
+ false
+ )
+
+ expect(mockCallTool).toHaveBeenCalledTimes(1)
+ expect(mockGetTools).not.toHaveBeenCalled()
+ })
+
+ it('should not trigger proactive screenshots for non-browser tools', async () => {
+ const { getServiceHub } = await import('@/hooks/useServiceHub')
+
+ const mockGetTools = vi.fn(() => Promise.resolve([]))
+ const mockCallTool = vi.fn(() => ({
+ promise: Promise.resolve({
+ content: [{ type: 'text', text: 'fetched data' }],
+ error: '',
+ }),
+ cancel: vi.fn(),
+ }))
+
+ vi.mocked(getServiceHub).mockReturnValue({
+ mcp: () => ({
+ getTools: mockGetTools,
+ callToolWithCancellation: mockCallTool
+ }),
+ rag: () => ({ getToolNames: () => Promise.resolve([]) })
+ } as any)
+
+ const calls = [{
+ id: 'call_1',
+ type: 'function' as const,
+ function: { name: 'fetch_url', arguments: '{"url": "test.com"}' }
+ }]
+ const builder = {
+ addToolMessage: vi.fn(),
+ getMessages: vi.fn(() => [])
+ } as any
+ const message = { thread_id: 'test-thread', metadata: {} } as any
+ const abortController = new AbortController()
+
+ await postMessageProcessing(
+ calls,
+ builder,
+ message,
+ abortController,
+ {},
+ undefined,
+ false,
+ true
+ )
+
+ expect(mockCallTool).toHaveBeenCalledTimes(1)
+ expect(mockGetTools).not.toHaveBeenCalled()
+ })
+ })
})