diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 99b8ee273..d0cc7cd7c 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -2,8 +2,11 @@ * IPC Handlers * Registers all IPC handlers for main-renderer communication */ -import { ipcMain, BrowserWindow, shell, dialog, app } from 'electron'; -import { existsSync } from 'node:fs'; +import { ipcMain, BrowserWindow, shell, dialog, app, nativeImage } from 'electron'; +import { existsSync, copyFileSync, statSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, extname, basename } from 'node:path'; +import crypto from 'node:crypto'; import { GatewayManager } from '../gateway/manager'; import { ClawHubService, ClawHubSearchParams, ClawHubInstallParams, ClawHubUninstallParams } from '../gateway/clawhub'; import { @@ -90,6 +93,9 @@ export function registerIpcHandlers( // WhatsApp handlers registerWhatsAppHandlers(mainWindow); + + // File staging handlers (upload/send separation) + registerFileHandlers(); } /** @@ -418,6 +424,78 @@ function registerGatewayHandlers( } }); + // Chat send with media — reads staged files from disk and builds attachments. + // Raster images (png/jpg/gif/webp) are inlined as base64 vision attachments. + // All other files are referenced by path in the message text so the model + // can access them via tools (the same format channels use). + const VISION_MIME_TYPES = new Set([ + 'image/png', 'image/jpeg', 'image/bmp', 'image/webp', + ]); + + ipcMain.handle('chat:sendWithMedia', async (_, params: { + sessionKey: string; + message: string; + deliver?: boolean; + idempotencyKey: string; + media?: Array<{ filePath: string; mimeType: string; fileName: string }>; + }) => { + try { + let message = params.message; + const imageAttachments: Array<{ type: string; mimeType: string; fileName: string; content: string }> = []; + const fileReferences: string[] = []; + + if (params.media && params.media.length > 0) { + for (const m of params.media) { + logger.info(`[chat:sendWithMedia] Processing file: ${m.fileName} (${m.mimeType}), path: ${m.filePath}, exists: ${existsSync(m.filePath)}, isVision: ${VISION_MIME_TYPES.has(m.mimeType)}`); + if (VISION_MIME_TYPES.has(m.mimeType)) { + // Raster image — inline as base64 vision attachment + const fileBuffer = readFileSync(m.filePath); + logger.info(`[chat:sendWithMedia] Read ${fileBuffer.length} bytes, base64 length: ${fileBuffer.toString('base64').length}`); + imageAttachments.push({ + type: 'image', + mimeType: m.mimeType, + fileName: m.fileName, + content: fileBuffer.toString('base64'), + }); + } else { + // Non-vision file — reference by path (same format as channel inbound media) + fileReferences.push( + `[media attached: ${m.filePath} (${m.mimeType}) | ${m.filePath}]`, + ); + } + } + } + + // Append file references to message text so the model knows about them + if (fileReferences.length > 0) { + const refs = fileReferences.join('\n'); + message = message ? `${message}\n\n${refs}` : refs; + } + + const rpcParams: Record = { + sessionKey: params.sessionKey, + message, + deliver: params.deliver ?? false, + idempotencyKey: params.idempotencyKey, + }; + + if (imageAttachments.length > 0) { + rpcParams.attachments = imageAttachments; + } + + logger.info(`[chat:sendWithMedia] Sending: message="${message.substring(0, 100)}", imageAttachments=${imageAttachments.length}, fileRefs=${fileReferences.length}`); + + // Use a longer timeout when attachments are present (120s vs default 30s) + const timeoutMs = imageAttachments.length > 0 ? 120000 : 30000; + const result = await gatewayManager.rpc('chat.send', rpcParams, timeoutMs); + logger.info(`[chat:sendWithMedia] RPC result: ${JSON.stringify(result)}`); + return { success: true, result }; + } catch (error) { + logger.error(`[chat:sendWithMedia] Error: ${String(error)}`); + return { success: false, error: String(error) }; + } + }); + // Get the Control UI URL with token for embedding ipcMain.handle('gateway:getControlUiUrl', async () => { try { @@ -1343,3 +1421,140 @@ function registerWindowHandlers(mainWindow: BrowserWindow): void { return mainWindow.isMaximized(); }); } + +// ── Mime type helpers ──────────────────────────────────────────── + +const EXT_MIME_MAP: Record = { + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + '.bmp': 'image/bmp', + '.ico': 'image/x-icon', + '.mp4': 'video/mp4', + '.webm': 'video/webm', + '.mov': 'video/quicktime', + '.avi': 'video/x-msvideo', + '.mkv': 'video/x-matroska', + '.mp3': 'audio/mpeg', + '.wav': 'audio/wav', + '.ogg': 'audio/ogg', + '.flac': 'audio/flac', + '.pdf': 'application/pdf', + '.zip': 'application/zip', + '.gz': 'application/gzip', + '.tar': 'application/x-tar', + '.7z': 'application/x-7z-compressed', + '.rar': 'application/vnd.rar', + '.json': 'application/json', + '.xml': 'application/xml', + '.csv': 'text/csv', + '.txt': 'text/plain', + '.md': 'text/markdown', + '.html': 'text/html', + '.css': 'text/css', + '.js': 'text/javascript', + '.ts': 'text/typescript', + '.py': 'text/x-python', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.ppt': 'application/vnd.ms-powerpoint', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', +}; + +function getMimeType(ext: string): string { + return EXT_MIME_MAP[ext.toLowerCase()] || 'application/octet-stream'; +} + +function mimeToExt(mimeType: string): string { + for (const [ext, mime] of Object.entries(EXT_MIME_MAP)) { + if (mime === mimeType) return ext; + } + return ''; +} + +const OUTBOUND_DIR = join(homedir(), '.openclaw', 'media', 'outbound'); + +/** + * Generate a small preview data URL for image files. + * Uses Electron nativeImage to resize large images for thumbnails. + */ +function generateImagePreview(filePath: string, mimeType: string): string | null { + try { + const img = nativeImage.createFromPath(filePath); + if (img.isEmpty()) return null; + const size = img.getSize(); + // If image is large, resize for thumbnail + if (size.width > 256 || size.height > 256) { + const resized = img.resize({ width: 256, height: 256 }); + return `data:image/png;base64,${resized.toPNG().toString('base64')}`; + } + // Small image — use original + const buf = readFileSync(filePath); + return `data:${mimeType};base64,${buf.toString('base64')}`; + } catch { + return null; + } +} + +/** + * File staging IPC handlers + * Stage files to ~/.openclaw/media/outbound/ for gateway access + */ +function registerFileHandlers(): void { + // Stage files from real disk paths (used with dialog:open) + ipcMain.handle('file:stage', async (_, filePaths: string[]) => { + mkdirSync(OUTBOUND_DIR, { recursive: true }); + + const results = []; + for (const filePath of filePaths) { + const id = crypto.randomUUID(); + const ext = extname(filePath); + const stagedPath = join(OUTBOUND_DIR, `${id}${ext}`); + copyFileSync(filePath, stagedPath); + + const stat = statSync(stagedPath); + const mimeType = getMimeType(ext); + const fileName = basename(filePath); + + // Generate preview for images + let preview: string | null = null; + if (mimeType.startsWith('image/')) { + preview = generateImagePreview(stagedPath, mimeType); + } + + results.push({ id, fileName, mimeType, fileSize: stat.size, stagedPath, preview }); + } + return results; + }); + + // Stage file from buffer (used for clipboard paste / drag-drop) + ipcMain.handle('file:stageBuffer', async (_, payload: { + base64: string; + fileName: string; + mimeType: string; + }) => { + mkdirSync(OUTBOUND_DIR, { recursive: true }); + + const id = crypto.randomUUID(); + const ext = extname(payload.fileName) || mimeToExt(payload.mimeType); + const stagedPath = join(OUTBOUND_DIR, `${id}${ext}`); + const buffer = Buffer.from(payload.base64, 'base64'); + writeFileSync(stagedPath, buffer); + + const mimeType = payload.mimeType || getMimeType(ext); + const fileSize = buffer.length; + + // Generate preview for images + let preview: string | null = null; + if (mimeType.startsWith('image/')) { + preview = generateImagePreview(stagedPath, mimeType); + } + + return { id, fileName: payload.fileName, mimeType, fileSize, stagedPath, preview }; + }); +} diff --git a/electron/preload/index.ts b/electron/preload/index.ts index 972c20ed7..80ad64cdb 100644 --- a/electron/preload/index.ts +++ b/electron/preload/index.ts @@ -115,6 +115,11 @@ const electronAPI = { 'log:getFilePath', 'log:getDir', 'log:listFiles', + // File staging + 'file:stage', + 'file:stageBuffer', + // Chat send with media (reads staged files in main process) + 'chat:sendWithMedia', // OpenClaw extras 'openclaw:getDir', 'openclaw:getConfigDir', diff --git a/src/pages/Chat/ChatInput.tsx b/src/pages/Chat/ChatInput.tsx index e7e79f3f5..29462e8ac 100644 --- a/src/pages/Chat/ChatInput.tsx +++ b/src/pages/Chat/ChatInput.tsx @@ -1,65 +1,76 @@ /** * Chat Input Component - * Textarea with send button and image upload support. + * Textarea with send button and universal file upload support. * Enter to send, Shift+Enter for new line. - * Supports: file picker, clipboard paste, drag & drop. + * Supports: native file picker, clipboard paste, drag & drop. + * Files are staged to disk via IPC — only lightweight path references + * are sent with the message (no base64 over WebSocket). */ import { useState, useRef, useEffect, useCallback } from 'react'; -import { Send, Square, X } from 'lucide-react'; +import { Send, Square, X, Paperclip, FileText, Film, Music, FileArchive, File, Loader2 } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Textarea } from '@/components/ui/textarea'; -export interface ChatAttachment { - type: 'image'; - mimeType: string; +// ── Types ──────────────────────────────────────────────────────── + +export interface FileAttachment { + id: string; fileName: string; - content: string; // base64 - preview: string; // data URL for display + mimeType: string; + fileSize: number; + stagedPath: string; // disk path for gateway + preview: string | null; // data URL for images, null for others + status: 'staging' | 'ready' | 'error'; + error?: string; } interface ChatInputProps { - onSend: (text: string, attachments?: ChatAttachment[]) => void; + onSend: (text: string, attachments?: FileAttachment[]) => void; onStop?: () => void; disabled?: boolean; sending?: boolean; } -const ACCEPTED_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp']; -const MAX_IMAGE_SIZE = 10 * 1024 * 1024; // 10MB +// ── Helpers ────────────────────────────────────────────────────── -function fileToAttachment(file: File): Promise { +function formatFileSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +function FileIcon({ mimeType, className }: { mimeType: string; className?: string }) { + if (mimeType.startsWith('video/')) return ; + if (mimeType.startsWith('audio/')) return ; + if (mimeType.startsWith('text/') || mimeType === 'application/json' || mimeType === 'application/xml') return ; + if (mimeType.includes('zip') || mimeType.includes('compressed') || mimeType.includes('archive') || mimeType.includes('tar') || mimeType.includes('rar') || mimeType.includes('7z')) return ; + if (mimeType === 'application/pdf') return ; + return ; +} + +/** + * Read a browser File object as base64 string (without the data URL prefix). + */ +function readFileAsBase64(file: globalThis.File): Promise { return new Promise((resolve, reject) => { - if (!ACCEPTED_IMAGE_TYPES.includes(file.type)) { - reject(new Error(`Unsupported image type: ${file.type}`)); - return; - } - if (file.size > MAX_IMAGE_SIZE) { - reject(new Error('Image too large (max 10MB)')); - return; - } const reader = new FileReader(); reader.onload = () => { const dataUrl = reader.result as string; - // Extract base64 content (remove "data:image/png;base64," prefix) const base64 = dataUrl.split(',')[1]; - resolve({ - type: 'image', - mimeType: file.type, - fileName: file.name, - content: base64, - preview: dataUrl, - }); + resolve(base64); }; reader.onerror = () => reject(new Error('Failed to read file')); reader.readAsDataURL(file); }); } +// ── Component ──────────────────────────────────────────────────── + export function ChatInput({ onSend, onStop, disabled = false, sending = false }: ChatInputProps) { const [input, setInput] = useState(''); - const [attachments, setAttachments] = useState([]); + const [attachments, setAttachments] = useState([]); const textareaRef = useRef(null); - const fileInputRef = useRef(null); const isComposingRef = useRef(false); // Auto-resize textarea @@ -70,28 +81,128 @@ export function ChatInput({ onSend, onStop, disabled = false, sending = false }: } }, [input]); - const addFiles = useCallback(async (files: FileList | File[]) => { - const fileArray = Array.from(files).filter((f) => ACCEPTED_IMAGE_TYPES.includes(f.type)); - if (fileArray.length === 0) return; + // ── File staging via native dialog ───────────────────────────── + const pickFiles = useCallback(async () => { try { - const newAttachments = await Promise.all(fileArray.map(fileToAttachment)); - setAttachments((prev) => [...prev, ...newAttachments]); + const result = await window.electron.ipcRenderer.invoke('dialog:open', { + properties: ['openFile', 'multiSelections'], + }) as { canceled: boolean; filePaths?: string[] }; + if (result.canceled || !result.filePaths?.length) return; + + // Add placeholder entries immediately + const tempIds: string[] = []; + for (const filePath of result.filePaths) { + const tempId = crypto.randomUUID(); + tempIds.push(tempId); + const fileName = filePath.split('/').pop() || filePath.split('\\').pop() || 'file'; + setAttachments(prev => [...prev, { + id: tempId, + fileName, + mimeType: '', + fileSize: 0, + stagedPath: '', + preview: null, + status: 'staging' as const, + }]); + } + + // Stage all files via IPC + const staged = await window.electron.ipcRenderer.invoke( + 'file:stage', + result.filePaths, + ) as Array<{ + id: string; + fileName: string; + mimeType: string; + fileSize: number; + stagedPath: string; + preview: string | null; + }>; + + // Update each placeholder with real data + setAttachments(prev => { + let updated = [...prev]; + for (let i = 0; i < tempIds.length; i++) { + const tempId = tempIds[i]; + const data = staged[i]; + if (data) { + updated = updated.map(a => + a.id === tempId + ? { ...data, status: 'ready' as const } + : a, + ); + } else { + updated = updated.map(a => + a.id === tempId + ? { ...a, status: 'error' as const, error: 'Staging failed' } + : a, + ); + } + } + return updated; + }); } catch (err) { - console.error('Failed to process image:', err); + console.error('Failed to pick files:', err); } }, []); - const removeAttachment = useCallback((index: number) => { - setAttachments((prev) => prev.filter((_, i) => i !== index)); + // ── Stage browser File objects (paste / drag-drop) ───────────── + + const stageBufferFiles = useCallback(async (files: globalThis.File[]) => { + for (const file of files) { + const tempId = crypto.randomUUID(); + setAttachments(prev => [...prev, { + id: tempId, + fileName: file.name, + mimeType: file.type || 'application/octet-stream', + fileSize: file.size, + stagedPath: '', + preview: null, + status: 'staging' as const, + }]); + + try { + const base64 = await readFileAsBase64(file); + const staged = await window.electron.ipcRenderer.invoke('file:stageBuffer', { + base64, + fileName: file.name, + mimeType: file.type || 'application/octet-stream', + }) as { + id: string; + fileName: string; + mimeType: string; + fileSize: number; + stagedPath: string; + preview: string | null; + }; + setAttachments(prev => prev.map(a => + a.id === tempId ? { ...staged, status: 'ready' as const } : a, + )); + } catch (err) { + setAttachments(prev => prev.map(a => + a.id === tempId + ? { ...a, status: 'error' as const, error: String(err) } + : a, + )); + } + } }, []); - const canSend = (input.trim() || attachments.length > 0) && !disabled && !sending; + // ── Attachment management ────────────────────────────────────── + + const removeAttachment = useCallback((id: string) => { + setAttachments(prev => prev.filter(a => a.id !== id)); + }, []); + + const allReady = attachments.length === 0 || attachments.every(a => a.status === 'ready'); + const canSend = (input.trim() || attachments.length > 0) && allReady && !disabled && !sending; const canStop = sending && !disabled && !!onStop; const handleSend = useCallback(() => { if (!canSend) return; - onSend(input.trim(), attachments.length > 0 ? attachments : undefined); + const readyAttachments = attachments.filter(a => a.status === 'ready'); + onSend(input.trim(), readyAttachments.length > 0 ? readyAttachments : undefined); setInput(''); setAttachments([]); if (textareaRef.current) { @@ -118,25 +229,25 @@ export function ChatInput({ onSend, onStop, disabled = false, sending = false }: [handleSend], ); - // Handle paste (Ctrl/Cmd+V with image) + // Handle paste (Ctrl/Cmd+V with files) const handlePaste = useCallback( (e: React.ClipboardEvent) => { const items = e.clipboardData?.items; if (!items) return; - const imageFiles: File[] = []; + const pastedFiles: globalThis.File[] = []; for (const item of Array.from(items)) { - if (item.type.startsWith('image/')) { + if (item.kind === 'file') { const file = item.getAsFile(); - if (file) imageFiles.push(file); + if (file) pastedFiles.push(file); } } - if (imageFiles.length > 0) { + if (pastedFiles.length > 0) { e.preventDefault(); - addFiles(imageFiles); + stageBufferFiles(pastedFiles); } }, - [addFiles], + [stageBufferFiles], ); // Handle drag & drop @@ -159,11 +270,11 @@ export function ChatInput({ onSend, onStop, disabled = false, sending = false }: e.preventDefault(); e.stopPropagation(); setDragOver(false); - if (e.dataTransfer?.files) { - addFiles(e.dataTransfer.files); + if (e.dataTransfer?.files?.length) { + stageBufferFiles(Array.from(e.dataTransfer.files)); } }, - [addFiles], + [stageBufferFiles], ); return ( @@ -174,26 +285,15 @@ export function ChatInput({ onSend, onStop, disabled = false, sending = false }: onDrop={handleDrop} >
- {/* Image Previews */} + {/* Attachment Previews */} {attachments.length > 0 && (
- {attachments.map((att, idx) => ( -
- {att.fileName} - -
+ {attachments.map((att) => ( + removeAttachment(att.id)} + /> ))}
)} @@ -201,19 +301,17 @@ export function ChatInput({ onSend, onStop, disabled = false, sending = false }: {/* Input Row */}
- { - if (e.target.files) { - addFiles(e.target.files); - e.target.value = ''; - } - }} - /> + {/* Attach Button */} + {/* Textarea */}
@@ -256,3 +354,63 @@ export function ChatInput({ onSend, onStop, disabled = false, sending = false }:
); } + +// ── Attachment Preview ─────────────────────────────────────────── + +function AttachmentPreview({ + attachment, + onRemove, +}: { + attachment: FileAttachment; + onRemove: () => void; +}) { + const isImage = attachment.mimeType.startsWith('image/') && attachment.preview; + + return ( +
+ {isImage ? ( + // Image thumbnail +
+ {attachment.fileName} +
+ ) : ( + // Generic file card +
+ +
+

{attachment.fileName}

+

+ {attachment.fileSize > 0 ? formatFileSize(attachment.fileSize) : '...'} +

+
+
+ )} + + {/* Staging overlay */} + {attachment.status === 'staging' && ( +
+ +
+ )} + + {/* Error overlay */} + {attachment.status === 'error' && ( +
+ Error +
+ )} + + {/* Remove button */} + +
+ ); +} diff --git a/src/pages/Chat/ChatMessage.tsx b/src/pages/Chat/ChatMessage.tsx index 2fb2b43d9..7d75ef1c9 100644 --- a/src/pages/Chat/ChatMessage.tsx +++ b/src/pages/Chat/ChatMessage.tsx @@ -4,12 +4,12 @@ * with markdown, thinking sections, images, and tool cards. */ import { useState, useCallback, memo } from 'react'; -import { User, Sparkles, Copy, Check, ChevronDown, ChevronRight, Wrench } from 'lucide-react'; +import { User, Sparkles, Copy, Check, ChevronDown, ChevronRight, Wrench, FileText, Film, Music, FileArchive, File } from 'lucide-react'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import { Button } from '@/components/ui/button'; import { cn } from '@/lib/utils'; -import type { RawMessage } from '@/stores/chat'; +import type { RawMessage, AttachedFileMeta } from '@/stores/chat'; import { extractText, extractThinking, extractImages, extractToolUse, formatTimestamp } from './message-utils'; interface ChatMessageProps { @@ -43,11 +43,13 @@ export const ChatMessage = memo(function ChatMessage({ const visibleThinking = showThinking ? thinking : null; const visibleTools = showThinking ? tools : []; + const attachedFiles = message._attachedFiles || []; + // Never render tool result messages in chat UI if (isToolResult) return null; // Don't render empty messages - if (!hasText && !visibleThinking && images.length === 0 && visibleTools.length === 0) return null; + if (!hasText && !visibleThinking && images.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0) return null; return (
)} - {/* Images */} + {/* Images (from assistant/channel content blocks) */} {images.length > 0 && (
{images.map((img, i) => ( @@ -116,6 +118,24 @@ export const ChatMessage = memo(function ChatMessage({ ))}
)} + + {/* File attachments (user-uploaded files) */} + {attachedFiles.length > 0 && ( +
+ {attachedFiles.map((file, i) => ( + file.mimeType.startsWith('image/') && file.preview ? ( + {file.fileName} + ) : ( + + ) + ))} +
+ )}
); @@ -292,6 +312,38 @@ function ThinkingBlock({ content }: { content: string }) { ); } +// ── File Card (for user-uploaded non-image files) ─────────────── + +function formatFileSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`; +} + +function FileIcon({ mimeType, className }: { mimeType: string; className?: string }) { + if (mimeType.startsWith('video/')) return ; + if (mimeType.startsWith('audio/')) return ; + if (mimeType.startsWith('text/') || mimeType === 'application/json' || mimeType === 'application/xml') return ; + if (mimeType.includes('zip') || mimeType.includes('compressed') || mimeType.includes('archive') || mimeType.includes('tar') || mimeType.includes('rar') || mimeType.includes('7z')) return ; + if (mimeType === 'application/pdf') return ; + return ; +} + +function FileCard({ file }: { file: AttachedFileMeta }) { + return ( +
+ +
+

{file.fileName}

+

+ {file.fileSize > 0 ? formatFileSize(file.fileSize) : 'File'} +

+
+
+ ); +} + // ── Tool Card ─────────────────────────────────────────────────── function ToolCard({ name, input }: { name: string; input: unknown }) { diff --git a/src/stores/chat.ts b/src/stores/chat.ts index cd7d93306..f63a66230 100644 --- a/src/stores/chat.ts +++ b/src/stores/chat.ts @@ -7,6 +7,14 @@ import { create } from 'zustand'; // ── Types ──────────────────────────────────────────────────────── +/** Metadata for locally-attached files (not from Gateway) */ +export interface AttachedFileMeta { + fileName: string; + mimeType: string; + fileSize: number; + preview: string | null; +} + /** Raw message from OpenClaw chat.history */ export interface RawMessage { role: 'user' | 'assistant' | 'system' | 'toolresult'; @@ -17,6 +25,8 @@ export interface RawMessage { toolName?: string; details?: unknown; isError?: boolean; + /** Local-only: file metadata for user-uploaded attachments (not sent to/from Gateway) */ + _attachedFiles?: AttachedFileMeta[]; } /** Content block inside a message */ @@ -79,7 +89,7 @@ interface ChatState { switchSession: (key: string) => void; newSession: () => void; loadHistory: () => Promise; - sendMessage: (text: string, attachments?: { type: string; mimeType: string; fileName: string; content: string }[]) => Promise; + sendMessage: (text: string, attachments?: Array<{ fileName: string; mimeType: string; fileSize: number; stagedPath: string; preview: string | null }>) => Promise; abortRun: () => Promise; handleChatEvent: (event: Record) => void; toggleThinking: () => void; @@ -480,18 +490,24 @@ export const useChatStore = create((set, get) => ({ // ── Send message ── - sendMessage: async (text: string, attachments?: { type: string; mimeType: string; fileName: string; content: string }[]) => { + sendMessage: async (text: string, attachments?: Array<{ fileName: string; mimeType: string; fileSize: number; stagedPath: string; preview: string | null }>) => { const trimmed = text.trim(); if (!trimmed && (!attachments || attachments.length === 0)) return; const { currentSessionKey } = get(); - // Add user message optimistically + // Add user message optimistically (with local file metadata for UI display) const userMsg: RawMessage = { role: 'user', - content: trimmed || '(image)', + content: trimmed || (attachments?.length ? '(file attached)' : ''), timestamp: Date.now() / 1000, id: crypto.randomUUID(), + _attachedFiles: attachments?.map(a => ({ + fileName: a.fileName, + mimeType: a.mimeType, + fileSize: a.fileSize, + preview: a.preview, + })), }; set((s) => ({ messages: [...s.messages, userMsg], @@ -506,29 +522,41 @@ export const useChatStore = create((set, get) => ({ try { const idempotencyKey = crypto.randomUUID(); - const rpcParams: Record = { - sessionKey: currentSessionKey, - message: trimmed || 'Describe this image.', - deliver: false, - idempotencyKey, - }; + const hasMedia = attachments && attachments.length > 0; - // Include image attachments if any - if (attachments && attachments.length > 0) { - rpcParams.attachments = attachments.map((a) => ({ - type: a.type, - mimeType: a.mimeType, - fileName: a.fileName, - content: a.content, - })); + let result: { success: boolean; result?: { runId?: string }; error?: string }; + + if (hasMedia) { + // Use dedicated chat:sendWithMedia handler — main process reads staged files + // from disk and builds base64 attachments, avoiding large IPC transfers + result = await window.electron.ipcRenderer.invoke( + 'chat:sendWithMedia', + { + sessionKey: currentSessionKey, + message: trimmed || 'Process the attached file(s).', + deliver: false, + idempotencyKey, + media: attachments.map((a) => ({ + filePath: a.stagedPath, + mimeType: a.mimeType, + fileName: a.fileName, + })), + }, + ) as { success: boolean; result?: { runId?: string }; error?: string }; + } else { + // No media — use standard lightweight RPC + result = await window.electron.ipcRenderer.invoke( + 'gateway:rpc', + 'chat.send', + { + sessionKey: currentSessionKey, + message: trimmed, + deliver: false, + idempotencyKey, + }, + ) as { success: boolean; result?: { runId?: string }; error?: string }; } - const result = await window.electron.ipcRenderer.invoke( - 'gateway:rpc', - 'chat.send', - rpcParams, - ) as { success: boolean; result?: { runId?: string }; error?: string }; - if (!result.success) { set({ error: result.error || 'Failed to send message', sending: false }); } else if (result.result?.runId) {