From d108a850efb7789224a9ee493f950b1ebec19ace Mon Sep 17 00:00:00 2001 From: Haze <709547807@qq.com> Date: Sat, 14 Feb 2026 18:00:57 +0800 Subject: [PATCH] feat(chat): opt pic show in chat history (#87) --- electron/main/index.ts | 10 +++ electron/main/ipc-handlers.ts | 15 ++-- src/pages/Chat/ChatMessage.tsx | 137 +++++++++++++++++++++++--------- src/pages/Chat/message-utils.ts | 50 +++++++++--- src/stores/chat.ts | 114 +++++++++++++++++++++----- src/stores/gateway.ts | 36 +++++++-- 6 files changed, 283 insertions(+), 79 deletions(-) diff --git a/electron/main/index.ts b/electron/main/index.ts index 5faa56f80..51af62855 100644 --- a/electron/main/index.ts +++ b/electron/main/index.ts @@ -120,6 +120,16 @@ async function initialize(): Promise { // Create system tray createTray(mainWindow); + // Inject OpenRouter site headers (HTTP-Referer & X-Title) for rankings on openrouter.ai + session.defaultSession.webRequest.onBeforeSendHeaders( + { urls: ['https://openrouter.ai/*'] }, + (details, callback) => { + details.requestHeaders['HTTP-Referer'] = 'https://claw-x.com'; + details.requestHeaders['X-Title'] = 'ClawX'; + callback({ requestHeaders: details.requestHeaders }); + }, + ); + // Override security headers ONLY for the OpenClaw Gateway Control UI session.defaultSession.webRequest.onHeadersReceived((details, callback) => { const isGatewayUrl = details.url.includes('127.0.0.1:18789') || details.url.includes('localhost:18789'); diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 2ba37f406..1407007e2 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -1505,17 +1505,22 @@ function mimeToExt(mimeType: string): string { const OUTBOUND_DIR = join(homedir(), '.openclaw', 'media', 'outbound'); /** - * Generate a small preview data URL for image files. - * Uses Electron nativeImage to resize large images for thumbnails. + * Generate a preview data URL for image files. + * Resizes large images while preserving aspect ratio (only constrain the + * longer side so the image is never squished). The frontend handles + * square cropping via CSS object-fit: cover. */ function generateImagePreview(filePath: string, mimeType: string): string | null { try { const img = nativeImage.createFromPath(filePath); if (img.isEmpty()) return null; const size = img.getSize(); - // If image is large, resize for thumbnail - if (size.width > 256 || size.height > 256) { - const resized = img.resize({ width: 256, height: 256 }); + const maxDim = 512; // keep enough resolution for crisp display on Retina + // Only resize if larger than threshold — specify ONE dimension to keep ratio + if (size.width > maxDim || size.height > maxDim) { + const resized = size.width >= size.height + ? img.resize({ width: maxDim }) // landscape / square → constrain width + : img.resize({ height: maxDim }); // portrait → constrain height return `data:image/png;base64,${resized.toPNG().toString('base64')}`; } // Small image — use original diff --git a/src/pages/Chat/ChatMessage.tsx b/src/pages/Chat/ChatMessage.tsx index e9acf50cd..326e880c7 100644 --- a/src/pages/Chat/ChatMessage.tsx +++ b/src/pages/Chat/ChatMessage.tsx @@ -48,8 +48,9 @@ export const ChatMessage = memo(function ChatMessage({ // Never render tool result messages in chat UI if (isToolResult) return null; - // Don't render empty messages - if (!hasText && !visibleThinking && images.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0) return null; + // Don't render empty messages (also keep messages with streaming tool status) + const hasStreamingToolStatus = showThinking && isStreaming && streamingTools.length > 0; + if (!hasText && !visibleThinking && images.length === 0 && visibleTools.length === 0 && attachedFiles.length === 0 && !hasStreamingToolStatus) return null; return (
)} + {/* Images — rendered ABOVE text bubble for user messages */} + {/* Images from content blocks (Gateway session data) */} + {isUser && images.length > 0 && ( +
+ {images.map((img, i) => ( +
+ attachment +
+ ))} +
+ )} + + {/* File attachments — images above text for user, file cards below */} + {isUser && attachedFiles.length > 0 && ( +
+ {attachedFiles.map((file, i) => { + const isImage = file.mimeType.startsWith('image/'); + // Skip image attachments if we already have images from content blocks + if (isImage && images.length > 0) return null; + // Image files → always render as square crop (with preview or placeholder) + if (isImage) { + return ( +
+ {file.preview ? ( + {file.fileName} + ) : ( +
+ +
+ )} +
+ ); + } + // Non-image files → file card + return ; + })} +
+ )} + {/* Main text bubble */} {hasText && ( )} - {/* Images from content blocks (Gateway session data — persists across history reloads) */} - {images.length > 0 && ( + {/* Images from content blocks — assistant messages (below text) */} + {!isUser && images.length > 0 && (
{images.map((img, i) => ( attachment ))}
)} - {/* File attachments (local preview — shown before history reload) */} - {/* Only show _attachedFiles images if no content-block images (avoid duplicates) */} - {attachedFiles.length > 0 && ( + {/* File attachments — assistant messages (below text) */} + {!isUser && attachedFiles.length > 0 && (
{attachedFiles.map((file, i) => { - // Skip image attachments if we already have images from content blocks - if (file.mimeType.startsWith('image/') && file.preview && images.length > 0) return null; - return file.mimeType.startsWith('image/') && file.preview ? ( - {file.fileName} - ) : ( - - ); + const isImage = file.mimeType.startsWith('image/'); + if (isImage && images.length > 0) return null; + if (isImage && file.preview) { + return ( + {file.fileName} + ); + } + if (isImage && !file.preview) { + return ( +
+ +
+ ); + } + return ; })}
)} + + {/* Hover timestamp for user messages (shown below content on hover) */} + {isUser && message.timestamp && ( + + {formatTimestamp(message.timestamp)} + + )}
); @@ -265,20 +330,14 @@ function MessageBubble({ )} - {/* Footer: timestamp + copy */} -
- {timestamp && ( - - {formatTimestamp(timestamp)} - - )} - {!isUser && ( + {/* Footer: copy button (assistant only; user timestamp is rendered outside the bubble) */} + {!isUser && ( +
+ {timestamp ? ( + + {formatTimestamp(timestamp)} + + ) : } - )} -
+
+ )} ); } diff --git a/src/pages/Chat/message-utils.ts b/src/pages/Chat/message-utils.ts index 529f00c4c..7f6406955 100644 --- a/src/pages/Chat/message-utils.ts +++ b/src/pages/Chat/message-utils.ts @@ -140,22 +140,50 @@ export function extractImages(message: RawMessage | unknown): Array<{ mimeType: /** * Extract tool use blocks from a message. + * Handles both Anthropic format (tool_use in content array) and + * OpenAI format (tool_calls array on the message object). */ export function extractToolUse(message: RawMessage | unknown): Array<{ id: string; name: string; input: unknown }> { if (!message || typeof message !== 'object') return []; const msg = message as Record; - const content = msg.content; - - if (!Array.isArray(content)) return []; - const tools: Array<{ id: string; name: string; input: unknown }> = []; - for (const block of content as ContentBlock[]) { - if ((block.type === 'tool_use' || block.type === 'toolCall') && block.name) { - tools.push({ - id: block.id || '', - name: block.name, - input: block.input ?? block.arguments, - }); + + // Path 1: Anthropic/normalized format — tool_use / toolCall blocks inside content array + const content = msg.content; + if (Array.isArray(content)) { + for (const block of content as ContentBlock[]) { + if ((block.type === 'tool_use' || block.type === 'toolCall') && block.name) { + tools.push({ + id: block.id || '', + name: block.name, + input: block.input ?? block.arguments, + }); + } + } + } + + // Path 2: OpenAI format — tool_calls array on the message itself + // Real-time streaming events from OpenAI-compatible models (DeepSeek, etc.) + // use this format; the Gateway normalizes to Path 1 when storing history. + if (tools.length === 0) { + const toolCalls = msg.tool_calls ?? msg.toolCalls; + if (Array.isArray(toolCalls)) { + for (const tc of toolCalls as Array>) { + const fn = (tc.function ?? tc) as Record; + const name = typeof fn.name === 'string' ? fn.name : ''; + if (!name) continue; + let input: unknown; + try { + input = typeof fn.arguments === 'string' ? JSON.parse(fn.arguments) : fn.arguments ?? fn.input; + } catch { + input = fn.arguments; + } + tools.push({ + id: typeof tc.id === 'string' ? tc.id : '', + name, + input, + }); + } } } diff --git a/src/stores/chat.ts b/src/stores/chat.ts index 48abfc3b5..b1781844a 100644 --- a/src/stores/chat.ts +++ b/src/stores/chat.ts @@ -241,10 +241,25 @@ function isToolOnlyMessage(message: RawMessage | undefined): boolean { if (!message) return false; if (isToolResultRole(message.role)) return true; + const msg = message as unknown as Record; const content = message.content; - if (!Array.isArray(content)) return false; - let hasTool = false; + // Check OpenAI-format tool_calls field (real-time streaming from OpenAI-compatible models) + const toolCalls = msg.tool_calls ?? msg.toolCalls; + const hasOpenAITools = Array.isArray(toolCalls) && toolCalls.length > 0; + + if (!Array.isArray(content)) { + // Content is not an array — check if there's OpenAI-format tool_calls + if (hasOpenAITools) { + // Has tool calls but content might be empty/string — treat as tool-only + // if there's no meaningful text content + const textContent = typeof content === 'string' ? content.trim() : ''; + return textContent.length === 0; + } + return false; + } + + let hasTool = hasOpenAITools; let hasText = false; let hasNonToolContent = false; @@ -312,19 +327,41 @@ function parseDurationMs(value: unknown): number | undefined { function extractToolUseUpdates(message: unknown): ToolStatus[] { if (!message || typeof message !== 'object') return []; const msg = message as Record; - const content = msg.content; - if (!Array.isArray(content)) return []; - const updates: ToolStatus[] = []; - for (const block of content as ContentBlock[]) { - if ((block.type !== 'tool_use' && block.type !== 'toolCall') || !block.name) continue; - updates.push({ - id: block.id || block.name, - toolCallId: block.id, - name: block.name, - status: 'running', - updatedAt: Date.now(), - }); + + // Path 1: Anthropic/normalized format — tool blocks inside content array + const content = msg.content; + if (Array.isArray(content)) { + for (const block of content as ContentBlock[]) { + if ((block.type !== 'tool_use' && block.type !== 'toolCall') || !block.name) continue; + updates.push({ + id: block.id || block.name, + toolCallId: block.id, + name: block.name, + status: 'running', + updatedAt: Date.now(), + }); + } + } + + // Path 2: OpenAI format — tool_calls array on the message itself + if (updates.length === 0) { + const toolCalls = msg.tool_calls ?? msg.toolCalls; + if (Array.isArray(toolCalls)) { + for (const tc of toolCalls as Array>) { + const fn = (tc.function ?? tc) as Record; + const name = typeof fn.name === 'string' ? fn.name : ''; + if (!name) continue; + const id = typeof tc.id === 'string' ? tc.id : name; + updates.push({ + id, + toolCallId: typeof tc.id === 'string' ? tc.id : undefined, + name, + status: 'running', + updatedAt: Date.now(), + }); + } + } } return updates; @@ -602,8 +639,16 @@ export const useChatStore = create((set, get) => ({ // Async: load missing image previews from disk (updates in background) loadMissingPreviews(enrichedMessages).then((updated) => { if (updated) { - // Trigger re-render with updated previews - set({ messages: [...enrichedMessages] }); + // Create new object references so React.memo detects changes. + // loadMissingPreviews mutates AttachedFileMeta in place, so we + // must produce fresh message + file references for each affected msg. + set({ + messages: enrichedMessages.map(msg => + msg._attachedFiles + ? { ...msg, _attachedFiles: msg._attachedFiles.map(f => ({ ...f })) } + : msg + ), + }); } }); const { pendingFinal, lastUserMessageAt } = get(); @@ -757,10 +802,26 @@ export const useChatStore = create((set, get) => ({ // Only process events for the active run (or if no active run set) if (activeRunId && runId && runId !== activeRunId) return; - switch (eventState) { + // Defensive: if state is missing but we have a message, try to infer state. + // This handles the case where the Gateway sends events without a state wrapper + // (e.g., protocol events where payload is the raw message). + let resolvedState = eventState; + if (!resolvedState && event.message && typeof event.message === 'object') { + const msg = event.message as Record; + const stopReason = msg.stopReason ?? msg.stop_reason; + if (stopReason) { + // Message has a stopReason → it's a final message + resolvedState = 'final'; + } else if (msg.role || msg.content) { + // Message has role/content but no stopReason → treat as delta (streaming) + resolvedState = 'delta'; + } + } + + switch (resolvedState) { case 'delta': { // Streaming update - store the cumulative message - const updates = collectToolUpdates(event.message, eventState); + const updates = collectToolUpdates(event.message, resolvedState); set((s) => ({ streamingMessage: (() => { if (event.message && typeof event.message === 'object') { @@ -777,7 +838,7 @@ export const useChatStore = create((set, get) => ({ // Message complete - add to history and clear streaming const finalMsg = event.message as RawMessage | undefined; if (finalMsg) { - const updates = collectToolUpdates(finalMsg, eventState); + const updates = collectToolUpdates(finalMsg, resolvedState); if (isToolResultRole(finalMsg.role)) { set((s) => ({ streamingText: '', @@ -867,6 +928,21 @@ export const useChatStore = create((set, get) => ({ }); break; } + default: { + // Unknown or empty state — if we're currently sending and receive an event + // with a message, attempt to process it as streaming data. This handles + // edge cases where the Gateway sends events without a state field. + const { sending } = get(); + if (sending && event.message && typeof event.message === 'object') { + console.warn(`[handleChatEvent] Unknown event state "${resolvedState}", treating message as streaming delta. Event keys:`, Object.keys(event)); + const updates = collectToolUpdates(event.message, 'delta'); + set((s) => ({ + streamingMessage: event.message ?? s.streamingMessage, + streamingTools: updates.length > 0 ? upsertToolStatuses(s.streamingTools, updates) : s.streamingTools, + })); + } + break; + } } }, diff --git a/src/stores/gateway.ts b/src/stores/gateway.ts index db8c4a4b0..844b75eee 100644 --- a/src/stores/gateway.ts +++ b/src/stores/gateway.ts @@ -64,6 +64,8 @@ export const useGatewayStore = create((set, get) => ({ // Some Gateway builds stream chat events via generic "agent" notifications. // Normalize and forward them to the chat store. + // The Gateway may put event fields (state, message, etc.) either inside + // params.data or directly on params — we must handle both layouts. window.electron.ipcRenderer.on('gateway:notification', (notification) => { const payload = notification as { method?: string; params?: Record } | undefined; if (!payload || payload.method !== 'agent' || !payload.params || typeof payload.params !== 'object') { @@ -73,11 +75,16 @@ export const useGatewayStore = create((set, get) => ({ const p = payload.params; const data = (p.data && typeof p.data === 'object') ? (p.data as Record) : {}; const normalizedEvent: Record = { + // Spread data sub-object first (nested layout) ...data, + // Then override with top-level params fields (flat layout takes precedence) runId: p.runId ?? data.runId, sessionKey: p.sessionKey ?? data.sessionKey, stream: p.stream ?? data.stream, seq: p.seq ?? data.seq, + // Critical: also pick up state and message from params (flat layout) + state: p.state ?? data.state, + message: p.message ?? data.message, }; import('./chat') @@ -89,16 +96,35 @@ export const useGatewayStore = create((set, get) => ({ }); }); - // Listen for chat events from the gateway and forward to chat store + // Listen for chat events from the gateway and forward to chat store. + // The data arrives as { message: payload } from handleProtocolEvent. + // The payload may be a full event wrapper ({ state, runId, message }) + // or the raw chat message itself. We need to handle both. window.electron.ipcRenderer.on('gateway:chat-message', (data) => { try { // Dynamic import to avoid circular dependency import('./chat').then(({ useChatStore }) => { - const chatData = data as { message?: Record } | Record; - const event = ('message' in chatData && typeof chatData.message === 'object') + const chatData = data as Record; + // Unwrap the { message: payload } wrapper from handleProtocolEvent + const payload = ('message' in chatData && typeof chatData.message === 'object') ? chatData.message as Record - : chatData as Record; - useChatStore.getState().handleChatEvent(event); + : chatData; + + // If payload has a 'state' field, it's already a proper event wrapper + if (payload.state) { + useChatStore.getState().handleChatEvent(payload); + return; + } + + // Otherwise, payload is the raw message — wrap it as a 'final' event + // so handleChatEvent can process it (this happens when the Gateway + // sends protocol events with the message directly as payload). + const syntheticEvent: Record = { + state: 'final', + message: payload, + runId: chatData.runId ?? payload.runId, + }; + useChatStore.getState().handleChatEvent(syntheticEvent); }); } catch (err) { console.warn('Failed to forward chat event:', err);