admin/DeskClaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(chat): improve media handling and caching for user messages (#81)

Browse Source
This commit is contained in:
Haze
2026-02-14 00:21:04 +08:00
committed by GitHub
Unverified
parent 051803869d
commit cf8091d81f
7 changed files with 280 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/pages/Chat/ChatMessage.tsx
View File

@@ -105,35 +105,41 @@ export const ChatMessage = memo(function ChatMessage({
/>
)}
{/* Images (from assistant/channel content blocks) */}
{/* Images from content blocks (Gateway session data — persists across history reloads) */}
{images.length > 0 && (
<div className="flex flex-wrap gap-2">
{images.map((img, i) => (
<img
key={i}
key={`content-${i}`}
src={`data:${img.mimeType};base64,${img.data}`}
alt="attachment"
className="max-w-xs rounded-lg border"
className={cn(
'rounded-lg border',
isUser ? 'max-w-[200px] max-h-48' : 'max-w-xs',
)}
/>
))}
</div>
)}
{/* File attachments (user-uploaded files) */}
{/* File attachments (local preview — shown before history reload) */}
{/* Only show _attachedFiles images if no content-block images (avoid duplicates) */}
{attachedFiles.length > 0 && (
<div className="flex flex-wrap gap-2">
{attachedFiles.map((file, i) => (
file.mimeType.startsWith('image/') && file.preview ? (
{attachedFiles.map((file, i) => {
// Skip image attachments if we already have images from content blocks
if (file.mimeType.startsWith('image/') && file.preview && images.length > 0) return null;
return file.mimeType.startsWith('image/') && file.preview ? (
<img
key={i}
key={`local-${i}`}
src={file.preview}
alt={file.fileName}
className="max-w-xs max-h-48 rounded-lg border"
className="max-w-[200px] max-h-48 rounded-lg border"
/>
) : (
<FileCard key={i} file={file} />
)
))}
<FileCard key={`local-${i}`} file={file} />
);
})}
</div>
)}
</div>

68
src/pages/Chat/message-utils.ts
View File

@@ -5,20 +5,38 @@
*/
import type { RawMessage, ContentBlock } from '@/stores/chat';
/**
* Clean Gateway metadata from user message text for display.
* Strips: [media attached: ... | ...], [message_id: ...],
* and the timestamp prefix [Day Date Time Timezone].
*/
function cleanUserText(text: string): string {
return text
// Remove [media attached: path (mime) | path] references
.replace(/\s*\[media attached:[^\]]*\]/g, '')
// Remove [message_id: uuid]
.replace(/\s*\[message_id:\s*[^\]]+\]/g, '')
// Remove Gateway timestamp prefix like [Fri 2026-02-13 22:39 GMT+8]
.replace(/^\[(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+[^\]]+\]\s*/i, '')
.trim();
}
/**
* Extract displayable text from a message's content field.
* Handles both string content and array-of-blocks content.
* For user messages, strips Gateway-injected metadata.
*/
export function extractText(message: RawMessage | unknown): string {
if (!message || typeof message !== 'object') return '';
const msg = message as Record<string, unknown>;
const content = msg.content;
const isUser = msg.role === 'user';
let result = '';
if (typeof content === 'string') {
return content.trim().length > 0 ? content : '';
}
if (Array.isArray(content)) {
result = content.trim().length > 0 ? content : '';
} else if (Array.isArray(content)) {
const parts: string[] = [];
for (const block of content as ContentBlock[]) {
if (block.type === 'text' && block.text) {
@@ -28,15 +46,18 @@ export function extractText(message: RawMessage | unknown): string {
}
}
const combined = parts.join('\n\n');
return combined.trim().length > 0 ? combined : '';
result = combined.trim().length > 0 ? combined : '';
} else if (typeof msg.text === 'string') {
// Fallback: try .text field
result = msg.text.trim().length > 0 ? msg.text : '';
}
// Fallback: try .text field
if (typeof msg.text === 'string') {
return msg.text.trim().length > 0 ? msg.text : '';
// Strip Gateway metadata from user messages for clean display
if (isUser && result) {
result = cleanUserText(result);
}
return '';
return result;
}
/**
@@ -64,6 +85,35 @@ export function extractThinking(message: RawMessage | unknown): string | null {
return combined.length > 0 ? combined : null;
}
/**
* Extract media file references from Gateway-formatted user message text.
* Returns array of { filePath, mimeType } from [media attached: path (mime) | path] patterns.
*/
export function extractMediaRefs(message: RawMessage | unknown): Array<{ filePath: string; mimeType: string }> {
if (!message || typeof message !== 'object') return [];
const msg = message as Record<string, unknown>;
if (msg.role !== 'user') return [];
const content = msg.content;
let text = '';
if (typeof content === 'string') {
text = content;
} else if (Array.isArray(content)) {
text = (content as ContentBlock[])
.filter(b => b.type === 'text' && b.text)
.map(b => b.text!)
.join('\n');
}
const refs: Array<{ filePath: string; mimeType: string }> = [];
const regex = /\[media attached:\s*([^\s(]+)\s*\(([^)]+)\)\s*\|[^\]]*\]/g;
let match;
while ((match = regex.exec(text)) !== null) {
refs.push({ filePath: match[1], mimeType: match[2] });
}
return refs;
}
/**
* Extract image attachments from a message.
* Returns array of { mimeType, data } for base64 images.

2
src/pages/Setup/index.tsx
View File

@@ -519,7 +519,7 @@ function RuntimeContent({ onStatusChange }: RuntimeContentProps) {
}
return prev;
});
}, 120 * 1000); // 120 seconds — enough for gateway to fully initialize
}, 600 * 1000); // 600 seconds — enough for gateway to fully initialize
return () => {
if (gatewayTimeoutRef.current) {

156
src/stores/chat.ts
View File

@@ -100,6 +100,135 @@ interface ChatState {
const DEFAULT_CANONICAL_PREFIX = 'agent:main';
const DEFAULT_SESSION_KEY = `${DEFAULT_CANONICAL_PREFIX}:main`;
// ── Local image cache ─────────────────────────────────────────
// The Gateway doesn't store image attachments in session content blocks,
// so we cache them locally keyed by staged file path (which appears in the
// [media attached: <path> ...] reference in the Gateway's user message text).
// Keying by path avoids the race condition of keying by runId (which is only
// available after the RPC returns, but history may load before that).
const IMAGE_CACHE_KEY = 'clawx:image-cache';
const IMAGE_CACHE_MAX = 100; // max entries to prevent unbounded growth
function loadImageCache(): Map<string, AttachedFileMeta> {
try {
const raw = localStorage.getItem(IMAGE_CACHE_KEY);
if (raw) {
const entries = JSON.parse(raw) as Array<[string, AttachedFileMeta]>;
return new Map(entries);
}
} catch { /* ignore parse errors */ }
return new Map();
}
function saveImageCache(cache: Map<string, AttachedFileMeta>): void {
try {
// Evict oldest entries if over limit
const entries = Array.from(cache.entries());
const trimmed = entries.length > IMAGE_CACHE_MAX
? entries.slice(entries.length - IMAGE_CACHE_MAX)
: entries;
localStorage.setItem(IMAGE_CACHE_KEY, JSON.stringify(trimmed));
} catch { /* ignore quota errors */ }
}
const _imageCache = loadImageCache();
/** Extract plain text from message content (string or content blocks) */
function getMessageText(content: unknown): string {
if (typeof content === 'string') return content;
if (Array.isArray(content)) {
return (content as Array<{ type?: string; text?: string }>)
.filter(b => b.type === 'text' && b.text)
.map(b => b.text!)
.join('\n');
}
return '';
}
/** Extract media file refs from [media attached: <path> (<mime>) | ...] patterns */
function extractMediaRefs(text: string): Array<{ filePath: string; mimeType: string }> {
const refs: Array<{ filePath: string; mimeType: string }> = [];
const regex = /\[media attached:\s*([^\s(]+)\s*\(([^)]+)\)\s*\|[^\]]*\]/g;
let match;
while ((match = regex.exec(text)) !== null) {
refs.push({ filePath: match[1], mimeType: match[2] });
}
return refs;
}
/**
* Restore _attachedFiles for user messages loaded from history.
* Uses local cache for previews when available, but ALWAYS creates entries
* from [media attached: ...] text patterns so file cards show even without cache.
*/
function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
return messages.map(msg => {
if (msg.role !== 'user' || msg._attachedFiles) return msg;
const text = getMessageText(msg.content);
const refs = extractMediaRefs(text);
if (refs.length === 0) return msg;
const files: AttachedFileMeta[] = refs.map(ref => {
const cached = _imageCache.get(ref.filePath);
if (cached) return cached;
// Fallback: create entry from text pattern (preview loaded later via IPC)
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null };
});
return { ...msg, _attachedFiles: files };
});
}
/**
* Async: load missing previews from disk via IPC for messages that have
* _attachedFiles with null previews. Updates messages in-place and triggers re-render.
*/
async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
// Collect all image paths that need previews
const needPreview: Array<{ filePath: string; mimeType: string }> = [];
for (const msg of messages) {
if (msg.role !== 'user' || !msg._attachedFiles) continue;
const text = getMessageText(msg.content);
const refs = extractMediaRefs(text);
for (let i = 0; i < refs.length; i++) {
const file = msg._attachedFiles[i];
if (file && file.mimeType.startsWith('image/') && !file.preview) {
needPreview.push(refs[i]);
}
}
}
if (needPreview.length === 0) return false;
try {
const thumbnails = await window.electron.ipcRenderer.invoke(
'media:getThumbnails',
needPreview,
) as Record<string, { preview: string | null; fileSize: number }>;
let updated = false;
for (const msg of messages) {
if (msg.role !== 'user' || !msg._attachedFiles) continue;
const text = getMessageText(msg.content);
const refs = extractMediaRefs(text);
for (let i = 0; i < refs.length; i++) {
const file = msg._attachedFiles[i];
const thumb = thumbnails[refs[i]?.filePath];
if (file && thumb && (thumb.preview || thumb.fileSize)) {
if (thumb.preview) file.preview = thumb.preview;
if (thumb.fileSize) file.fileSize = thumb.fileSize;
// Update cache for future loads
_imageCache.set(refs[i].filePath, { ...file });
updated = true;
}
}
}
if (updated) saveImageCache(_imageCache);
return updated;
} catch (err) {
console.warn('[loadMissingPreviews] Failed:', err);
return false;
}
}
function getCanonicalPrefixFromSessions(sessions: ChatSession[]): string | null {
const canonical = sessions.find((s) => s.key.startsWith('agent:'))?.key;
if (!canonical) return null;
@@ -465,8 +594,18 @@ export const useChatStore = create<ChatState>((set, get) => ({
const data = result.result;
const rawMessages = Array.isArray(data.messages) ? data.messages as RawMessage[] : [];
const filteredMessages = rawMessages.filter((msg) => !isToolResultRole(msg.role));
// Restore file attachments for user messages (from cache + text patterns)
const enrichedMessages = enrichWithCachedImages(filteredMessages);
const thinkingLevel = data.thinkingLevel ? String(data.thinkingLevel) : null;
set({ messages: filteredMessages, thinkingLevel, loading: false });
set({ messages: enrichedMessages, thinkingLevel, loading: false });
// Async: load missing image previews from disk (updates in background)
loadMissingPreviews(enrichedMessages).then((updated) => {
if (updated) {
// Trigger re-render with updated previews
set({ messages: [...enrichedMessages] });
}
});
const { pendingFinal, lastUserMessageAt } = get();
if (pendingFinal) {
const recentAssistant = [...filteredMessages].reverse().find((msg) => {
@@ -528,6 +667,21 @@ export const useChatStore = create<ChatState>((set, get) => ({
console.log('[sendMessage] Media paths:', attachments!.map(a => a.stagedPath));
}
// Cache image attachments BEFORE the IPC call to avoid race condition:
// history may reload (via Gateway event) before the RPC returns.
// Keyed by staged file path which appears in [media attached: <path> ...].
if (hasMedia && attachments) {
for (const a of attachments) {
_imageCache.set(a.stagedPath, {
fileName: a.fileName,
mimeType: a.mimeType,
fileSize: a.fileSize,
preview: a.preview,
});
}
saveImageCache(_imageCache);
}
let result: { success: boolean; result?: { runId?: string }; error?: string };
if (hasMedia) {