fix(chat): optimize chat with pic response (#103)
This commit is contained in:
@@ -1589,6 +1589,40 @@ function registerFileHandlers(): void {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Load thumbnails for file paths on disk (used to restore previews in history)
|
// Load thumbnails for file paths on disk (used to restore previews in history)
|
||||||
|
// Save an image to a user-chosen location (base64 data URI or existing file path)
|
||||||
|
ipcMain.handle('media:saveImage', async (_, params: {
|
||||||
|
base64?: string;
|
||||||
|
mimeType?: string;
|
||||||
|
filePath?: string;
|
||||||
|
defaultFileName: string;
|
||||||
|
}) => {
|
||||||
|
try {
|
||||||
|
const ext = params.defaultFileName.includes('.')
|
||||||
|
? params.defaultFileName.split('.').pop()!
|
||||||
|
: (params.mimeType?.split('/')[1] || 'png');
|
||||||
|
const result = await dialog.showSaveDialog({
|
||||||
|
defaultPath: join(homedir(), 'Downloads', params.defaultFileName),
|
||||||
|
filters: [
|
||||||
|
{ name: 'Images', extensions: [ext, 'png', 'jpg', 'jpeg', 'webp', 'gif'] },
|
||||||
|
{ name: 'All Files', extensions: ['*'] },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
if (result.canceled || !result.filePath) return { success: false };
|
||||||
|
|
||||||
|
if (params.filePath && existsSync(params.filePath)) {
|
||||||
|
copyFileSync(params.filePath, result.filePath);
|
||||||
|
} else if (params.base64) {
|
||||||
|
const buffer = Buffer.from(params.base64, 'base64');
|
||||||
|
writeFileSync(result.filePath, buffer);
|
||||||
|
} else {
|
||||||
|
return { success: false, error: 'No image data provided' };
|
||||||
|
}
|
||||||
|
return { success: true, savedPath: result.filePath };
|
||||||
|
} catch (err) {
|
||||||
|
return { success: false, error: String(err) };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
ipcMain.handle('media:getThumbnails', async (_, paths: Array<{ filePath: string; mimeType: string }>) => {
|
ipcMain.handle('media:getThumbnails', async (_, paths: Array<{ filePath: string; mimeType: string }>) => {
|
||||||
const results: Record<string, { preview: string | null; fileSize: number }> = {};
|
const results: Record<string, { preview: string | null; fileSize: number }> = {};
|
||||||
for (const { filePath, mimeType } of paths) {
|
for (const { filePath, mimeType } of paths) {
|
||||||
|
|||||||
@@ -119,6 +119,7 @@ const electronAPI = {
|
|||||||
'file:stage',
|
'file:stage',
|
||||||
'file:stageBuffer',
|
'file:stageBuffer',
|
||||||
'media:getThumbnails',
|
'media:getThumbnails',
|
||||||
|
'media:saveImage',
|
||||||
// Chat send with media (reads staged files in main process)
|
// Chat send with media (reads staged files in main process)
|
||||||
'chat:sendWithMedia',
|
'chat:sendWithMedia',
|
||||||
// OpenClaw extras
|
// OpenClaw extras
|
||||||
|
|||||||
@@ -3,10 +3,11 @@
|
|||||||
* Renders user / assistant / system / toolresult messages
|
* Renders user / assistant / system / toolresult messages
|
||||||
* with markdown, thinking sections, images, and tool cards.
|
* with markdown, thinking sections, images, and tool cards.
|
||||||
*/
|
*/
|
||||||
import { useState, useCallback, memo } from 'react';
|
import { useState, useCallback, useEffect, memo } from 'react';
|
||||||
import { User, Sparkles, Copy, Check, ChevronDown, ChevronRight, Wrench, FileText, Film, Music, FileArchive, File } from 'lucide-react';
|
import { User, Sparkles, Copy, Check, ChevronDown, ChevronRight, Wrench, FileText, Film, Music, FileArchive, File, X, FolderOpen, ZoomIn } from 'lucide-react';
|
||||||
import ReactMarkdown from 'react-markdown';
|
import ReactMarkdown from 'react-markdown';
|
||||||
import remarkGfm from 'remark-gfm';
|
import remarkGfm from 'remark-gfm';
|
||||||
|
import { createPortal } from 'react-dom';
|
||||||
import { Button } from '@/components/ui/button';
|
import { Button } from '@/components/ui/button';
|
||||||
import { cn } from '@/lib/utils';
|
import { cn } from '@/lib/utils';
|
||||||
import type { RawMessage, AttachedFileMeta } from '@/stores/chat';
|
import type { RawMessage, AttachedFileMeta } from '@/stores/chat';
|
||||||
@@ -26,6 +27,15 @@ interface ChatMessageProps {
|
|||||||
}>;
|
}>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface ExtractedImage { url?: string; data?: string; mimeType: string; }
|
||||||
|
|
||||||
|
/** Resolve an ExtractedImage to a displayable src string, or null if not possible. */
|
||||||
|
function imageSrc(img: ExtractedImage): string | null {
|
||||||
|
if (img.url) return img.url;
|
||||||
|
if (img.data) return `data:${img.mimeType};base64,${img.data}`;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
export const ChatMessage = memo(function ChatMessage({
|
export const ChatMessage = memo(function ChatMessage({
|
||||||
message,
|
message,
|
||||||
showThinking,
|
showThinking,
|
||||||
@@ -44,6 +54,7 @@ export const ChatMessage = memo(function ChatMessage({
|
|||||||
const visibleTools = showThinking ? tools : [];
|
const visibleTools = showThinking ? tools : [];
|
||||||
|
|
||||||
const attachedFiles = message._attachedFiles || [];
|
const attachedFiles = message._attachedFiles || [];
|
||||||
|
const [lightboxImg, setLightboxImg] = useState<{ src: string; fileName: string; filePath?: string; base64?: string; mimeType?: string } | null>(null);
|
||||||
|
|
||||||
// Never render tool result messages in chat UI
|
// Never render tool result messages in chat UI
|
||||||
if (isToolResult) return null;
|
if (isToolResult) return null;
|
||||||
@@ -97,21 +108,23 @@ export const ChatMessage = memo(function ChatMessage({
|
|||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Images — rendered ABOVE text bubble for user messages */}
|
{/* Images — rendered ABOVE text bubble for user messages */}
|
||||||
{/* Images from content blocks (Gateway session data) */}
|
{/* Images from content blocks (Gateway session data / channel push photos) */}
|
||||||
{isUser && images.length > 0 && (
|
{isUser && images.length > 0 && (
|
||||||
<div className="flex flex-wrap gap-2">
|
<div className="flex flex-wrap gap-2">
|
||||||
{images.map((img, i) => (
|
{images.map((img, i) => {
|
||||||
<div
|
const src = imageSrc(img);
|
||||||
|
if (!src) return null;
|
||||||
|
return (
|
||||||
|
<ImageThumbnail
|
||||||
key={`content-${i}`}
|
key={`content-${i}`}
|
||||||
className="w-36 h-36 rounded-xl border overflow-hidden"
|
src={src}
|
||||||
>
|
fileName="image"
|
||||||
<img
|
base64={img.data}
|
||||||
src={`data:${img.mimeType};base64,${img.data}`}
|
mimeType={img.mimeType}
|
||||||
alt="attachment"
|
onPreview={() => setLightboxImg({ src, fileName: 'image', base64: img.data, mimeType: img.mimeType })}
|
||||||
className="w-full h-full object-cover"
|
|
||||||
/>
|
/>
|
||||||
</div>
|
);
|
||||||
))}
|
})}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@@ -122,25 +135,23 @@ export const ChatMessage = memo(function ChatMessage({
|
|||||||
const isImage = file.mimeType.startsWith('image/');
|
const isImage = file.mimeType.startsWith('image/');
|
||||||
// Skip image attachments if we already have images from content blocks
|
// Skip image attachments if we already have images from content blocks
|
||||||
if (isImage && images.length > 0) return null;
|
if (isImage && images.length > 0) return null;
|
||||||
// Image files → always render as square crop (with preview or placeholder)
|
|
||||||
if (isImage) {
|
if (isImage) {
|
||||||
return (
|
return file.preview ? (
|
||||||
<div
|
<ImageThumbnail
|
||||||
key={`local-${i}`}
|
key={`local-${i}`}
|
||||||
className="w-36 h-36 rounded-xl border overflow-hidden bg-muted"
|
|
||||||
>
|
|
||||||
{file.preview ? (
|
|
||||||
<img
|
|
||||||
src={file.preview}
|
src={file.preview}
|
||||||
alt={file.fileName}
|
fileName={file.fileName}
|
||||||
className="w-full h-full object-cover"
|
filePath={file.filePath}
|
||||||
|
mimeType={file.mimeType}
|
||||||
|
onPreview={() => setLightboxImg({ src: file.preview!, fileName: file.fileName, filePath: file.filePath, mimeType: file.mimeType })}
|
||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
<div className="w-full h-full flex items-center justify-center text-muted-foreground">
|
<div
|
||||||
|
key={`local-${i}`}
|
||||||
|
className="w-36 h-36 rounded-xl border overflow-hidden bg-muted flex items-center justify-center text-muted-foreground"
|
||||||
|
>
|
||||||
<File className="h-8 w-8" />
|
<File className="h-8 w-8" />
|
||||||
</div>
|
</div>
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
// Non-image files → file card
|
// Non-image files → file card
|
||||||
@@ -155,21 +166,26 @@ export const ChatMessage = memo(function ChatMessage({
|
|||||||
text={text}
|
text={text}
|
||||||
isUser={isUser}
|
isUser={isUser}
|
||||||
isStreaming={isStreaming}
|
isStreaming={isStreaming}
|
||||||
timestamp={message.timestamp}
|
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Images from content blocks — assistant messages (below text) */}
|
{/* Images from content blocks — assistant messages (below text) */}
|
||||||
{!isUser && images.length > 0 && (
|
{!isUser && images.length > 0 && (
|
||||||
<div className="flex flex-wrap gap-2">
|
<div className="flex flex-wrap gap-2">
|
||||||
{images.map((img, i) => (
|
{images.map((img, i) => {
|
||||||
<img
|
const src = imageSrc(img);
|
||||||
|
if (!src) return null;
|
||||||
|
return (
|
||||||
|
<ImagePreviewCard
|
||||||
key={`content-${i}`}
|
key={`content-${i}`}
|
||||||
src={`data:${img.mimeType};base64,${img.data}`}
|
src={src}
|
||||||
alt="attachment"
|
fileName="image"
|
||||||
className="max-w-xs rounded-lg border"
|
base64={img.data}
|
||||||
|
mimeType={img.mimeType}
|
||||||
|
onPreview={() => setLightboxImg({ src, fileName: 'image', base64: img.data, mimeType: img.mimeType })}
|
||||||
/>
|
/>
|
||||||
))}
|
);
|
||||||
|
})}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@@ -181,11 +197,13 @@ export const ChatMessage = memo(function ChatMessage({
|
|||||||
if (isImage && images.length > 0) return null;
|
if (isImage && images.length > 0) return null;
|
||||||
if (isImage && file.preview) {
|
if (isImage && file.preview) {
|
||||||
return (
|
return (
|
||||||
<img
|
<ImagePreviewCard
|
||||||
key={`local-${i}`}
|
key={`local-${i}`}
|
||||||
src={file.preview}
|
src={file.preview}
|
||||||
alt={file.fileName}
|
fileName={file.fileName}
|
||||||
className="max-w-xs rounded-lg border"
|
filePath={file.filePath}
|
||||||
|
mimeType={file.mimeType}
|
||||||
|
onPreview={() => setLightboxImg({ src: file.preview!, fileName: file.fileName, filePath: file.filePath, mimeType: file.mimeType })}
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -201,13 +219,30 @@ export const ChatMessage = memo(function ChatMessage({
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Hover timestamp for user messages (shown below content on hover) */}
|
{/* Hover row for user messages — timestamp only */}
|
||||||
{isUser && message.timestamp && (
|
{isUser && message.timestamp && (
|
||||||
<span className="text-xs text-muted-foreground opacity-0 group-hover:opacity-100 transition-opacity duration-200 select-none">
|
<span className="text-xs text-muted-foreground opacity-0 group-hover:opacity-100 transition-opacity duration-200 select-none">
|
||||||
{formatTimestamp(message.timestamp)}
|
{formatTimestamp(message.timestamp)}
|
||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Hover row for assistant messages — only when there is real text content */}
|
||||||
|
{!isUser && hasText && (
|
||||||
|
<AssistantHoverBar text={text} timestamp={message.timestamp} />
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Image lightbox portal */}
|
||||||
|
{lightboxImg && (
|
||||||
|
<ImageLightbox
|
||||||
|
src={lightboxImg.src}
|
||||||
|
fileName={lightboxImg.fileName}
|
||||||
|
filePath={lightboxImg.filePath}
|
||||||
|
base64={lightboxImg.base64}
|
||||||
|
mimeType={lightboxImg.mimeType}
|
||||||
|
onClose={() => setLightboxImg(null)}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -257,19 +292,9 @@ function ToolStatusBar({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Message Bubble ──────────────────────────────────────────────
|
// ── Assistant hover bar (timestamp + copy, shown on group hover) ─
|
||||||
|
|
||||||
function MessageBubble({
|
function AssistantHoverBar({ text, timestamp }: { text: string; timestamp?: number }) {
|
||||||
text,
|
|
||||||
isUser,
|
|
||||||
isStreaming,
|
|
||||||
timestamp,
|
|
||||||
}: {
|
|
||||||
text: string;
|
|
||||||
isUser: boolean;
|
|
||||||
isStreaming: boolean;
|
|
||||||
timestamp?: number;
|
|
||||||
}) {
|
|
||||||
const [copied, setCopied] = useState(false);
|
const [copied, setCopied] = useState(false);
|
||||||
|
|
||||||
const copyContent = useCallback(() => {
|
const copyContent = useCallback(() => {
|
||||||
@@ -278,6 +303,34 @@ function MessageBubble({
|
|||||||
setTimeout(() => setCopied(false), 2000);
|
setTimeout(() => setCopied(false), 2000);
|
||||||
}, [text]);
|
}, [text]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex items-center justify-between w-full opacity-0 group-hover:opacity-100 transition-opacity duration-200 select-none px-1">
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
{timestamp ? formatTimestamp(timestamp) : ''}
|
||||||
|
</span>
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
className="h-6 w-6"
|
||||||
|
onClick={copyContent}
|
||||||
|
>
|
||||||
|
{copied ? <Check className="h-3 w-3 text-green-500" /> : <Copy className="h-3 w-3" />}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Message Bubble ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
function MessageBubble({
|
||||||
|
text,
|
||||||
|
isUser,
|
||||||
|
isStreaming,
|
||||||
|
}: {
|
||||||
|
text: string;
|
||||||
|
isUser: boolean;
|
||||||
|
isStreaming: boolean;
|
||||||
|
}) {
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
className={cn(
|
className={cn(
|
||||||
@@ -330,24 +383,6 @@ function MessageBubble({
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* Footer: copy button (assistant only; user timestamp is rendered outside the bubble) */}
|
|
||||||
{!isUser && (
|
|
||||||
<div className="flex items-center justify-between mt-2">
|
|
||||||
{timestamp ? (
|
|
||||||
<span className="text-xs text-muted-foreground">
|
|
||||||
{formatTimestamp(timestamp)}
|
|
||||||
</span>
|
|
||||||
) : <span />}
|
|
||||||
<Button
|
|
||||||
variant="ghost"
|
|
||||||
size="icon"
|
|
||||||
className="h-6 w-6 opacity-0 group-hover:opacity-100 transition-opacity"
|
|
||||||
onClick={copyContent}
|
|
||||||
>
|
|
||||||
{copied ? <Check className="h-3 w-3 text-green-500" /> : <Copy className="h-3 w-3" />}
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -409,6 +444,146 @@ function FileCard({ file }: { file: AttachedFileMeta }) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Image Thumbnail (user bubble — square crop with zoom hint) ──
|
||||||
|
|
||||||
|
function ImageThumbnail({
|
||||||
|
src,
|
||||||
|
fileName,
|
||||||
|
filePath,
|
||||||
|
base64,
|
||||||
|
mimeType,
|
||||||
|
onPreview,
|
||||||
|
}: {
|
||||||
|
src: string;
|
||||||
|
fileName: string;
|
||||||
|
filePath?: string;
|
||||||
|
base64?: string;
|
||||||
|
mimeType?: string;
|
||||||
|
onPreview: () => void;
|
||||||
|
}) {
|
||||||
|
void filePath; void base64; void mimeType;
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="relative w-36 h-36 rounded-xl border overflow-hidden bg-muted group/img cursor-zoom-in"
|
||||||
|
onClick={onPreview}
|
||||||
|
>
|
||||||
|
<img src={src} alt={fileName} className="w-full h-full object-cover" />
|
||||||
|
<div className="absolute inset-0 bg-black/0 group-hover/img:bg-black/25 transition-colors flex items-center justify-center">
|
||||||
|
<ZoomIn className="h-6 w-6 text-white opacity-0 group-hover/img:opacity-100 transition-opacity drop-shadow" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Image Preview Card (assistant bubble — natural size with overlay actions) ──
|
||||||
|
|
||||||
|
function ImagePreviewCard({
|
||||||
|
src,
|
||||||
|
fileName,
|
||||||
|
filePath,
|
||||||
|
base64,
|
||||||
|
mimeType,
|
||||||
|
onPreview,
|
||||||
|
}: {
|
||||||
|
src: string;
|
||||||
|
fileName: string;
|
||||||
|
filePath?: string;
|
||||||
|
base64?: string;
|
||||||
|
mimeType?: string;
|
||||||
|
onPreview: () => void;
|
||||||
|
}) {
|
||||||
|
void filePath; void base64; void mimeType;
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="relative max-w-xs rounded-lg border overflow-hidden group/img cursor-zoom-in"
|
||||||
|
onClick={onPreview}
|
||||||
|
>
|
||||||
|
<img src={src} alt={fileName} className="block w-full" />
|
||||||
|
<div className="absolute inset-0 bg-black/0 group-hover/img:bg-black/20 transition-colors flex items-center justify-center">
|
||||||
|
<ZoomIn className="h-6 w-6 text-white opacity-0 group-hover/img:opacity-100 transition-opacity drop-shadow" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Image Lightbox ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
function ImageLightbox({
|
||||||
|
src,
|
||||||
|
fileName,
|
||||||
|
filePath,
|
||||||
|
base64,
|
||||||
|
mimeType,
|
||||||
|
onClose,
|
||||||
|
}: {
|
||||||
|
src: string;
|
||||||
|
fileName: string;
|
||||||
|
filePath?: string;
|
||||||
|
base64?: string;
|
||||||
|
mimeType?: string;
|
||||||
|
onClose: () => void;
|
||||||
|
}) {
|
||||||
|
void src; void base64; void mimeType; void fileName;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const handleKey = (e: KeyboardEvent) => {
|
||||||
|
if (e.key === 'Escape') onClose();
|
||||||
|
};
|
||||||
|
window.addEventListener('keydown', handleKey);
|
||||||
|
return () => window.removeEventListener('keydown', handleKey);
|
||||||
|
}, [onClose]);
|
||||||
|
|
||||||
|
const handleShowInFolder = useCallback(() => {
|
||||||
|
if (filePath) {
|
||||||
|
window.electron.ipcRenderer.invoke('shell:showItemInFolder', filePath);
|
||||||
|
}
|
||||||
|
}, [filePath]);
|
||||||
|
|
||||||
|
return createPortal(
|
||||||
|
<div
|
||||||
|
className="fixed inset-0 z-50 flex items-center justify-center bg-black/80 backdrop-blur-sm"
|
||||||
|
onClick={onClose}
|
||||||
|
>
|
||||||
|
{/* Image + buttons stacked */}
|
||||||
|
<div
|
||||||
|
className="flex flex-col items-center gap-3"
|
||||||
|
onClick={(e) => e.stopPropagation()}
|
||||||
|
>
|
||||||
|
<img
|
||||||
|
src={src}
|
||||||
|
alt={fileName}
|
||||||
|
className="max-w-[90vw] max-h-[85vh] rounded-lg shadow-2xl object-contain"
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* Action buttons below image */}
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{filePath && (
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
className="h-8 w-8 bg-white/10 hover:bg-white/20 text-white"
|
||||||
|
onClick={handleShowInFolder}
|
||||||
|
title="在文件夹中显示"
|
||||||
|
>
|
||||||
|
<FolderOpen className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
className="h-8 w-8 bg-white/10 hover:bg-white/20 text-white"
|
||||||
|
onClick={onClose}
|
||||||
|
title="关闭"
|
||||||
|
>
|
||||||
|
<X className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>,
|
||||||
|
document.body,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// ── Tool Card ───────────────────────────────────────────────────
|
// ── Tool Card ───────────────────────────────────────────────────
|
||||||
|
|
||||||
function ToolCard({ name, input }: { name: string; input: unknown }) {
|
function ToolCard({ name, input }: { name: string; input: unknown }) {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ export interface AttachedFileMeta {
|
|||||||
mimeType: string;
|
mimeType: string;
|
||||||
fileSize: number;
|
fileSize: number;
|
||||||
preview: string | null;
|
preview: string | null;
|
||||||
|
filePath?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Raw message from OpenClaw chat.history */
|
/** Raw message from OpenClaw chat.history */
|
||||||
@@ -34,7 +35,7 @@ export interface ContentBlock {
|
|||||||
type: 'text' | 'image' | 'thinking' | 'tool_use' | 'tool_result' | 'toolCall' | 'toolResult';
|
type: 'text' | 'image' | 'thinking' | 'tool_use' | 'tool_result' | 'toolCall' | 'toolResult';
|
||||||
text?: string;
|
text?: string;
|
||||||
thinking?: string;
|
thinking?: string;
|
||||||
source?: { type: string; media_type: string; data: string };
|
source?: { type: string; media_type?: string; data?: string; url?: string };
|
||||||
id?: string;
|
id?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
input?: unknown;
|
input?: unknown;
|
||||||
@@ -75,6 +76,8 @@ interface ChatState {
|
|||||||
streamingTools: ToolStatus[];
|
streamingTools: ToolStatus[];
|
||||||
pendingFinal: boolean;
|
pendingFinal: boolean;
|
||||||
lastUserMessageAt: number | null;
|
lastUserMessageAt: number | null;
|
||||||
|
/** Images collected from tool results, attached to the next assistant message */
|
||||||
|
pendingToolImages: AttachedFileMeta[];
|
||||||
|
|
||||||
// Sessions
|
// Sessions
|
||||||
sessions: ChatSession[];
|
sessions: ChatSession[];
|
||||||
@@ -88,7 +91,7 @@ interface ChatState {
|
|||||||
loadSessions: () => Promise<void>;
|
loadSessions: () => Promise<void>;
|
||||||
switchSession: (key: string) => void;
|
switchSession: (key: string) => void;
|
||||||
newSession: () => void;
|
newSession: () => void;
|
||||||
loadHistory: () => Promise<void>;
|
loadHistory: (quiet?: boolean) => Promise<void>;
|
||||||
sendMessage: (text: string, attachments?: Array<{ fileName: string; mimeType: string; fileSize: number; stagedPath: string; preview: string | null }>) => Promise<void>;
|
sendMessage: (text: string, attachments?: Array<{ fileName: string; mimeType: string; fileSize: number; stagedPath: string; preview: string | null }>) => Promise<void>;
|
||||||
abortRun: () => Promise<void>;
|
abortRun: () => Promise<void>;
|
||||||
handleChatEvent: (event: Record<string, unknown>) => void;
|
handleChatEvent: (event: Record<string, unknown>) => void;
|
||||||
@@ -156,23 +159,232 @@ function extractMediaRefs(text: string): Array<{ filePath: string; mimeType: str
|
|||||||
return refs;
|
return refs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Map common file extensions to MIME types */
|
||||||
|
function mimeFromExtension(filePath: string): string {
|
||||||
|
const ext = filePath.split('.').pop()?.toLowerCase() || '';
|
||||||
|
const map: Record<string, string> = {
|
||||||
|
// Images
|
||||||
|
'png': 'image/png',
|
||||||
|
'jpg': 'image/jpeg',
|
||||||
|
'jpeg': 'image/jpeg',
|
||||||
|
'gif': 'image/gif',
|
||||||
|
'webp': 'image/webp',
|
||||||
|
'bmp': 'image/bmp',
|
||||||
|
'avif': 'image/avif',
|
||||||
|
'svg': 'image/svg+xml',
|
||||||
|
// Documents
|
||||||
|
'pdf': 'application/pdf',
|
||||||
|
'doc': 'application/msword',
|
||||||
|
'docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||||
|
'xls': 'application/vnd.ms-excel',
|
||||||
|
'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
'ppt': 'application/vnd.ms-powerpoint',
|
||||||
|
'pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||||
|
'txt': 'text/plain',
|
||||||
|
'csv': 'text/csv',
|
||||||
|
'md': 'text/markdown',
|
||||||
|
'rtf': 'application/rtf',
|
||||||
|
'epub': 'application/epub+zip',
|
||||||
|
// Archives
|
||||||
|
'zip': 'application/zip',
|
||||||
|
'tar': 'application/x-tar',
|
||||||
|
'gz': 'application/gzip',
|
||||||
|
'rar': 'application/vnd.rar',
|
||||||
|
'7z': 'application/x-7z-compressed',
|
||||||
|
// Audio
|
||||||
|
'mp3': 'audio/mpeg',
|
||||||
|
'wav': 'audio/wav',
|
||||||
|
'ogg': 'audio/ogg',
|
||||||
|
'aac': 'audio/aac',
|
||||||
|
'flac': 'audio/flac',
|
||||||
|
'm4a': 'audio/mp4',
|
||||||
|
// Video
|
||||||
|
'mp4': 'video/mp4',
|
||||||
|
'mov': 'video/quicktime',
|
||||||
|
'avi': 'video/x-msvideo',
|
||||||
|
'mkv': 'video/x-matroska',
|
||||||
|
'webm': 'video/webm',
|
||||||
|
'm4v': 'video/mp4',
|
||||||
|
};
|
||||||
|
return map[ext] || 'application/octet-stream';
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Restore _attachedFiles for user messages loaded from history.
|
* Extract raw file paths from message text.
|
||||||
* Uses local cache for previews when available, but ALWAYS creates entries
|
* Detects absolute Unix paths (/ or ~/) ending with common file extensions.
|
||||||
* from [media attached: ...] text patterns so file cards show even without cache.
|
* Handles both image and non-image files, consistent with channel push message behavior.
|
||||||
|
*/
|
||||||
|
function extractRawFilePaths(text: string): Array<{ filePath: string; mimeType: string }> {
|
||||||
|
const refs: Array<{ filePath: string; mimeType: string }> = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
// Match absolute Unix paths with common file extensions (including Unicode filenames)
|
||||||
|
const regex = /((?:\/|~\/)[^\s\n"'()[\],<>]*?\.(?:png|jpe?g|gif|webp|bmp|avif|svg|pdf|docx?|xlsx?|pptx?|txt|csv|md|rtf|epub|zip|tar|gz|rar|7z|mp3|wav|ogg|aac|flac|m4a|mp4|mov|avi|mkv|webm|m4v))/gi;
|
||||||
|
let match;
|
||||||
|
while ((match = regex.exec(text)) !== null) {
|
||||||
|
const p = match[1];
|
||||||
|
if (p && !seen.has(p)) {
|
||||||
|
seen.add(p);
|
||||||
|
refs.push({ filePath: p, mimeType: mimeFromExtension(p) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return refs;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract images from a content array (including nested tool_result content).
|
||||||
|
* Converts them to AttachedFileMeta entries with preview set to data URL or remote URL.
|
||||||
|
*/
|
||||||
|
function extractImagesAsAttachedFiles(content: unknown): AttachedFileMeta[] {
|
||||||
|
if (!Array.isArray(content)) return [];
|
||||||
|
const files: AttachedFileMeta[] = [];
|
||||||
|
|
||||||
|
for (const block of content as ContentBlock[]) {
|
||||||
|
if (block.type === 'image' && block.source) {
|
||||||
|
const src = block.source;
|
||||||
|
const mimeType = src.media_type || 'image/jpeg';
|
||||||
|
|
||||||
|
if (src.type === 'base64' && src.data) {
|
||||||
|
files.push({
|
||||||
|
fileName: 'image',
|
||||||
|
mimeType,
|
||||||
|
fileSize: 0,
|
||||||
|
preview: `data:${mimeType};base64,${src.data}`,
|
||||||
|
});
|
||||||
|
} else if (src.type === 'url' && src.url) {
|
||||||
|
files.push({
|
||||||
|
fileName: 'image',
|
||||||
|
mimeType,
|
||||||
|
fileSize: 0,
|
||||||
|
preview: src.url,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Recurse into tool_result content blocks
|
||||||
|
if ((block.type === 'tool_result' || block.type === 'toolResult') && block.content) {
|
||||||
|
files.push(...extractImagesAsAttachedFiles(block.content));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build an AttachedFileMeta entry for a file ref, using cache if available.
|
||||||
|
*/
|
||||||
|
function makeAttachedFile(ref: { filePath: string; mimeType: string }): AttachedFileMeta {
|
||||||
|
const cached = _imageCache.get(ref.filePath);
|
||||||
|
if (cached) return { ...cached, filePath: ref.filePath };
|
||||||
|
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
|
||||||
|
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Before filtering tool_result messages from history, scan them for any file/image
|
||||||
|
* content and attach those to the immediately following assistant message.
|
||||||
|
* This mirrors channel push message behavior where tool outputs surface files to the UI.
|
||||||
|
* Handles:
|
||||||
|
* - Image content blocks (base64 / url)
|
||||||
|
* - [media attached: path (mime) | path] text patterns in tool result output
|
||||||
|
* - Raw file paths in tool result text
|
||||||
|
*/
|
||||||
|
function enrichWithToolResultFiles(messages: RawMessage[]): RawMessage[] {
|
||||||
|
const pending: AttachedFileMeta[] = [];
|
||||||
|
|
||||||
|
return messages.map((msg) => {
|
||||||
|
if (isToolResultRole(msg.role)) {
|
||||||
|
// 1. Image/file content blocks in the structured content array
|
||||||
|
pending.push(...extractImagesAsAttachedFiles(msg.content));
|
||||||
|
|
||||||
|
// 2. [media attached: ...] patterns in tool result text output
|
||||||
|
const text = getMessageText(msg.content);
|
||||||
|
if (text) {
|
||||||
|
const mediaRefs = extractMediaRefs(text);
|
||||||
|
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
|
||||||
|
for (const ref of mediaRefs) {
|
||||||
|
pending.push(makeAttachedFile(ref));
|
||||||
|
}
|
||||||
|
// 3. Raw file paths in tool result text (documents, audio, video, etc.)
|
||||||
|
for (const ref of extractRawFilePaths(text)) {
|
||||||
|
if (!mediaRefPaths.has(ref.filePath)) {
|
||||||
|
pending.push(makeAttachedFile(ref));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return msg; // will be filtered later
|
||||||
|
}
|
||||||
|
|
||||||
|
if (msg.role === 'assistant' && pending.length > 0) {
|
||||||
|
const toAttach = pending.splice(0);
|
||||||
|
// Deduplicate against files already on the assistant message
|
||||||
|
const existingPaths = new Set(
|
||||||
|
(msg._attachedFiles || []).map(f => f.filePath).filter(Boolean),
|
||||||
|
);
|
||||||
|
const newFiles = toAttach.filter(f => !f.filePath || !existingPaths.has(f.filePath));
|
||||||
|
if (newFiles.length === 0) return msg;
|
||||||
|
return {
|
||||||
|
...msg,
|
||||||
|
_attachedFiles: [...(msg._attachedFiles || []), ...newFiles],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return msg;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Restore _attachedFiles for messages loaded from history.
|
||||||
|
* Handles:
|
||||||
|
* 1. [media attached: path (mime) | path] patterns (attachment-button flow)
|
||||||
|
* 2. Raw image file paths typed in message text (e.g. /Users/.../image.png)
|
||||||
|
* Uses local cache for previews when available; missing previews are loaded async.
|
||||||
*/
|
*/
|
||||||
function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
|
function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
|
||||||
return messages.map(msg => {
|
return messages.map((msg, idx) => {
|
||||||
if (msg.role !== 'user' || msg._attachedFiles) return msg;
|
// Only process user and assistant messages; skip if already enriched
|
||||||
|
if ((msg.role !== 'user' && msg.role !== 'assistant') || msg._attachedFiles) return msg;
|
||||||
const text = getMessageText(msg.content);
|
const text = getMessageText(msg.content);
|
||||||
const refs = extractMediaRefs(text);
|
|
||||||
if (refs.length === 0) return msg;
|
// Path 1: [media attached: path (mime) | path] — guaranteed format from attachment button
|
||||||
const files: AttachedFileMeta[] = refs.map(ref => {
|
const mediaRefs = extractMediaRefs(text);
|
||||||
|
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
|
||||||
|
|
||||||
|
// Path 2: Raw file paths.
|
||||||
|
// For assistant messages: scan own text AND the nearest preceding user message text,
|
||||||
|
// but only for non-tool-only assistant messages (i.e. the final answer turn).
|
||||||
|
// Tool-only messages (thinking + tool calls) should not show file previews — those
|
||||||
|
// belong to the final answer message that comes after the tool results.
|
||||||
|
// User messages never get raw-path previews so the image is not shown twice.
|
||||||
|
let rawRefs: Array<{ filePath: string; mimeType: string }> = [];
|
||||||
|
if (msg.role === 'assistant' && !isToolOnlyMessage(msg)) {
|
||||||
|
// Own text
|
||||||
|
rawRefs = extractRawFilePaths(text).filter(r => !mediaRefPaths.has(r.filePath));
|
||||||
|
|
||||||
|
// Nearest preceding user message text (look back up to 5 messages)
|
||||||
|
const seenPaths = new Set(rawRefs.map(r => r.filePath));
|
||||||
|
for (let i = idx - 1; i >= Math.max(0, idx - 5); i--) {
|
||||||
|
const prev = messages[i];
|
||||||
|
if (!prev) break;
|
||||||
|
if (prev.role === 'user') {
|
||||||
|
const prevText = getMessageText(prev.content);
|
||||||
|
for (const ref of extractRawFilePaths(prevText)) {
|
||||||
|
if (!mediaRefPaths.has(ref.filePath) && !seenPaths.has(ref.filePath)) {
|
||||||
|
seenPaths.add(ref.filePath);
|
||||||
|
rawRefs.push(ref);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break; // only use the nearest user message
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const allRefs = [...mediaRefs, ...rawRefs];
|
||||||
|
if (allRefs.length === 0) return msg;
|
||||||
|
|
||||||
|
const files: AttachedFileMeta[] = allRefs.map(ref => {
|
||||||
const cached = _imageCache.get(ref.filePath);
|
const cached = _imageCache.get(ref.filePath);
|
||||||
if (cached) return cached;
|
if (cached) return { ...cached, filePath: ref.filePath };
|
||||||
// Fallback: create entry from text pattern (preview loaded later via IPC)
|
|
||||||
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
|
const fileName = ref.filePath.split(/[\\/]/).pop() || 'file';
|
||||||
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null };
|
return { fileName, mimeType: ref.mimeType, fileSize: 0, preview: null, filePath: ref.filePath };
|
||||||
});
|
});
|
||||||
return { ...msg, _attachedFiles: files };
|
return { ...msg, _attachedFiles: files };
|
||||||
});
|
});
|
||||||
@@ -181,21 +393,47 @@ function enrichWithCachedImages(messages: RawMessage[]): RawMessage[] {
|
|||||||
/**
|
/**
|
||||||
* Async: load missing previews from disk via IPC for messages that have
|
* Async: load missing previews from disk via IPC for messages that have
|
||||||
* _attachedFiles with null previews. Updates messages in-place and triggers re-render.
|
* _attachedFiles with null previews. Updates messages in-place and triggers re-render.
|
||||||
|
* Handles both [media attached: ...] patterns and raw filePath entries.
|
||||||
*/
|
*/
|
||||||
async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
|
async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
|
||||||
// Collect all image paths that need previews
|
// Collect all image paths that need previews
|
||||||
const needPreview: Array<{ filePath: string; mimeType: string }> = [];
|
const needPreview: Array<{ filePath: string; mimeType: string }> = [];
|
||||||
|
const seenPaths = new Set<string>();
|
||||||
|
|
||||||
for (const msg of messages) {
|
for (const msg of messages) {
|
||||||
if (msg.role !== 'user' || !msg._attachedFiles) continue;
|
if (!msg._attachedFiles) continue;
|
||||||
|
|
||||||
|
// Path 1: files with explicit filePath field (raw path detection or enriched refs)
|
||||||
|
for (const file of msg._attachedFiles) {
|
||||||
|
const fp = file.filePath;
|
||||||
|
if (!fp || seenPaths.has(fp)) continue;
|
||||||
|
// Images: need preview. Non-images: need file size (for FileCard display).
|
||||||
|
const needsLoad = file.mimeType.startsWith('image/')
|
||||||
|
? !file.preview
|
||||||
|
: file.fileSize === 0;
|
||||||
|
if (needsLoad) {
|
||||||
|
seenPaths.add(fp);
|
||||||
|
needPreview.push({ filePath: fp, mimeType: file.mimeType });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Path 2: [media attached: ...] patterns (legacy — in case filePath wasn't stored)
|
||||||
|
if (msg.role === 'user') {
|
||||||
const text = getMessageText(msg.content);
|
const text = getMessageText(msg.content);
|
||||||
const refs = extractMediaRefs(text);
|
const refs = extractMediaRefs(text);
|
||||||
for (let i = 0; i < refs.length; i++) {
|
for (let i = 0; i < refs.length; i++) {
|
||||||
const file = msg._attachedFiles[i];
|
const file = msg._attachedFiles[i];
|
||||||
if (file && file.mimeType.startsWith('image/') && !file.preview) {
|
const ref = refs[i];
|
||||||
needPreview.push(refs[i]);
|
if (!file || !ref || seenPaths.has(ref.filePath)) continue;
|
||||||
|
const needsLoad = ref.mimeType.startsWith('image/') ? !file.preview : file.fileSize === 0;
|
||||||
|
if (needsLoad) {
|
||||||
|
seenPaths.add(ref.filePath);
|
||||||
|
needPreview.push(ref);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (needPreview.length === 0) return false;
|
if (needPreview.length === 0) return false;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -206,21 +444,39 @@ async function loadMissingPreviews(messages: RawMessage[]): Promise<boolean> {
|
|||||||
|
|
||||||
let updated = false;
|
let updated = false;
|
||||||
for (const msg of messages) {
|
for (const msg of messages) {
|
||||||
if (msg.role !== 'user' || !msg._attachedFiles) continue;
|
if (!msg._attachedFiles) continue;
|
||||||
|
|
||||||
|
// Update files that have filePath
|
||||||
|
for (const file of msg._attachedFiles) {
|
||||||
|
const fp = file.filePath;
|
||||||
|
if (!fp) continue;
|
||||||
|
const thumb = thumbnails[fp];
|
||||||
|
if (thumb && (thumb.preview || thumb.fileSize)) {
|
||||||
|
if (thumb.preview) file.preview = thumb.preview;
|
||||||
|
if (thumb.fileSize) file.fileSize = thumb.fileSize;
|
||||||
|
_imageCache.set(fp, { ...file });
|
||||||
|
updated = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Legacy: update by index for [media attached: ...] refs
|
||||||
|
if (msg.role === 'user') {
|
||||||
const text = getMessageText(msg.content);
|
const text = getMessageText(msg.content);
|
||||||
const refs = extractMediaRefs(text);
|
const refs = extractMediaRefs(text);
|
||||||
for (let i = 0; i < refs.length; i++) {
|
for (let i = 0; i < refs.length; i++) {
|
||||||
const file = msg._attachedFiles[i];
|
const file = msg._attachedFiles[i];
|
||||||
const thumb = thumbnails[refs[i]?.filePath];
|
const ref = refs[i];
|
||||||
if (file && thumb && (thumb.preview || thumb.fileSize)) {
|
if (!file || !ref || file.filePath) continue; // skip if already handled via filePath
|
||||||
|
const thumb = thumbnails[ref.filePath];
|
||||||
|
if (thumb && (thumb.preview || thumb.fileSize)) {
|
||||||
if (thumb.preview) file.preview = thumb.preview;
|
if (thumb.preview) file.preview = thumb.preview;
|
||||||
if (thumb.fileSize) file.fileSize = thumb.fileSize;
|
if (thumb.fileSize) file.fileSize = thumb.fileSize;
|
||||||
// Update cache for future loads
|
_imageCache.set(ref.filePath, { ...file });
|
||||||
_imageCache.set(refs[i].filePath, { ...file });
|
|
||||||
updated = true;
|
updated = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if (updated) saveImageCache(_imageCache);
|
if (updated) saveImageCache(_imageCache);
|
||||||
return updated;
|
return updated;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -272,7 +528,10 @@ function isToolOnlyMessage(message: RawMessage | undefined): boolean {
|
|||||||
hasText = true;
|
hasText = true;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (block.type === 'image' || block.type === 'thinking') {
|
// Only actual image output disqualifies a tool-only message.
|
||||||
|
// Thinking blocks are internal reasoning that can accompany tool_use — they
|
||||||
|
// should NOT prevent the message from being treated as an intermediate tool step.
|
||||||
|
if (block.type === 'image') {
|
||||||
hasNonToolContent = true;
|
hasNonToolContent = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -496,6 +755,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
streamingTools: [],
|
streamingTools: [],
|
||||||
pendingFinal: false,
|
pendingFinal: false,
|
||||||
lastUserMessageAt: null,
|
lastUserMessageAt: null,
|
||||||
|
pendingToolImages: [],
|
||||||
|
|
||||||
sessions: [],
|
sessions: [],
|
||||||
currentSessionKey: DEFAULT_SESSION_KEY,
|
currentSessionKey: DEFAULT_SESSION_KEY,
|
||||||
@@ -588,6 +848,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
error: null,
|
error: null,
|
||||||
pendingFinal: false,
|
pendingFinal: false,
|
||||||
lastUserMessageAt: null,
|
lastUserMessageAt: null,
|
||||||
|
pendingToolImages: [],
|
||||||
});
|
});
|
||||||
// Load history for new session
|
// Load history for new session
|
||||||
get().loadHistory();
|
get().loadHistory();
|
||||||
@@ -611,14 +872,15 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
error: null,
|
error: null,
|
||||||
pendingFinal: false,
|
pendingFinal: false,
|
||||||
lastUserMessageAt: null,
|
lastUserMessageAt: null,
|
||||||
|
pendingToolImages: [],
|
||||||
}));
|
}));
|
||||||
},
|
},
|
||||||
|
|
||||||
// ── Load chat history ──
|
// ── Load chat history ──
|
||||||
|
|
||||||
loadHistory: async () => {
|
loadHistory: async (quiet = false) => {
|
||||||
const { currentSessionKey } = get();
|
const { currentSessionKey } = get();
|
||||||
set({ loading: true, error: null });
|
if (!quiet) set({ loading: true, error: null });
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await window.electron.ipcRenderer.invoke(
|
const result = await window.electron.ipcRenderer.invoke(
|
||||||
@@ -630,8 +892,10 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
if (result.success && result.result) {
|
if (result.success && result.result) {
|
||||||
const data = result.result;
|
const data = result.result;
|
||||||
const rawMessages = Array.isArray(data.messages) ? data.messages as RawMessage[] : [];
|
const rawMessages = Array.isArray(data.messages) ? data.messages as RawMessage[] : [];
|
||||||
const filteredMessages = rawMessages.filter((msg) => !isToolResultRole(msg.role));
|
// Before filtering: attach images/files from tool_result messages to the next assistant message
|
||||||
// Restore file attachments for user messages (from cache + text patterns)
|
const messagesWithToolImages = enrichWithToolResultFiles(rawMessages);
|
||||||
|
const filteredMessages = messagesWithToolImages.filter((msg) => !isToolResultRole(msg.role));
|
||||||
|
// Restore file attachments for user/assistant messages (from cache + text patterns)
|
||||||
const enrichedMessages = enrichWithCachedImages(filteredMessages);
|
const enrichedMessages = enrichWithCachedImages(filteredMessages);
|
||||||
const thinkingLevel = data.thinkingLevel ? String(data.thinkingLevel) : null;
|
const thinkingLevel = data.thinkingLevel ? String(data.thinkingLevel) : null;
|
||||||
set({ messages: enrichedMessages, thinkingLevel, loading: false });
|
set({ messages: enrichedMessages, thinkingLevel, loading: false });
|
||||||
@@ -691,6 +955,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
mimeType: a.mimeType,
|
mimeType: a.mimeType,
|
||||||
fileSize: a.fileSize,
|
fileSize: a.fileSize,
|
||||||
preview: a.preview,
|
preview: a.preview,
|
||||||
|
filePath: a.stagedPath,
|
||||||
})),
|
})),
|
||||||
};
|
};
|
||||||
set((s) => ({
|
set((s) => ({
|
||||||
@@ -778,7 +1043,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
|
|
||||||
abortRun: async () => {
|
abortRun: async () => {
|
||||||
const { currentSessionKey } = get();
|
const { currentSessionKey } = get();
|
||||||
set({ sending: false, streamingText: '', streamingMessage: null, pendingFinal: false, lastUserMessageAt: null });
|
set({ sending: false, streamingText: '', streamingMessage: null, pendingFinal: false, lastUserMessageAt: null, pendingToolImages: [] });
|
||||||
set({ streamingTools: [] });
|
set({ streamingTools: [] });
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -840,11 +1105,53 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
if (finalMsg) {
|
if (finalMsg) {
|
||||||
const updates = collectToolUpdates(finalMsg, resolvedState);
|
const updates = collectToolUpdates(finalMsg, resolvedState);
|
||||||
if (isToolResultRole(finalMsg.role)) {
|
if (isToolResultRole(finalMsg.role)) {
|
||||||
set((s) => ({
|
// Mirror enrichWithToolResultFiles: collect images + file refs for next assistant msg
|
||||||
|
const toolFiles: AttachedFileMeta[] = [
|
||||||
|
...extractImagesAsAttachedFiles(finalMsg.content),
|
||||||
|
];
|
||||||
|
const text = getMessageText(finalMsg.content);
|
||||||
|
if (text) {
|
||||||
|
const mediaRefs = extractMediaRefs(text);
|
||||||
|
const mediaRefPaths = new Set(mediaRefs.map(r => r.filePath));
|
||||||
|
for (const ref of mediaRefs) toolFiles.push(makeAttachedFile(ref));
|
||||||
|
for (const ref of extractRawFilePaths(text)) {
|
||||||
|
if (!mediaRefPaths.has(ref.filePath)) toolFiles.push(makeAttachedFile(ref));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set((s) => {
|
||||||
|
// Snapshot the current streaming assistant message (thinking + tool_use) into
|
||||||
|
// messages[] before clearing it. The Gateway does NOT send separate 'final'
|
||||||
|
// events for intermediate tool-use turns — it only sends deltas and then the
|
||||||
|
// tool result. Without snapshotting here, the intermediate thinking+tool steps
|
||||||
|
// would be overwritten by the next turn's deltas and never appear in the UI.
|
||||||
|
const currentStream = s.streamingMessage as RawMessage | null;
|
||||||
|
const snapshotMsgs: RawMessage[] = [];
|
||||||
|
if (currentStream) {
|
||||||
|
const streamRole = currentStream.role;
|
||||||
|
if (streamRole === 'assistant' || streamRole === undefined) {
|
||||||
|
// Use message's own id if available, otherwise derive a stable one from runId
|
||||||
|
const snapId = currentStream.id
|
||||||
|
|| `${runId || 'run'}-turn-${s.messages.length}`;
|
||||||
|
if (!s.messages.some(m => m.id === snapId)) {
|
||||||
|
snapshotMsgs.push({
|
||||||
|
...(currentStream as RawMessage),
|
||||||
|
role: 'assistant',
|
||||||
|
id: snapId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
messages: snapshotMsgs.length > 0 ? [...s.messages, ...snapshotMsgs] : s.messages,
|
||||||
streamingText: '',
|
streamingText: '',
|
||||||
|
streamingMessage: null,
|
||||||
pendingFinal: true,
|
pendingFinal: true,
|
||||||
|
pendingToolImages: toolFiles.length > 0
|
||||||
|
? [...s.pendingToolImages, ...toolFiles]
|
||||||
|
: s.pendingToolImages,
|
||||||
streamingTools: updates.length > 0 ? upsertToolStatuses(s.streamingTools, updates) : s.streamingTools,
|
streamingTools: updates.length > 0 ? upsertToolStatuses(s.streamingTools, updates) : s.streamingTools,
|
||||||
}));
|
};
|
||||||
|
});
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
const toolOnly = isToolOnlyMessage(finalMsg);
|
const toolOnly = isToolOnlyMessage(finalMsg);
|
||||||
@@ -853,15 +1160,28 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
set((s) => {
|
set((s) => {
|
||||||
const nextTools = updates.length > 0 ? upsertToolStatuses(s.streamingTools, updates) : s.streamingTools;
|
const nextTools = updates.length > 0 ? upsertToolStatuses(s.streamingTools, updates) : s.streamingTools;
|
||||||
const streamingTools = hasOutput ? [] : nextTools;
|
const streamingTools = hasOutput ? [] : nextTools;
|
||||||
|
|
||||||
|
// Attach any images collected from preceding tool results
|
||||||
|
const pendingImgs = s.pendingToolImages;
|
||||||
|
const msgWithImages: RawMessage = pendingImgs.length > 0
|
||||||
|
? {
|
||||||
|
...finalMsg,
|
||||||
|
role: (finalMsg.role || 'assistant') as RawMessage['role'],
|
||||||
|
id: msgId,
|
||||||
|
_attachedFiles: [...(finalMsg._attachedFiles || []), ...pendingImgs],
|
||||||
|
}
|
||||||
|
: { ...finalMsg, role: (finalMsg.role || 'assistant') as RawMessage['role'], id: msgId };
|
||||||
|
const clearPendingImages = { pendingToolImages: [] as AttachedFileMeta[] };
|
||||||
|
|
||||||
// Check if message already exists (prevent duplicates)
|
// Check if message already exists (prevent duplicates)
|
||||||
const alreadyExists = s.messages.some(m => m.id === msgId);
|
const alreadyExists = s.messages.some(m => m.id === msgId);
|
||||||
if (alreadyExists) {
|
if (alreadyExists) {
|
||||||
// Just clear streaming state, don't add duplicate
|
|
||||||
return toolOnly ? {
|
return toolOnly ? {
|
||||||
streamingText: '',
|
streamingText: '',
|
||||||
streamingMessage: null,
|
streamingMessage: null,
|
||||||
pendingFinal: true,
|
pendingFinal: true,
|
||||||
streamingTools,
|
streamingTools,
|
||||||
|
...clearPendingImages,
|
||||||
} : {
|
} : {
|
||||||
streamingText: '',
|
streamingText: '',
|
||||||
streamingMessage: null,
|
streamingMessage: null,
|
||||||
@@ -869,32 +1189,32 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
activeRunId: hasOutput ? null : s.activeRunId,
|
activeRunId: hasOutput ? null : s.activeRunId,
|
||||||
pendingFinal: hasOutput ? false : true,
|
pendingFinal: hasOutput ? false : true,
|
||||||
streamingTools,
|
streamingTools,
|
||||||
|
...clearPendingImages,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return toolOnly ? {
|
return toolOnly ? {
|
||||||
messages: [...s.messages, {
|
messages: [...s.messages, msgWithImages],
|
||||||
...finalMsg,
|
|
||||||
role: finalMsg.role || 'assistant',
|
|
||||||
id: msgId,
|
|
||||||
}],
|
|
||||||
streamingText: '',
|
streamingText: '',
|
||||||
streamingMessage: null,
|
streamingMessage: null,
|
||||||
pendingFinal: true,
|
pendingFinal: true,
|
||||||
streamingTools,
|
streamingTools,
|
||||||
|
...clearPendingImages,
|
||||||
} : {
|
} : {
|
||||||
messages: [...s.messages, {
|
messages: [...s.messages, msgWithImages],
|
||||||
...finalMsg,
|
|
||||||
role: finalMsg.role || 'assistant',
|
|
||||||
id: msgId,
|
|
||||||
}],
|
|
||||||
streamingText: '',
|
streamingText: '',
|
||||||
streamingMessage: null,
|
streamingMessage: null,
|
||||||
sending: hasOutput ? false : s.sending,
|
sending: hasOutput ? false : s.sending,
|
||||||
activeRunId: hasOutput ? null : s.activeRunId,
|
activeRunId: hasOutput ? null : s.activeRunId,
|
||||||
pendingFinal: hasOutput ? false : true,
|
pendingFinal: hasOutput ? false : true,
|
||||||
streamingTools,
|
streamingTools,
|
||||||
|
...clearPendingImages,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
// After the final response, quietly reload history to surface all intermediate
|
||||||
|
// tool-use turns (thinking + tool blocks) from the Gateway's authoritative record.
|
||||||
|
if (hasOutput && !toolOnly) {
|
||||||
|
void get().loadHistory(true);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
// No message in final event - reload history to get complete data
|
// No message in final event - reload history to get complete data
|
||||||
set({ streamingText: '', streamingMessage: null, pendingFinal: true });
|
set({ streamingText: '', streamingMessage: null, pendingFinal: true });
|
||||||
@@ -913,6 +1233,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
streamingTools: [],
|
streamingTools: [],
|
||||||
pendingFinal: false,
|
pendingFinal: false,
|
||||||
lastUserMessageAt: null,
|
lastUserMessageAt: null,
|
||||||
|
pendingToolImages: [],
|
||||||
});
|
});
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -925,6 +1246,7 @@ export const useChatStore = create<ChatState>((set, get) => ({
|
|||||||
streamingTools: [],
|
streamingTools: [],
|
||||||
pendingFinal: false,
|
pendingFinal: false,
|
||||||
lastUserMessageAt: null,
|
lastUserMessageAt: null,
|
||||||
|
pendingToolImages: [],
|
||||||
});
|
});
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user