From 5ce76a12e2a95ec1f752a503ff4b3f5655e3b601 Mon Sep 17 00:00:00 2001 From: Federico Jaramillo Martinez Date: Tue, 14 Jul 2026 08:48:48 +0200 Subject: [PATCH] fix: show live tool result images --- src/client/src/chatTranscript.test.ts | 84 +++++++++++++++++++++++++++ src/client/src/chatTranscript.ts | 67 +++++++++++++++++---- 2 files changed, 140 insertions(+), 11 deletions(-) diff --git a/src/client/src/chatTranscript.test.ts b/src/client/src/chatTranscript.test.ts index 09c349b..34c2249 100644 --- a/src/client/src/chatTranscript.test.ts +++ b/src/client/src/chatTranscript.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { groupChatMessages } from "./chatGroups"; import { textMessage } from "./chatMessages"; import { applyTranscriptEvent } from "./chatTranscript"; import type { ChatLine } from "./components/shared"; @@ -169,6 +170,89 @@ describe("applyTranscriptEvent", () => { ]); }); + it("projects live tool-result images and reconciles final content and metadata", () => { + const provisionalImage = { type: "image" as const, mimeType: "image/png", data: "UFJFVklFVw==" }; + const finalImage = { type: "image" as const, mimeType: "image/png", data: "RklOQUw=" }; + const finalContent = [{ type: "text", text: "Read image file [image/png]" }, finalImage]; + const timestamp = "2026-07-13T22:00:00.000Z"; + let messages: ChatLine[] = []; + + messages = applyTranscriptEvent(messages, { type: "tool.start", toolName: "read", toolCallId: "read-image-1", summary: "image.png", args: { path: "image.png" } }) ?? messages; + messages = applyTranscriptEvent(messages, { + type: "tool.end", + toolName: "read", + toolCallId: "read-image-1", + text: "Read image file [image/png]", + isError: false, + content: [{ type: "text", text: "Read image file [image/png]" }, provisionalImage], + details: { source: "tool.end" }, + }) ?? messages; + + expect(messages[0]?.parts.filter((part) => part.type === "image")).toEqual([provisionalImage]); + + messages = applyTranscriptEvent(messages, { + type: "message.end", + message: { + role: "toolResult", + toolCallId: "read-image-1", + toolName: "read", + content: finalContent, + details: { source: "message.end" }, + isError: false, + timestamp, + }, + }) ?? messages; + messages = applyTranscriptEvent(messages, { type: "assistant.delta", text: "done" }) ?? messages; + + const finalizedToolLine: ChatLine = { + role: "tool", + parts: [{ + type: "toolExecution", + toolCallId: "read-image-1", + toolName: "read", + summary: "image.png", + args: { path: "image.png" }, + status: "success", + resultText: "Read image file [image/png]", + content: finalContent, + details: { source: "message.end" }, + }, finalImage], + meta: { timestamp }, + }; + expect(messages).toEqual([finalizedToolLine, textMessage("assistant", "done")]); + expect(groupChatMessages(messages)).toEqual([ + { kind: "group", startIndex: 0, endIndex: 0, messages: [{ ...finalizedToolLine, parts: [finalizedToolLine.parts[0]] }] }, + { kind: "message", index: 0, message: { ...finalizedToolLine, parts: [finalImage] } }, + { kind: "message", index: 1, message: textMessage("assistant", "done") }, + ]); + }); + + it("keeps image-only live tool results visible without inventing text", () => { + const image = { type: "image" as const, mimeType: "image/webp", data: "QUJD" }; + let messages: ChatLine[] = []; + + messages = applyTranscriptEvent(messages, { type: "tool.start", toolName: "capture", toolCallId: "capture-1", summary: "screenshot" }) ?? messages; + messages = applyTranscriptEvent(messages, { type: "tool.end", toolName: "capture", toolCallId: "capture-1", text: "", isError: false, content: [image] }) ?? messages; + messages = applyTranscriptEvent(messages, { + type: "message.end", + message: { role: "toolResult", toolCallId: "capture-1", toolName: "capture", content: [image], isError: false }, + }) ?? messages; + + expect(messages).toEqual([{ + role: "tool", + parts: [{ + type: "toolExecution", + toolCallId: "capture-1", + toolName: "capture", + summary: "screenshot", + status: "success", + resultText: "", + content: [image], + }, image], + }]); + expect(groupChatMessages(messages).map((group) => group.kind)).toEqual(["group", "message"]); + }); + it("does not merge consecutive streamed skill reads", () => { let messages: ChatLine[] = []; messages = applyTranscriptEvent(messages, { type: "tool.start", toolName: "read", toolCallId: "1", summary: "", args: { path: "/skills/playwright/SKILL.md" } }) ?? messages; diff --git a/src/client/src/chatTranscript.ts b/src/client/src/chatTranscript.ts index bad38d5..179b84d 100644 --- a/src/client/src/chatTranscript.ts +++ b/src/client/src/chatTranscript.ts @@ -3,13 +3,40 @@ import type { ChatLine, ToolExecutionPart } from "./components/shared"; import { appendShellChunk, finalizeShellMessage, shellStartMessage } from "./shellMessages"; import type { SessionUiEvent } from "./sessionSocket"; +type ToolResultImage = Extract; + +interface ToolResultPresentation { + images: ToolResultImage[]; + meta?: ChatLine["meta"]; +} + +interface ToolResultUpdate { + toolCallId?: string; + toolName: string; + text: string; + isError: boolean; + content: unknown; + details: unknown; + presentation: ToolResultPresentation; +} + export function applyTranscriptEvent(messages: ChatLine[], event: SessionUiEvent): ChatLine[] | undefined { if (event.type === "message.append") return appendNewMessage(messages, event.message); if (event.type === "assistant.delta") return appendText(messages, "assistant", event.text); if (event.type === "assistant.thinking.delta") return appendThinking(messages, event.text); if (event.type === "tool.start") return appendToolExecutionStart(messages, event); if (event.type === "tool.update") return updateToolExecution(messages, event.toolCallId, (part) => mergeToolExecutionUpdate(part, event)); - if (event.type === "tool.end") return finalizeToolExecution(messages, event.toolCallId, event.toolName, summarizeArgs(event.content), event.text, event.isError, event.content, event.details); + if (event.type === "tool.end") { + return finalizeToolExecution(messages, { + toolCallId: event.toolCallId, + toolName: event.toolName, + text: event.text, + isError: event.isError, + content: event.content, + details: event.details, + presentation: toolResultPresentation({ role: "toolResult", content: event.content }), + }); + } if (event.type === "shell.start") return [...messages, shellStartMessage(event.command, event.excludeFromContext)]; if (event.type === "shell.chunk") return appendShellChunk(messages, event.chunk); if (event.type === "shell.end") return finalizeShellMessage(messages, event); @@ -21,9 +48,7 @@ export function applyTranscriptEvent(messages: ChatLine[], event: SessionUiEvent function applyFinalMessage(messages: ChatLine[], rawMessage: unknown): ChatLine[] | undefined { const rawToolResult = toolResultFromRawMessage(rawMessage); - if (rawToolResult !== undefined) { - return finalizeToolExecution(messages, rawToolResult.toolCallId, rawToolResult.toolName, summarizeArgs(rawToolResult.content), rawToolResult.text, rawToolResult.isError, rawToolResult.content, rawToolResult.details); - } + if (rawToolResult !== undefined) return finalizeToolExecution(messages, rawToolResult); const ended = normalizeMessage(rawMessage); if (ended.length === 0) return undefined; @@ -85,7 +110,8 @@ function mergeToolExecutionUpdate(part: ToolExecutionPart, event: Extract { const preview = previewFromDetails(details) ?? part.preview; return { @@ -96,7 +122,7 @@ function finalizeToolExecution(messages: ChatLine[], toolCallId: string | undefi ...(details === undefined ? {} : { details }), ...(preview === undefined ? {} : { preview }), }; - }); + }, (line) => reconcileToolResultPresentation(line, presentation)); if (updated !== messages) return updated; const preview = previewFromDetails(details); @@ -104,17 +130,22 @@ function finalizeToolExecution(messages: ChatLine[], toolCallId: string | undefi type: "toolExecution", ...(toolCallId === undefined || toolCallId === "" ? {} : { toolCallId }), toolName, - summary: fallbackSummary, + summary: summarizeArgs(content), status: isError ? "error" : "success", resultText: text, ...(content === undefined ? {} : { content }), ...(details === undefined ? {} : { details }), ...(preview === undefined ? {} : { preview }), }; - return [...messages, { role: "tool", parts: [part] }]; + return [...messages, reconcileToolResultPresentation({ role: "tool", parts: [part] }, presentation)]; } -function updateToolExecution(messages: ChatLine[], toolCallId: string | undefined, update: (part: ToolExecutionPart) => ToolExecutionPart): ChatLine[] { +function updateToolExecution( + messages: ChatLine[], + toolCallId: string | undefined, + update: (part: ToolExecutionPart) => ToolExecutionPart, + reconcileLine: (line: ChatLine) => ChatLine = (line) => line, +): ChatLine[] { if (toolCallId === undefined || toolCallId === "") return messages; for (let lineIndex = messages.length - 1; lineIndex >= 0; lineIndex--) { const line = messages[lineIndex]; @@ -123,13 +154,26 @@ function updateToolExecution(messages: ChatLine[], toolCallId: string | undefine if (partIndex < 0) continue; const part = line.parts[partIndex]; if (part?.type !== "toolExecution") continue; - const nextLine = { ...line, parts: [...line.parts.slice(0, partIndex), update(part), ...line.parts.slice(partIndex + 1)] }; + const updatedLine = { ...line, parts: [...line.parts.slice(0, partIndex), update(part), ...line.parts.slice(partIndex + 1)] }; + const nextLine = reconcileLine(updatedLine); return [...messages.slice(0, lineIndex), nextLine, ...messages.slice(lineIndex + 1)]; } return messages; } -function toolResultFromRawMessage(message: unknown): { toolCallId?: string; toolName: string; text: string; isError: boolean; content: unknown; details: unknown } | undefined { +function reconcileToolResultPresentation(line: ChatLine, presentation: ToolResultPresentation): ChatLine { + const next = { ...line, parts: [...line.parts.filter((part) => part.type !== "image"), ...presentation.images] }; + return presentation.meta === undefined ? next : { ...next, meta: presentation.meta }; +} + +function toolResultPresentation(message: unknown): ToolResultPresentation { + const normalized = normalizeMessage(message); + const images = normalized.flatMap((line) => line.parts.filter((part): part is ToolResultImage => part.type === "image")); + const meta = normalized.find((line) => line.meta !== undefined)?.meta; + return { images, ...(meta === undefined ? {} : { meta }) }; +} + +function toolResultFromRawMessage(message: unknown): ToolResultUpdate | undefined { if (getString(message, "role") !== "toolResult") return undefined; const toolCallId = getString(message, "toolCallId"); const content = getProperty(message, "content"); @@ -140,6 +184,7 @@ function toolResultFromRawMessage(message: unknown): { toolCallId?: string; tool isError: getBoolean(message, "isError") === true, content, details: getProperty(message, "details"), + presentation: toolResultPresentation(message), }; }