From eae5c765a89af692167e83fd2d1bdd0ff493bcbc Mon Sep 17 00:00:00 2001 From: Ignacio Jimenez Rocabado Date: Thu, 3 Sep 2026 16:30:01 -0700 Subject: [PATCH 1/7] fix(evals): format JSON tool results --- .changeset/format-eval-json-results.md | 6 + chat-ui/src/internal/trace-adapter.ts | 47 +++++-- .../__tests__/trace-viewer-adapter.test.ts | 115 +++++++++++++++++- 3 files changed, 153 insertions(+), 15 deletions(-) create mode 100644 .changeset/format-eval-json-results.md diff --git a/.changeset/format-eval-json-results.md b/.changeset/format-eval-json-results.md new file mode 100644 index 0000000000..3ce52a890c --- /dev/null +++ b/.changeset/format-eval-json-results.md @@ -0,0 +1,6 @@ +--- +"@mcpjam/chat-ui": patch +"@mcpjam/inspector": patch +--- + +Render JSON tool results in eval transcripts with the structured JSON viewer instead of raw minified text. diff --git a/chat-ui/src/internal/trace-adapter.ts b/chat-ui/src/internal/trace-adapter.ts index c92ececfec..b9ee08e43b 100644 --- a/chat-ui/src/internal/trace-adapter.ts +++ b/chat-ui/src/internal/trace-adapter.ts @@ -15,7 +15,10 @@ function isUIResourceShape(value: unknown): boolean { return false; } import { buildPersistedExecutionReplay } from "./persisted-execution-replay"; -import { extractTextFromToolResult } from "./tool-result-text"; +import { + extractDisplayFromToolResult, + extractTextFromToolResult, +} from "./tool-result-text"; import { getToolServerId } from "./tool-server"; import { detectUIType, getUIResourceUri, UIType } from "./widget-detection"; import { @@ -213,6 +216,14 @@ export interface AdaptedTraceResult { type ToolResultDisplay = "sibling-text" | "attached-to-tool"; type TraceDisplayMode = "markdown" | "json-markdown"; +type TraceDisplayAttachment = + | { kind: "text"; text: string; mode: "markdown" } + | { + kind: "json"; + value: unknown; + text: string; + mode: "json-markdown"; + }; interface TraceToolResultEntry { part: TraceContentPart; @@ -556,14 +567,23 @@ function getTraceDisplayAttachment(params: { displayedOutput: unknown; adaptedOutput: unknown; canReplayWidget: boolean; -}): { text: string; mode: TraceDisplayMode } | null { - const extractedText = extractTextFromToolResult(params.displayedOutput); - if (extractedText) { +}): TraceDisplayAttachment | null { + const display = extractDisplayFromToolResult(params.displayedOutput); + if (display?.kind === "text") { return { - text: extractedText, + kind: "text", + text: display.text, mode: "markdown", }; } + if (display?.kind === "json") { + return { + kind: "json", + value: display.value, + text: toMarkdownJson(display.value) ?? "", + mode: "json-markdown", + }; + } if (params.canReplayWidget) { return null; @@ -575,6 +595,8 @@ function getTraceDisplayAttachment(params: { } return { + kind: "json", + value: params.adaptedOutput, text: jsonMarkdown, mode: "json-markdown", }; @@ -711,10 +733,17 @@ function buildToolParts(params: { } if (traceDisplayAttachment) { - parts.push({ - type: "text", - text: traceDisplayAttachment.text, - }); + if (traceDisplayAttachment.kind === "json") { + parts.push({ + type: "data-result", + data: traceDisplayAttachment.value, + } as any); + } else { + parts.push({ + type: "text", + text: traceDisplayAttachment.text, + }); + } } return parts; diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts index 2db4b26ae0..ebe662096c 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts +++ b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts @@ -544,7 +544,62 @@ describe("adaptTraceToUiMessages", () => { ); }); - it("appends fenced JSON when tool output has only structured data", () => { + it("renders JSON text-block output as a structured result", () => { + const trace: TraceEnvelope = { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call-json-text", + toolName: "list_searches", + input: {}, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-json-text", + toolName: "list_searches", + output: { + type: "json", + value: { + content: [ + { + type: "text", + text: '{"searches":[{"id":"admissions-community"}],"nextCursor":null}', + }, + ], + }, + }, + }, + ], + }, + ], + }; + + const result = adaptTraceToUiMessages({ trace }); + expect(result.messages[0].parts).toContainEqual({ + type: "data-result", + data: { + searches: [{ id: "admissions-community" }], + nextCursor: null, + }, + }); + expect( + result.messages[0].parts.some( + (part) => + part.type === "text" && + (part as { text?: string }).text?.startsWith('{"searches"'), + ), + ).toBe(false); + }); + + it("renders structured tool output as a structured result", () => { const trace: TraceEnvelope = { messages: [ { @@ -576,11 +631,59 @@ describe("adaptTraceToUiMessages", () => { }; const result = adaptTraceToUiMessages({ trace }); - const textParts = result.messages[0].parts.filter((p) => p.type === "text"); - const jsonFallback = textParts.find((p) => - (p as any).text?.startsWith("```json"), - ); - expect(jsonFallback).toBeDefined(); + expect(result.messages[0].parts).toContainEqual({ + type: "data-result", + data: { count: 5, items: ["a", "b"] }, + }); + }); + + it("formats JSON text output alongside a replayed widget", () => { + const trace: TraceEnvelope = { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call-widget-json", + toolName: "create_view", + input: {}, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-widget-json", + toolName: "create_view", + result: { + content: [ + { + type: "text", + text: '{"team":"Barcelona","teams":["Arsenal","Barcelona"]}', + }, + ], + }, + }, + ], + }, + ], + widgetSnapshots: [ + makeWidgetSnapshot({ + toolCallId: "call-widget-json", + widgetHtmlUrl: "https://storage.example.com/widget.html", + }), + ], + }; + + const result = adaptTraceToUiMessages({ trace }); + expect(result.toolRenderOverrides["call-widget-json"]).toBeDefined(); + expect(result.messages[0].parts).toContainEqual({ + type: "data-result", + data: { team: "Barcelona", teams: ["Arsenal", "Barcelona"] }, + }); }); // --- Test 9: Reasoning parts with no state --- From 8a210a50226d58b1b5ef207c5b3079d58db92780 Mon Sep 17 00:00:00 2001 From: Ignacio Jimenez Rocabado Date: Fri, 4 Sep 2026 16:02:31 -0700 Subject: [PATCH 2/7] Show full tool cards in eval transcripts --- .changeset/format-eval-json-results.md | 2 +- .../evals/__tests__/trace-viewer.test.tsx | 25 ++++++++++++++++--- .../src/components/evals/trace-viewer.tsx | 3 ++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.changeset/format-eval-json-results.md b/.changeset/format-eval-json-results.md index 3ce52a890c..a5c4613f69 100644 --- a/.changeset/format-eval-json-results.md +++ b/.changeset/format-eval-json-results.md @@ -3,4 +3,4 @@ "@mcpjam/inspector": patch --- -Render JSON tool results in eval transcripts with the structured JSON viewer instead of raw minified text. +Render eval tool results inside the full shared tool card, including structured JSON and widget diagnostics, instead of raw sibling text. diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer.test.tsx b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer.test.tsx index d59550d036..966231943e 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer.test.tsx +++ b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer.test.tsx @@ -1225,18 +1225,33 @@ describe("TraceViewer", () => { // --- Props pass-through --- - it("passes minimalMode={true} and interactive={false} to MessageView", () => { + it("uses the full read-only tool card in non-interactive traces", () => { render(); openChatTab(); expect(mockMessageView).toHaveBeenCalledWith( expect.objectContaining({ - minimalMode: true, + minimalMode: false, interactive: false, + showInlineEdit: false, }), ); }); + it("keeps the tool result inside the shared tool card", () => { + render(); + openChatTab(); + + const message = mockMessageView.mock.calls[0][0].message as { + parts: Array>; + }; + expect(message.parts).toHaveLength(1); + expect(message.parts[0]).toMatchObject({ + type: "dynamic-tool", + traceDisplayMode: "json-markdown", + }); + }); + it("requests collapsed reasoning rendering in chat trace mode", () => { render(); openChatTab(); @@ -1418,7 +1433,8 @@ describe("TraceViewer", () => { const lastCall = mockMessageView.mock.calls[0][0]; expect(lastCall.interactive).toBe(false); - expect(lastCall.minimalMode).toBe(true); + expect(lastCall.minimalMode).toBe(false); + expect(lastCall.showInlineEdit).toBe(false); }); it("keeps trace chat read-only when only onFullscreenChange is provided", () => { @@ -1429,7 +1445,8 @@ describe("TraceViewer", () => { const lastCall = mockMessageView.mock.calls[0][0]; expect(lastCall.interactive).toBe(false); - expect(lastCall.minimalMode).toBe(true); + expect(lastCall.minimalMode).toBe(false); + expect(lastCall.showInlineEdit).toBe(false); }); }); diff --git a/mcpjam-inspector/client/src/components/evals/trace-viewer.tsx b/mcpjam-inspector/client/src/components/evals/trace-viewer.tsx index 76375fa111..31c7487af8 100644 --- a/mcpjam-inspector/client/src/components/evals/trace-viewer.tsx +++ b/mcpjam-inspector/client/src/components/evals/trace-viewer.tsx @@ -486,6 +486,7 @@ export function TraceViewer({ toolsMetadata, toolServerMap, connectedServerIds, + toolResultDisplay: "attached-to-tool", }), [trace, toolsMetadata, toolServerMap, connectedServerIds] ); @@ -973,7 +974,7 @@ export function TraceViewer({ toolRenderOverrides={toolRenderOverrides} appToolInvocationsOverride={appToolInvocationsOverride} showInlineEdit={false} - minimalMode={true} + minimalMode={false} interactive={threadInteractive} recorder={recorder} reasoningDisplayMode="collapsed" From 216482ec4bb83b1751a20a2cae49c8e9fb76fca5 Mon Sep 17 00:00:00 2001 From: Ignacio Jimenez Rocabado Date: Fri, 4 Sep 2026 16:16:57 -0700 Subject: [PATCH 3/7] Use shared tool result UI in evals --- .changeset/format-eval-json-results.md | 2 +- chat-ui/src/internal/trace-adapter.ts | 7 +++- .../__tests__/trace-viewer-adapter.test.ts | 42 +++++++++++++++++++ .../evals/__tests__/trace-viewer.test.tsx | 3 +- .../components/evals/trace-viewer-adapter.ts | 2 +- .../src/components/evals/trace-viewer.tsx | 2 +- 6 files changed, 52 insertions(+), 6 deletions(-) diff --git a/.changeset/format-eval-json-results.md b/.changeset/format-eval-json-results.md index a5c4613f69..0f55df9862 100644 --- a/.changeset/format-eval-json-results.md +++ b/.changeset/format-eval-json-results.md @@ -3,4 +3,4 @@ "@mcpjam/inspector": patch --- -Render eval tool results inside the full shared tool card, including structured JSON and widget diagnostics, instead of raw sibling text. +Render eval tool calls with the full shared playground card, including its Input, Result, widget controls, and diagnostics, without a duplicate sibling result. diff --git a/chat-ui/src/internal/trace-adapter.ts b/chat-ui/src/internal/trace-adapter.ts index b9ee08e43b..0e7683eb78 100644 --- a/chat-ui/src/internal/trace-adapter.ts +++ b/chat-ui/src/internal/trace-adapter.ts @@ -214,7 +214,7 @@ export interface AdaptedTraceResult { sourceMessageIndexToFocusUiMessageId: Record; } -type ToolResultDisplay = "sibling-text" | "attached-to-tool"; +type ToolResultDisplay = "sibling-text" | "attached-to-tool" | "tool-card"; type TraceDisplayMode = "markdown" | "json-markdown"; type TraceDisplayAttachment = | { kind: "text"; text: string; mode: "markdown" } @@ -728,7 +728,10 @@ function buildToolParts(params: { return parts; } - if (params.toolResultDisplay === "attached-to-tool") { + if ( + params.toolResultDisplay === "attached-to-tool" || + params.toolResultDisplay === "tool-card" + ) { return parts; } diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts index ebe662096c..93eb6f0ac9 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts +++ b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts @@ -186,6 +186,48 @@ describe("adaptTraceToUiMessages", () => { ); }); + it("keeps the raw result in the shared tool card without a sibling", () => { + const trace: TraceEnvelope = { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call-1", + toolName: "read_me", + input: { id: 42 }, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-1", + toolName: "read_me", + output: { type: "json", value: { hello: "world" } }, + }, + ], + }, + ], + }; + + const result = adaptTraceToUiMessages({ + trace, + toolResultDisplay: "tool-card", + }); + + expect(result.messages[0].parts).toHaveLength(1); + expect(result.messages[0].parts[0]).toMatchObject({ + type: "dynamic-tool", + input: { id: 42 }, + output: { hello: "world" }, + }); + expect(result.messages[0].parts[0]).not.toHaveProperty("traceDisplayMode"); + }); + // --- Test 2: Multiple tool calls --- it("groups multiple tool-calls and results into a single assistant UIMessage", () => { const trace: TraceEnvelope = { diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer.test.tsx b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer.test.tsx index 966231943e..7830c954df 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer.test.tsx +++ b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer.test.tsx @@ -1248,8 +1248,9 @@ describe("TraceViewer", () => { expect(message.parts).toHaveLength(1); expect(message.parts[0]).toMatchObject({ type: "dynamic-tool", - traceDisplayMode: "json-markdown", + output: { ok: true }, }); + expect(message.parts[0]).not.toHaveProperty("traceDisplayMode"); }); it("requests collapsed reasoning rendering in chat trace mode", () => { diff --git a/mcpjam-inspector/client/src/components/evals/trace-viewer-adapter.ts b/mcpjam-inspector/client/src/components/evals/trace-viewer-adapter.ts index c0f4656d0b..6230b5ee57 100644 --- a/mcpjam-inspector/client/src/components/evals/trace-viewer-adapter.ts +++ b/mcpjam-inspector/client/src/components/evals/trace-viewer-adapter.ts @@ -35,7 +35,7 @@ export type { TraceWidgetSnapshot, } from "@mcpjam/chat-ui/trace"; -type ToolResultDisplay = "sibling-text" | "attached-to-tool"; +type ToolResultDisplay = "sibling-text" | "attached-to-tool" | "tool-card"; /** * Inspector trace envelope: the package's structural trace input plus diff --git a/mcpjam-inspector/client/src/components/evals/trace-viewer.tsx b/mcpjam-inspector/client/src/components/evals/trace-viewer.tsx index 31c7487af8..3f189a5c97 100644 --- a/mcpjam-inspector/client/src/components/evals/trace-viewer.tsx +++ b/mcpjam-inspector/client/src/components/evals/trace-viewer.tsx @@ -486,7 +486,7 @@ export function TraceViewer({ toolsMetadata, toolServerMap, connectedServerIds, - toolResultDisplay: "attached-to-tool", + toolResultDisplay: "tool-card", }), [trace, toolsMetadata, toolServerMap, connectedServerIds] ); From 5dd197159f4e96513ebbaafcaed30ba8b7203796 Mon Sep 17 00:00:00 2001 From: Ignacio Jimenez Rocabado Date: Sat, 5 Sep 2026 10:38:14 -0700 Subject: [PATCH 4/7] Finish recorded eval tool cards --- .changeset/format-eval-json-results.md | 2 +- chat-ui/src/internal/trace-adapter.ts | 24 ++- .../chat-v2/__tests__/PartSwitch.test.tsx | 180 +++++++++++++----- .../components/chat-v2/thread/part-switch.tsx | 67 +++++-- .../parts/__tests__/display-modes.test.tsx | 36 ++++ .../chat-v2/thread/parts/tool-part.tsx | 134 ++++++++++--- .../__tests__/trace-viewer-adapter.test.ts | 94 ++++++++- 7 files changed, 427 insertions(+), 110 deletions(-) diff --git a/.changeset/format-eval-json-results.md b/.changeset/format-eval-json-results.md index 0f55df9862..be54511fbc 100644 --- a/.changeset/format-eval-json-results.md +++ b/.changeset/format-eval-json-results.md @@ -3,4 +3,4 @@ "@mcpjam/inspector": patch --- -Render eval tool calls with the full shared playground card, including its Input, Result, widget controls, and diagnostics, without a duplicate sibling result. +Render eval tool calls with the shared playground card, including Input, Result, and truthful recorded widget diagnostics, without duplicate result rows or live-only controls on frozen screenshots. diff --git a/chat-ui/src/internal/trace-adapter.ts b/chat-ui/src/internal/trace-adapter.ts index 0e7683eb78..fb95662759 100644 --- a/chat-ui/src/internal/trace-adapter.ts +++ b/chat-ui/src/internal/trace-adapter.ts @@ -132,8 +132,7 @@ export function snapshotsToTraceWidgetSnapshots( widgetHtmlUrl: snap.widgetHtmlUrl, toolOutput: snap.toolOutput, injectedOpenAiCompat: snap.injectedOpenAiCompat, - injectedOpenAiCompatCapabilities: - snap.injectedOpenAiCompatCapabilities, + injectedOpenAiCompatCapabilities: snap.injectedOpenAiCompatCapabilities, }; }); } @@ -179,8 +178,7 @@ export function buildToolRenderOverridesFromSnapshots( widgetPermissive: snap.widgetPermissive, prefersBorder: snap.prefersBorder, injectedOpenAiCompat: snap.injectedOpenAiCompat, - injectedOpenAiCompatCapabilities: - snap.injectedOpenAiCompatCapabilities, + injectedOpenAiCompatCapabilities: snap.injectedOpenAiCompatCapabilities, }); overrides[snap.toolCallId] = replay.renderOverride; } @@ -542,8 +540,7 @@ function createReplayOverride( widgetPermissive: snapshot.widgetPermissive, prefersBorder: snapshot.prefersBorder, injectedOpenAiCompat: snapshot.injectedOpenAiCompat, - injectedOpenAiCompatCapabilities: - snapshot.injectedOpenAiCompatCapabilities, + injectedOpenAiCompatCapabilities: snapshot.injectedOpenAiCompatCapabilities, }).renderOverride; } @@ -558,8 +555,7 @@ function createLiveSnapshotOverride(snapshot: TraceWidgetSnapshot) { widgetPermissive: snapshot.widgetPermissive, prefersBorder: snapshot.prefersBorder, injectedOpenAiCompat: snapshot.injectedOpenAiCompat, - injectedOpenAiCompatCapabilities: - snapshot.injectedOpenAiCompatCapabilities, + injectedOpenAiCompatCapabilities: snapshot.injectedOpenAiCompatCapabilities, } satisfies ToolRenderOverride; } @@ -670,7 +666,7 @@ function buildToolParts(params: { } const traceDisplayAttachment = - !isError && params.matchedResult + params.toolResultDisplay !== "tool-card" && !isError && params.matchedResult ? getTraceDisplayAttachment({ displayedOutput, adaptedOutput, @@ -717,10 +713,12 @@ function buildToolParts(params: { parts.push(toolPart); if (isError) { - parts.push({ - type: "text", - text: `Tool error: ${errorText}`, - }); + if (params.toolResultDisplay !== "tool-card") { + parts.push({ + type: "text", + text: `Tool error: ${errorText}`, + }); + } return parts; } diff --git a/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx b/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx index ab4a7a8c1c..fdc3cded6f 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx @@ -5,8 +5,9 @@ import { PartSwitch } from "../thread/part-switch"; import { ActiveHostCapsResolverProvider } from "@/contexts/active-host-client-capabilities-context"; import type { UIMessage } from "@ai-sdk/react"; -const { mockDetectUIType, mockWidgetReplay } = vi.hoisted(() => ({ +const { mockDetectUIType, mockToolPart, mockWidgetReplay } = vi.hoisted(() => ({ mockDetectUIType: vi.fn(), + mockToolPart: vi.fn(), mockWidgetReplay: vi.fn(), })); @@ -20,23 +21,18 @@ vi.mock("../thread/parts/text-part", () => ({ })); vi.mock("../thread/parts/tool-part", () => ({ - ToolPart: ({ - part, - serverId, - rawOutput, - }: { - part: any; - serverId?: string; - rawOutput?: unknown; - }) => ( -
- {part.toolName || "tool"} -
- ), + ToolPart: (props: { part: any; serverId?: string; rawOutput?: unknown }) => { + mockToolPart(props); + return ( +
+ {props.part.toolName || "tool"} +
+ ); + }, })); vi.mock("../thread/parts/reasoning-part", () => ({ @@ -192,12 +188,12 @@ describe("PartSwitch", () => { const part = { type: "text", text: "Hello" }; render( - + , ); expect(screen.getByTestId("text-part")).toHaveAttribute( "data-role", - "assistant" + "assistant", ); }); }); @@ -214,7 +210,7 @@ describe("PartSwitch", () => { expect(screen.getByTestId("reasoning-part")).toBeInTheDocument(); expect(screen.getByTestId("reasoning-part")).toHaveTextContent( - "Thinking..." + "Thinking...", ); }); @@ -225,7 +221,7 @@ describe("PartSwitch", () => { expect(screen.getByTestId("reasoning-part")).toHaveAttribute( "data-state", - "done" + "done", ); }); @@ -241,12 +237,12 @@ describe("PartSwitch", () => { {...defaultProps} part={part as any} reasoningDisplayMode="collapsed" - /> + />, ); expect(screen.getByTestId("reasoning-part")).toHaveAttribute( "data-display-mode", - "collapsed" + "collapsed", ); }); @@ -262,12 +258,12 @@ describe("PartSwitch", () => { {...defaultProps} part={part as any} reasoningDisplayMode="collapsible" - /> + />, ); expect(screen.getByTestId("reasoning-part")).toHaveAttribute( "data-display-mode", - "collapsible" + "collapsible", ); }); }); @@ -290,7 +286,7 @@ describe("PartSwitch", () => { expect(screen.getByTestId("source-url-part")).toBeInTheDocument(); expect(screen.getByTestId("source-url-part")).toHaveTextContent( - "https://example.com" + "https://example.com", ); }); @@ -301,7 +297,7 @@ describe("PartSwitch", () => { expect(screen.getByTestId("source-document-part")).toBeInTheDocument(); expect(screen.getByTestId("source-document-part")).toHaveTextContent( - "Doc Title" + "Doc Title", ); }); }); @@ -311,7 +307,7 @@ describe("PartSwitch", () => { const part = { type: "step-start" }; const { container } = render( - + , ); expect(container.firstChild).toBeNull(); @@ -327,7 +323,7 @@ describe("PartSwitch", () => { expect(screen.getByTestId("json-part")).toBeInTheDocument(); expect(screen.getByTestId("json-part")).toHaveAttribute( "data-label", - "Unknown part" + "Unknown part", ); }); }); @@ -359,7 +355,7 @@ describe("PartSwitch", () => { part={part as any} toolsMetadata={{}} toolServerMap={{}} - /> + />, ); expect(screen.getByTestId("tool-part")).toBeInTheDocument(); @@ -392,12 +388,12 @@ describe("PartSwitch", () => { part={part as any} toolsMetadata={{}} toolServerMap={{}} - /> + />, ); expect(screen.getByTestId("tool-part")).toHaveAttribute( "data-server-id", - "qa-server" + "qa-server", ); }); @@ -434,12 +430,12 @@ describe("PartSwitch", () => { part={part as any} toolsMetadata={{}} toolServerMap={{}} - /> + />, ); expect(screen.getByTestId("tool-part")).toHaveAttribute( "data-raw-output", - JSON.stringify(rawResult) + JSON.stringify(rawResult), ); }); @@ -472,7 +468,7 @@ describe("PartSwitch", () => { }, }} /> - + , ); expect(screen.getByTestId("widget-replay")).toBeInTheDocument(); }); @@ -501,7 +497,7 @@ describe("PartSwitch", () => { }, }} /> - + , ); expect(screen.queryByTestId("widget-replay")).not.toBeInTheDocument(); expect(screen.getByTestId("tool-part")).toBeInTheDocument(); @@ -528,7 +524,7 @@ describe("PartSwitch", () => { ui: { resourceUri: "ui://widget/create-view.html" }, }, }} - /> + />, ); expect(screen.getByTestId("widget-replay")).toBeInTheDocument(); }); @@ -565,13 +561,13 @@ describe("PartSwitch", () => { }, }, }} - /> + />, ); expect(screen.getByTestId("widget-replay")).toBeInTheDocument(); expect(screen.getByTestId("widget-replay")).toHaveAttribute( "data-cached-url", - "https://storage.example.com/widget.html" + "https://storage.example.com/widget.html", ); expect(mockWidgetReplay).toHaveBeenCalledWith( expect.objectContaining({ @@ -579,7 +575,101 @@ describe("PartSwitch", () => { renderOverride: expect.objectContaining({ cachedWidgetHtmlUrl: "https://storage.example.com/widget.html", }), - }) + }), + ); + }); + + it("uses recorded diagnostics and hides live controls for frozen widgets", () => { + mockDetectUIType.mockReturnValue("mcp-apps"); + const part = { + type: "tool-invocation", + toolName: "create_view", + toolCallId: "call-frozen", + state: "output-available", + input: { title: "Flow" }, + output: { content: "saved" }, + }; + + render( + , + ); + + expect(screen.getByTestId("frozen-widget-replay")).toBeInTheDocument(); + expect(screen.queryByTestId("widget-replay")).not.toBeInTheDocument(); + expect(mockToolPart).toHaveBeenCalledWith( + expect.objectContaining({ + displayMode: undefined, + onDisplayModeChange: undefined, + allowInlineEdit: false, + recordedWidgetDiagnostics: { + resourceUri: "ui://widget/create-view.html", + csp: { connectDomains: ["https://api.example.com"] }, + permissions: { clipboardWrite: {} }, + permissive: false, + prefersBorder: true, + }, + }), + ); + }); + + it("keeps display modes, diagnostics, and edit enabled for live widgets", () => { + mockDetectUIType.mockReturnValue("mcp-apps"); + const onDisplayModeChange = vi.fn(); + const part = { + type: "tool-invocation", + toolName: "create_view", + toolCallId: "call-live", + state: "output-available", + input: { title: "Flow" }, + output: { content: "saved" }, + }; + + render( + , + ); + + expect(screen.getByTestId("widget-replay")).toBeInTheDocument(); + expect(mockToolPart).toHaveBeenCalledWith( + expect.objectContaining({ + displayMode: "inline", + onDisplayModeChange, + allowInlineEdit: true, + recordedWidgetDiagnostics: undefined, + }), ); }); @@ -612,7 +702,7 @@ describe("PartSwitch", () => { part={dismissedPart as any} toolsMetadata={widgetMetadata} tornDownWidgetIds={new Set(["call-1"])} - /> + />, ); expect(screen.queryByTestId("widget-replay")).not.toBeInTheDocument(); expect(screen.getByTestId("tool-part")).toBeInTheDocument(); @@ -626,7 +716,7 @@ describe("PartSwitch", () => { part={dismissedPart as any} toolsMetadata={widgetMetadata} tornDownWidgetIds={new Set(["other-call"])} - /> + />, ); expect(screen.getByTestId("widget-replay")).toBeInTheDocument(); }); @@ -640,10 +730,10 @@ describe("PartSwitch", () => { part={dismissedPart as any} toolsMetadata={widgetMetadata} onRequestTeardown={handleTeardown} - /> + />, ); expect(mockWidgetReplay).toHaveBeenCalledWith( - expect.objectContaining({ onRequestTeardown: handleTeardown }) + expect.objectContaining({ onRequestTeardown: handleTeardown }), ); }); }); diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx index 43a39bd271..fdf374395c 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx @@ -152,7 +152,7 @@ export function PartSwitch({ context: { content?: ContentBlock[]; structuredContent?: Record; - } + }, ) => void; onAppToolInvocationChange?: (invocation: AppToolInvocationUpdate) => void; pipWidgetId: string | null; @@ -214,18 +214,18 @@ export function PartSwitch({ ); const [isEditing, setIsEditing] = useState(false); const [editedInput, setEditedInput] = useState<{ value: unknown } | null>( - null + null, ); // Manual hand-edits to the Result JSON (null = none). const [editedOutput, setEditedOutput] = useState<{ value: unknown } | null>( - null + null, ); // The most recent server Run result, kept as the metadata anchor: it becomes // the base whose _meta / toolResponseMetadata applies to later manual edits, // so "Run, then tweak the result" keeps the latest run's metadata, not the // original tool result's. const [lastRunOutput, setLastRunOutput] = useState<{ value: unknown } | null>( - null + null, ); const [isRunning, setIsRunning] = useState(false); // Bumped to remount + reseed the JsonEditors on a hard reset (Revert / @@ -242,16 +242,16 @@ export function PartSwitch({ const handleInputChange = useCallback( (value: unknown) => setEditedInput({ value }), - [] + [], ); const handleOutputChange = useCallback( (value: unknown) => setEditedOutput({ value }), - [] + [], ); // The input editor reports its parse state on every keystroke (null = valid). const handleInputValidityChange = useCallback( (valid: boolean) => setInputInvalid(!valid), - [] + [], ); const handleToggleEdit = useCallback(() => setIsEditing((p) => !p), []); const handleRevert = useCallback(() => { @@ -346,10 +346,13 @@ export function PartSwitch({ Object.prototype.hasOwnProperty.call(renderOverride, "toolOutput"); const resolvedToolOutput = hasRenderOverrideToolOutput ? renderOverride.toolOutput - : toolInfo.output ?? toolInfo.rawOutput; + : (toolInfo.output ?? toolInfo.rawOutput); // --- Inline edit: effective values fed to BOTH the editors and the iframe --- - const baseInput = (toolInfo.input ?? null) as Record | null; + const baseInput = (toolInfo.input ?? null) as Record< + string, + unknown + > | null; // Tool input is an arguments object. Mirror the output normalization: ignore // non-object edits (null / array / string) for BOTH the live widget feed and // Run, falling back to the original — otherwise the preview could render one @@ -393,8 +396,7 @@ export function PartSwitch({ // swapping rawOutput would drop serverId on raw-result-resolved cards.) const toolResponseMetadataOverride = lastRunOutput ? (readToolResultMeta(lastRunOutput.value) as - | Record - | undefined) + Record | undefined) : undefined; const hasEdits = editedInput !== null || editedOutput !== null || lastRunOutput !== null; @@ -503,22 +505,36 @@ export function PartSwitch({ // `uiType` checks at view-time, but we still have its capture. The inner // ternary below shows the screenshot in place of the live . if (widgetSlotShouldRender(shouldRenderWidget, renderOverride)) { + const isFrozenWidget = !!renderOverride?.frozenScreenshotUrl; + const allowWidgetDisplayModeChanges = interactive && !isFrozenWidget; return ( <> {renderOverride?.frozenScreenshotUrl ? ( @@ -570,7 +597,7 @@ export function PartSwitch({ toolName: toolInfo.toolName, toolCallId: tcid, widgetPromptIndex, - } + }, ); recorderDebug("part record decision", { toolName: toolInfo.toolName, @@ -611,7 +638,7 @@ export function PartSwitch({ }); }, onReplayControllerReady: ( - replay: ReplayControllerEvent["replay"] + replay: ReplayControllerEvent["replay"], ) => { onReplayControllerReady?.({ promptIndex: pi, diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/__tests__/display-modes.test.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/__tests__/display-modes.test.tsx index 6459e68176..fef57eeb2c 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/__tests__/display-modes.test.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/__tests__/display-modes.test.tsx @@ -249,4 +249,40 @@ describe("ToolPart display mode controls", () => { screen.queryByLabelText("Edit input and output"), ).not.toBeInTheDocument(); }); + + it("shows recorded Data and CSP without live display or edit controls", async () => { + const user = userEvent.setup(); + render( + , + ); + + expect(screen.getByLabelText("Data")).toBeInTheDocument(); + expect(screen.getByLabelText("Sandbox")).toBeInTheDocument(); + expect(screen.queryByLabelText("Inline")).not.toBeInTheDocument(); + expect( + screen.queryByLabelText("Edit input and output"), + ).not.toBeInTheDocument(); + + await user.click(screen.getByText("test-tool")); + expect(screen.getByText("Recorded tool data")).toBeInTheDocument(); + expect(screen.getAllByTestId("json-editor")).toHaveLength(2); + + await user.click(screen.getByLabelText("Sandbox")); + expect(screen.getByTestId("recorded-widget-diagnostics")).toHaveTextContent( + "Saved with this eval run", + ); + expect(screen.getByTestId("json-editor")).toHaveTextContent( + "https://api.example.com", + ); + }); }); diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx index f3f5a90a75..4d1e3c58b7 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx @@ -66,6 +66,14 @@ import { McpToolResultImagePreviewGrid } from "@/components/chat-v2/shared/mcp-t type ApprovalVisualState = "pending" | "approved" | "denied"; type TraceDisplayMode = "markdown" | "json-markdown"; +export interface RecordedWidgetDiagnostics { + resourceUri?: string; + csp?: unknown; + permissions?: unknown; + permissive?: boolean; + prefersBorder?: boolean; +} + export function ToolPart({ part, chatSessionId, @@ -101,6 +109,7 @@ export function ToolPart({ serverId, mcpToolResultImageRendering, rawOutput, + recordedWidgetDiagnostics, }: { part: ToolUIPart | DynamicToolUIPart; chatSessionId?: string; @@ -152,6 +161,7 @@ export function ToolPart({ serverId?: string; mcpToolResultImageRendering?: McpToolResultImageRenderingPolicy; rawOutput?: unknown; + recordedWidgetDiagnostics?: RecordedWidgetDiagnostics; }) { const hasTrackedSkillLoad = useRef(false); @@ -204,7 +214,7 @@ export function ToolPart({ "data" | "state" | "sandbox" | "context" | null >("data"); const [resultImageMode, setResultImageMode] = useState<"images" | "raw">( - "images" + "images", ); const inputData = (part as any).input; @@ -225,7 +235,7 @@ export function ToolPart({ outputValue !== undefined ? outputValue : rawResultData; const imagePreviewData = rawResultData; const imageRenderPlacement = getMcpToolResultImageRenderPlacement( - mcpToolResultImageRendering + mcpToolResultImageRendering, ); const showInlineImagePreview = imageRenderPlacement === "inline"; const showPanelImagePreview = imageRenderPlacement === "collapsed"; @@ -233,7 +243,7 @@ export function ToolPart({ showInlineImagePreview || (showPanelImagePreview && isExpanded); const resultImageState = useMcpToolResultImagePreviews( canRenderToolImages ? imagePreviewData : undefined, - { serverId, renderingPolicy: mcpToolResultImageRendering } + { serverId, renderingPolicy: mcpToolResultImageRendering }, ); // Editors render the effective values (what the widget sees) when the parent // supplies them; fall back to the raw part data otherwise (non-widget branch). @@ -250,7 +260,7 @@ export function ToolPart({ .traceDisplayMode; const hasAttachedTraceDisplay = Boolean( traceDisplayText && - (traceDisplayMode === "markdown" || traceDisplayMode === "json-markdown") + (traceDisplayMode === "markdown" || traceDisplayMode === "json-markdown"), ); const hasInput = inputData !== undefined && inputData !== null; const paramCount = useMemo(() => { @@ -266,16 +276,34 @@ export function ToolPart({ const hasError = state === "output-error" && !!errorText; const showRawResult = hasOutput && !hasAttachedTraceDisplay; - const widgetDebugInfo = useWidgetDebugStore((s) => - toolCallId ? s.widgets.get(toolCallId) : undefined + const storedWidgetDebugInfo = useWidgetDebugStore((s) => + toolCallId ? s.widgets.get(toolCallId) : undefined, ); + // A completed eval can retain live store data from the streaming phase. + // Once its widget becomes a frozen screenshot, the recorded snapshot is the + // source of truth and stale live diagnostics must not leak into the card. + const widgetDebugInfo = recordedWidgetDiagnostics + ? undefined + : storedWidgetDebugInfo; const hostContext = useHostContextStore((s) => s.draftHostContext); const hostAvailableDisplayModes = useMemo( () => extractHostDisplayModes(hostContext), - [hostContext] + [hostContext], ); const hasWidgetDebug = !!widgetDebugInfo; - const hasWidgetDebugUI = !hideDiagnosticsUI && hasWidgetDebug; + const hasRecordedWidgetDebug = !!recordedWidgetDiagnostics; + const hasWidgetDebugData = hasWidgetDebug || hasRecordedWidgetDebug; + const hasWidgetDebugUI = !hideDiagnosticsUI && hasWidgetDebugData; + + useEffect(() => { + if ( + hasRecordedWidgetDebug && + activeDebugTab !== "data" && + activeDebugTab !== "sandbox" + ) { + setActiveDebugTab("data"); + } + }, [activeDebugTab, hasRecordedWidgetDebug]); const showDisplayModeControls = displayMode !== undefined && @@ -305,12 +333,16 @@ export function ToolPart({ badge?: number; }[] = [{ tab: "data", icon: Database, label: "Data" }]; - if (uiType === UIType.OPENAI_SDK) { + if (hasWidgetDebug && uiType === UIType.OPENAI_SDK) { options.push({ tab: "state", icon: Box, label: "Widget State" }); } // Add model context tab for MCP Apps - if (uiType === UIType.MCP_APPS && widgetDebugInfo?.modelContext) { + if ( + hasWidgetDebug && + uiType === UIType.MCP_APPS && + widgetDebugInfo?.modelContext + ) { options.push({ tab: "context", icon: MessageCircle, @@ -328,6 +360,7 @@ export function ToolPart({ return options; }, [ uiType, + hasWidgetDebug, widgetDebugInfo?.csp?.violations?.length, widgetDebugInfo?.modelContext, ]); @@ -396,8 +429,8 @@ export function ToolPart({ isDisabled ? "text-muted-foreground/30 cursor-not-allowed" : isActive - ? "bg-background text-foreground shadow-sm cursor-pointer" - : "text-muted-foreground/60 hover:text-muted-foreground hover:bg-background/50 cursor-pointer" + ? "bg-background text-foreground shadow-sm cursor-pointer" + : "text-muted-foreground/60 hover:text-muted-foreground hover:bg-background/50 cursor-pointer" }`} > @@ -419,18 +452,18 @@ export function ToolPart({ tab === "data" ? "Data" : tab === "state" - ? "State" - : tab === "sandbox" - ? "Sandbox" - : "Context"; + ? "State" + : tab === "sandbox" + ? "Sandbox" + : "Context"; const tooltipLabel = tab === "data" ? "Data" : tab === "state" - ? "Widget State" - : tab === "sandbox" - ? "Sandbox" - : "Model Context"; + ? "Widget State" + : tab === "sandbox" + ? "Sandbox" + : "Model Context"; return ( @@ -446,8 +479,8 @@ export function ToolPart({ activeDebugTab === tab ? "bg-background text-foreground shadow-sm" : badge && badge > 0 - ? "text-destructive hover:text-destructive hover:bg-destructive/10" - : "text-muted-foreground/60 hover:text-muted-foreground hover:bg-background/50" + ? "text-destructive hover:text-destructive hover:bg-destructive/10" + : "text-muted-foreground/60 hover:text-muted-foreground hover:bg-background/50" }`} > @@ -534,7 +567,7 @@ export function ToolPart({

{canRun ? "Re-run tool with edited input" - : runDisabledReason ?? "Re-run tool with edited input"} + : (runDisabledReason ?? "Re-run tool with edited input")}

@@ -755,7 +788,7 @@ export function ToolPart({ // here — never render `javascript:`/`data:`/etc. as a clickable link. const renderAuthUrls = () => { const urls = filterSafeExternalLinkUrls( - (resultDisplayData as { authUrls?: unknown })?.authUrls + (resultDisplayData as { authUrls?: unknown })?.authUrls, ); if (urls.length === 0) return null; return ( @@ -813,6 +846,34 @@ export function ToolPart({ ); }; + const renderRecordedSandbox = () => + recordedWidgetDiagnostics ? ( +
+
+ Recorded widget policy +
+
+ Saved with this eval run; live policy and violations are unavailable. +
+
+ +
+
+ ) : null; + if (needsApproval) { return (
@@ -823,8 +884,8 @@ export function ToolPart({ approvalVisualState === "approved" ? "border-success/40 bg-success/10" : approvalVisualState === "denied" - ? "border-destructive/40 bg-destructive/10" - : "border-border/60 bg-muted/30" + ? "border-destructive/40 bg-destructive/10" + : "border-border/60 bg-muted/30", )} > @@ -853,7 +914,7 @@ export function ToolPart({ @@ -1020,7 +1081,16 @@ export function ToolPart({
{!hideDiagnosticsUI && ( <> - {hasWidgetDebug && activeDebugTab === "data" && renderToolData()} + {hasWidgetDebugData && activeDebugTab === "data" && ( +
+ {hasRecordedWidgetDebug && !hasWidgetDebug && ( +
+ Recorded tool data +
+ )} + {renderToolData()} +
+ )} {hasWidgetDebug && activeDebugTab === "state" && (
@@ -1070,6 +1140,10 @@ export function ToolPart({ protocol={widgetDebugInfo.protocol} /> )} + {!hasWidgetDebug && + hasRecordedWidgetDebug && + activeDebugTab === "sandbox" && + renderRecordedSandbox()} {hasWidgetDebug && activeDebugTab === "context" && (
@@ -1080,7 +1154,7 @@ export function ToolPart({
Updated:{" "} {new Date( - widgetDebugInfo.modelContext.updatedAt + widgetDebugInfo.modelContext.updatedAt, ).toLocaleTimeString()}
)} @@ -1139,7 +1213,7 @@ export function ToolPart({ )}
)} - {!hasWidgetDebug && renderToolData()} + {!hasWidgetDebugData && renderToolData()} )}
diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts index 93eb6f0ac9..cf1f8d7051 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts +++ b/mcpjam-inspector/client/src/components/evals/__tests__/trace-viewer-adapter.test.ts @@ -228,6 +228,96 @@ describe("adaptTraceToUiMessages", () => { expect(result.messages[0].parts[0]).not.toHaveProperty("traceDisplayMode"); }); + it("does not parse large JSON text that the shared tool card renders raw", () => { + const largeJsonText = JSON.stringify({ items: ["x".repeat(100_000)] }); + const trace: TraceEnvelope = { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call-large", + toolName: "read_large_result", + input: {}, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-large", + toolName: "read_large_result", + result: { + content: [{ type: "text", text: largeJsonText }], + }, + }, + ], + }, + ], + }; + const parseSpy = vi.spyOn(JSON, "parse"); + + try { + const result = adaptTraceToUiMessages({ + trace, + toolResultDisplay: "tool-card", + }); + + expect(parseSpy).not.toHaveBeenCalled(); + expect(result.messages[0].parts).toHaveLength(1); + } finally { + parseSpy.mockRestore(); + } + }); + + it("keeps tool-card errors inside the shared card without a duplicate", () => { + const trace: TraceEnvelope = { + messages: [ + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call-error-card", + toolName: "read_me", + input: {}, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-error-card", + toolName: "read_me", + result: { + isError: true, + content: [{ type: "text", text: "Request failed" }], + }, + }, + ], + }, + ], + }; + + const result = adaptTraceToUiMessages({ + trace, + toolResultDisplay: "tool-card", + }); + + expect(result.messages[0].parts).toEqual([ + expect.objectContaining({ + type: "dynamic-tool", + state: "output-error", + errorText: "Request failed", + }), + ]); + }); + // --- Test 2: Multiple tool calls --- it("groups multiple tool-calls and results into a single assistant UIMessage", () => { const trace: TraceEnvelope = { @@ -1294,7 +1384,9 @@ describe("adaptTraceToUiMessages", () => { connectedServerIds: [], }); - expect(result.toolRenderOverrides["call-disconnected-widget"]).toBeDefined(); + expect( + result.toolRenderOverrides["call-disconnected-widget"], + ).toBeDefined(); expect( result.toolRenderOverrides["call-disconnected-widget"].toolMetadata, ).toEqual({}); From 4cd401b07e7b449fe80c99567181f379569dcec8 Mon Sep 17 00:00:00 2001 From: Ignacio Jimenez Rocabado Date: Sat, 5 Sep 2026 11:34:36 -0700 Subject: [PATCH 5/7] Reuse sandbox workbench for recorded evals --- .../chat-v2/__tests__/PartSwitch.test.tsx | 4 +- .../thread/csp-workbench/CspWorkbench.tsx | 107 ++++++++++++++---- .../thread/csp-workbench/PolicyDiffTab.tsx | 31 +++-- .../thread/csp-workbench/SandboxStackTab.tsx | 103 +++++++++++++---- .../__tests__/recorded-policy.test.tsx | 42 +++++++ .../chat-v2/thread/csp-workbench/index.ts | 2 +- .../components/chat-v2/thread/part-switch.tsx | 11 +- .../parts/__tests__/display-modes.test.tsx | 9 +- .../chat-v2/thread/parts/tool-part.tsx | 48 ++------ 9 files changed, 258 insertions(+), 99 deletions(-) create mode 100644 mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/__tests__/recorded-policy.test.tsx diff --git a/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx b/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx index fdc3cded6f..b1c9a56492 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx @@ -146,7 +146,8 @@ vi.mock("@/lib/toast", () => ({ // Mock mcp-apps-utils vi.mock("@/lib/mcp-ui/mcp-apps-utils", () => ({ detectUIType: mockDetectUIType, - getUIResourceUri: () => null, + getUIResourceUri: (_uiType: unknown, metadata: any) => + metadata?.ui?.resourceUri ?? null, UIType: { OPENAI_SDK: "openai-apps", MCP_APPS: "mcp-apps", @@ -606,7 +607,6 @@ describe("PartSwitch", () => { toolRenderOverrides={{ "call-frozen": { frozenScreenshotUrl: "https://storage.example.com/widget.png", - resourceUri: "ui://widget/create-view.html", widgetCsp: { connectDomains: ["https://api.example.com"] }, widgetPermissions: { clipboardWrite: {} }, widgetPermissive: false, diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/CspWorkbench.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/CspWorkbench.tsx index 8a96a55bc5..8fbe83e693 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/CspWorkbench.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/CspWorkbench.tsx @@ -10,7 +10,7 @@ * Sandbox Stack — outer proxy iframe + inner View iframe */ -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { Tabs, TabsContent, @@ -31,6 +31,14 @@ import { SandboxStackTab } from "./SandboxStackTab"; type TabKey = "findings" | "policy-diff" | "sandbox"; +export interface RecordedWidgetPolicy { + resourceUri?: string; + csp?: unknown; + permissions?: unknown; + permissive?: boolean; + prefersBorder?: boolean; +} + /** Subset of the existing `sandboxInfo` prop the workbench needs. Mirrors * the shape `tool-part.tsx` already constructs. */ export interface CspWorkbenchProps { @@ -42,25 +50,56 @@ export interface CspWorkbenchProps { hostInfo?: { name: string; version: string } | null; }; protocol?: "openai-apps" | "mcp-apps"; + recordedPolicy?: RecordedWidgetPolicy; +} + +function stringList(value: unknown): string[] | undefined { + return Array.isArray(value) && value.every((item) => typeof item === "string") + ? value + : undefined; +} + +function recordedDeclaration(policy: RecordedWidgetPolicy | undefined) { + if (!policy?.csp || typeof policy.csp !== "object") return null; + const csp = policy.csp as Record; + return { + connectDomains: stringList(csp.connectDomains), + resourceDomains: stringList(csp.resourceDomains), + frameDomains: stringList(csp.frameDomains), + baseUriDomains: stringList(csp.baseUriDomains), + connect_domains: stringList(csp.connect_domains), + resource_domains: stringList(csp.resource_domains), + }; } -export function CspWorkbench({ sandboxInfo, protocol }: CspWorkbenchProps) { - const [activeTab, setActiveTab] = useState("findings"); +export function CspWorkbench({ + sandboxInfo, + protocol, + recordedPolicy, +}: CspWorkbenchProps) { + const [activeTab, setActiveTab] = useState(() => + recordedPolicy ? "policy-diff" : "findings", + ); const [jumpToHost, setJumpToHost] = useState(null); + const isRecorded = !!recordedPolicy; const input = useMemo( () => ({ effective: { - connectDomains: sandboxInfo?.connectDomains ?? [], - resourceDomains: sandboxInfo?.resourceDomains ?? [], - frameDomains: sandboxInfo?.frameDomains, - baseUriDomains: sandboxInfo?.baseUriDomains, + connectDomains: isRecorded ? [] : (sandboxInfo?.connectDomains ?? []), + resourceDomains: isRecorded ? [] : (sandboxInfo?.resourceDomains ?? []), + frameDomains: isRecorded ? undefined : sandboxInfo?.frameDomains, + baseUriDomains: isRecorded ? undefined : sandboxInfo?.baseUriDomains, }, - widgetDeclared: sandboxInfo?.widgetDeclared ?? null, - subtypePolicy: sandboxInfo?.applied?.cspSubtypePolicy, - violations: sandboxInfo?.violations ?? [], + widgetDeclared: isRecorded + ? recordedDeclaration(recordedPolicy) + : (sandboxInfo?.widgetDeclared ?? null), + subtypePolicy: isRecorded + ? undefined + : sandboxInfo?.applied?.cspSubtypePolicy, + violations: isRecorded ? [] : (sandboxInfo?.violations ?? []), }), - [sandboxInfo] + [isRecorded, recordedPolicy, sandboxInfo], ); const diagnoses = useMemo(() => classifyDiagnoses(input), [input]); @@ -70,12 +109,32 @@ export function CspWorkbench({ sandboxInfo, protocol }: CspWorkbenchProps) { setActiveTab("policy-diff"); }, []); + useEffect(() => { + if (isRecorded && activeTab === "findings") { + setActiveTab("policy-diff"); + } + }, [activeTab, isRecorded]); + // Absence-of-data: keep parity with the old panel — return null rather // than rendering an empty workbench. - if (!sandboxInfo) return null; + if (!sandboxInfo && !recordedPolicy) return null; return ( -
+
+ {isRecorded && ( +
+
+ Recorded widget policy +
+
+ Saved with this eval run; live policy and violations are + unavailable. +
+
+ )} { @@ -84,9 +143,11 @@ export function CspWorkbench({ sandboxInfo, protocol }: CspWorkbenchProps) { }} > - - Findings - + {!isRecorded && ( + + Findings + + )} Policy Diff @@ -99,8 +160,8 @@ export function CspWorkbench({ sandboxInfo, protocol }: CspWorkbenchProps) { @@ -108,6 +169,7 @@ export function CspWorkbench({ sandboxInfo, protocol }: CspWorkbenchProps) { setJumpToHost(null)} /> @@ -115,11 +177,12 @@ export function CspWorkbench({ sandboxInfo, protocol }: CspWorkbenchProps) { diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/PolicyDiffTab.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/PolicyDiffTab.tsx index 4aaf3b088c..26a3e8d1ca 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/PolicyDiffTab.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/PolicyDiffTab.tsx @@ -10,6 +10,7 @@ interface PolicyDiffTabProps { jumpToHost?: string | null; /** Cleared after the jump animation completes. */ onJumpHandled?: () => void; + recorded?: boolean; } type RowState = "allowed" | "blocked" | "stripped" | "cors" | "mismatch"; @@ -24,7 +25,8 @@ interface Row { function expressionToHost(expr: string): string | null { const trimmed = expr.trim(); if (!trimmed || trimmed.startsWith("'")) return null; - if (trimmed === "*" || trimmed === "data:" || trimmed === "blob:") return trimmed; + if (trimmed === "*" || trimmed === "data:" || trimmed === "blob:") + return trimmed; if (/^[a-zA-Z][a-zA-Z0-9+\-.]*:$/.test(trimmed)) return trimmed; let rest = trimmed.replace(/^[a-zA-Z][a-zA-Z0-9+\-.]*:\/\//, ""); const slash = rest.indexOf("/"); @@ -45,7 +47,10 @@ function buildRequestedRows( } }; pushAll(declared.connectDomains ?? declared.connect_domains, "connect-src"); - pushAll(declared.resourceDomains ?? declared.resource_domains, "img/script/font/style-src"); + pushAll( + declared.resourceDomains ?? declared.resource_domains, + "img/script/font/style-src", + ); pushAll(declared.frameDomains, "frame-src"); pushAll(declared.baseUriDomains, "base-uri"); return rows; @@ -193,7 +198,7 @@ function PolicyColumn({ const summaryText = rows.length === 0 ? emptyLabel - : tone === "warn" && (title === "Observed") + : tone === "warn" && title === "Observed" ? `${rows.length} ${rows.length === 1 ? "block" : "blocks"}` : `${summary.directives} ${summary.directives === 1 ? "directive" : "directives"} · ${summary.sources} ${summary.sources === 1 ? "source" : "sources"}`; @@ -278,7 +283,7 @@ function hostMatches(rowHost: string, target: string): boolean { if (!target) return false; const r = rowHost.toLowerCase(); const t = target.toLowerCase().replace(/^https?:\/\//, ""); - return r === t || r.endsWith("." + t) || ("*." + r.replace(/^\*\./, "")) === t; + return r === t || r.endsWith("." + t) || "*." + r.replace(/^\*\./, "") === t; } export function PolicyDiffTab({ @@ -286,6 +291,7 @@ export function PolicyDiffTab({ diagnoses, jumpToHost, onJumpHandled, + recorded = false, }: PolicyDiffTabProps) { const containerRef = useRef(null); @@ -308,11 +314,14 @@ export function PolicyDiffTab({ [input.widgetDeclared], ); const effectiveRows = useMemo( - () => [ - ...buildEffectiveRows(input.effective, mismatchHosts), - ...buildStrippedRows(input.widgetDeclared, input.effective), - ], - [input.effective, input.widgetDeclared, mismatchHosts], + () => + recorded + ? [] + : [ + ...buildEffectiveRows(input.effective, mismatchHosts), + ...buildStrippedRows(input.widgetDeclared, input.effective), + ], + [input.effective, input.widgetDeclared, mismatchHosts, recorded], ); const observedRows = useMemo(() => buildObservedRows(diagnoses), [diagnoses]); @@ -350,7 +359,7 @@ export function PolicyDiffTab({ title="Effective" subtitle="host granted" rows={effectiveRows} - emptyLabel="No allowlist captured" + emptyLabel={recorded ? "Not recorded" : "No allowlist captured"} jumpHost={jumpToHost} forceOpen={Boolean(jumpToHost)} /> @@ -358,7 +367,7 @@ export function PolicyDiffTab({ title="Observed" subtitle="browser saw" rows={observedRows} - emptyLabel="No violations" + emptyLabel={recorded ? "Not recorded" : "No violations"} jumpHost={jumpToHost} forceOpen={Boolean(jumpToHost)} /> diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/SandboxStackTab.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/SandboxStackTab.tsx index 06b43b4db6..4a787e288d 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/SandboxStackTab.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/SandboxStackTab.tsx @@ -5,6 +5,7 @@ import type { WidgetSandboxApplied, } from "@/stores/widget-debug-store"; import { copyToClipboard } from "@/lib/clipboard"; +import type { RecordedWidgetPolicy } from "./CspWorkbench"; interface SandboxStackTabProps { applied?: WidgetSandboxApplied; @@ -12,6 +13,7 @@ interface SandboxStackTabProps { mounts?: WidgetMount[]; hostInfo?: { name: string; version: string } | null; protocol?: "openai-apps" | "mcp-apps"; + recordedPolicy?: RecordedWidgetPolicy; } /** Compact label/value chip — the workbench's atomic unit. */ @@ -136,19 +138,31 @@ export function SandboxStackTab({ lifecycle, mounts, protocol, + recordedPolicy, }: SandboxStackTabProps) { + const isRecorded = !!recordedPolicy; const sandboxAttrs = applied?.sandboxAttrs && applied.sandboxAttrs.length > 0 ? applied.sandboxAttrs : ["allow-scripts", "allow-same-origin"]; - const permissionsList = applied?.permissions - ? Object.keys(applied.permissions).map((p) => - p.replace(/([A-Z])/g, "-$1").toLowerCase(), - ) - : applied?.allowFeatures - ? Object.keys(applied.allowFeatures) + const recordedPermissions = + recordedPolicy?.permissions && + typeof recordedPolicy.permissions === "object" && + !Array.isArray(recordedPolicy.permissions) + ? Object.keys(recordedPolicy.permissions).map((permission) => + permission.replace(/([A-Z])/g, "-$1").toLowerCase(), + ) : []; + const permissionsList = isRecorded + ? recordedPermissions + : applied?.permissions + ? Object.keys(applied.permissions).map((p) => + p.replace(/([A-Z])/g, "-$1").toLowerCase(), + ) + : applied?.allowFeatures + ? Object.keys(applied.allowFeatures) + : []; const lc = lifecycleStatus(lifecycle); const mountCount = mounts?.length ?? 0; @@ -168,7 +182,15 @@ export function SandboxStackTab({
} + value={ + isRecorded ? ( + + not recorded + + ) : ( + + ) + } /> - {lc.text} - + isRecorded ? ( + (recordedPolicy.resourceUri ?? "not recorded") + ) : ( + + {lc.text} + + ) + } + tone={ + isRecorded && !recordedPolicy.resourceUri + ? "muted" + : !isRecorded && lc.tone === "muted" + ? "muted" + : "neutral" } - tone={lc.tone === "muted" ? "muted" : "neutral"} /> - + {isRecorded ? ( + + ) : ( + + )} + {isRecorded && ( + + )} {mountCount > 1 && ( { + it("reuses the policy and sandbox views without claiming live evidence", async () => { + const user = userEvent.setup(); + render( + , + ); + + expect(screen.getByText("Recorded widget policy")).toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Findings" })).toBeNull(); + expect(screen.getAllByText("Not recorded")).toHaveLength(2); + + await user.click(screen.getByRole("tab", { name: "Sandbox Stack" })); + expect( + screen.getByText("ui://widget/create-view.html"), + ).toBeInTheDocument(); + expect(screen.getByText("clipboard-write")).toBeInTheDocument(); + expect(screen.getByText("restricted")).toBeInTheDocument(); + expect(screen.getByText("yes")).toBeInTheDocument(); + expect(screen.getByText("not recorded")).toBeInTheDocument(); + }); + + it("shows missing old snapshot fields as unavailable instead of null JSON", () => { + render(); + + expect(screen.getByText("No CSP declared")).toBeInTheDocument(); + expect(screen.queryByText('"resourceUri": null')).toBeNull(); + }); +}); diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/index.ts b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/index.ts index 40d5561c07..7958f43928 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/index.ts +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/index.ts @@ -1,2 +1,2 @@ export { CspWorkbench } from "./CspWorkbench"; -export type { CspWorkbenchProps } from "./CspWorkbench"; +export type { CspWorkbenchProps, RecordedWidgetPolicy } from "./CspWorkbench"; diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx index fdf374395c..1bd3efde24 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx @@ -27,7 +27,11 @@ import { getToolServerId, ToolServerMap, } from "@/lib/apis/mcp-tools-api"; -import { detectUIType, UIType } from "@/lib/mcp-ui/mcp-apps-utils"; +import { + detectUIType, + getUIResourceUri, + UIType, +} from "@/lib/mcp-ui/mcp-apps-utils"; import { AnyPart, getDataLabel, @@ -558,7 +562,10 @@ export function PartSwitch({ recordedWidgetDiagnostics={ isFrozenWidget ? { - resourceUri: renderOverride?.resourceUri, + resourceUri: + renderOverride?.resourceUri ?? + getUIResourceUri(uiType, effectiveToolMeta) ?? + undefined, csp: renderOverride?.widgetCsp, permissions: renderOverride?.widgetPermissions, permissive: renderOverride?.widgetPermissive, diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/__tests__/display-modes.test.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/__tests__/display-modes.test.tsx index fef57eeb2c..1221e834df 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/__tests__/display-modes.test.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/__tests__/display-modes.test.tsx @@ -68,7 +68,12 @@ vi.mock("@mcpjam/design-system/badge", () => ({ })); vi.mock("../../csp-workbench", () => ({ - CspWorkbench: () => null, + CspWorkbench: ({ recordedPolicy }: any) => ( +
+ Saved with this eval run + {JSON.stringify(recordedPolicy)} +
+ ), })); // Mock JsonEditor to avoid pulling in additional lucide icons @@ -281,7 +286,7 @@ describe("ToolPart display mode controls", () => { expect(screen.getByTestId("recorded-widget-diagnostics")).toHaveTextContent( "Saved with this eval run", ); - expect(screen.getByTestId("json-editor")).toHaveTextContent( + expect(screen.getByTestId("recorded-widget-diagnostics")).toHaveTextContent( "https://api.example.com", ); }); diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx index 4d1e3c58b7..3fcd27a859 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/parts/tool-part.tsx @@ -48,7 +48,7 @@ import { ToggleGroup, ToggleGroupItem, } from "@mcpjam/design-system/toggle-group"; -import { CspWorkbench } from "../csp-workbench"; +import { CspWorkbench, type RecordedWidgetPolicy } from "../csp-workbench"; import { JsonEditor } from "@/components/ui/json-editor"; import { cn } from "@/lib/chat-utils"; import { @@ -66,14 +66,6 @@ import { McpToolResultImagePreviewGrid } from "@/components/chat-v2/shared/mcp-t type ApprovalVisualState = "pending" | "approved" | "denied"; type TraceDisplayMode = "markdown" | "json-markdown"; -export interface RecordedWidgetDiagnostics { - resourceUri?: string; - csp?: unknown; - permissions?: unknown; - permissive?: boolean; - prefersBorder?: boolean; -} - export function ToolPart({ part, chatSessionId, @@ -161,7 +153,7 @@ export function ToolPart({ serverId?: string; mcpToolResultImageRendering?: McpToolResultImageRenderingPolicy; rawOutput?: unknown; - recordedWidgetDiagnostics?: RecordedWidgetDiagnostics; + recordedWidgetDiagnostics?: RecordedWidgetPolicy; }) { const hasTrackedSkillLoad = useRef(false); @@ -846,34 +838,6 @@ export function ToolPart({ ); }; - const renderRecordedSandbox = () => - recordedWidgetDiagnostics ? ( -
-
- Recorded widget policy -
-
- Saved with this eval run; live policy and violations are unavailable. -
-
- -
-
- ) : null; - if (needsApproval) { return (
@@ -1142,8 +1106,12 @@ export function ToolPart({ )} {!hasWidgetDebug && hasRecordedWidgetDebug && - activeDebugTab === "sandbox" && - renderRecordedSandbox()} + activeDebugTab === "sandbox" && ( + + )} {hasWidgetDebug && activeDebugTab === "context" && (
From af4d54a66f23ecdaa701c1ac12498ca508c54af7 Mon Sep 17 00:00:00 2001 From: Ignacio Jimenez Rocabado Date: Sat, 5 Sep 2026 11:46:25 -0700 Subject: [PATCH 6/7] Show recorded eval widget errors --- .../chat-v2/__tests__/PartSwitch.test.tsx | 6 +++ .../thread/csp-workbench/CspWorkbench.tsx | 44 ++++++++++++++++++ .../__tests__/recorded-policy.test.tsx | 5 +++ .../components/chat-v2/thread/part-switch.tsx | 4 ++ .../chat-v2/thread/tool-render-overrides.ts | 7 ++- .../frozen-screenshot-overrides.test.ts | 45 +++++++++++++++++-- .../evals/frozen-screenshot-overrides.ts | 41 ++++++++++++++--- 7 files changed, 141 insertions(+), 11 deletions(-) diff --git a/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx b/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx index b1c9a56492..8cf1e9546d 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/__tests__/PartSwitch.test.tsx @@ -611,6 +611,10 @@ describe("PartSwitch", () => { widgetPermissions: { clipboardWrite: {} }, widgetPermissive: false, prefersBorder: true, + recordedWidgetErrors: { + consoleErrors: ["TypeError: broken"], + blockedRequests: ["https://blocked.example.com"], + }, }, }} />, @@ -629,6 +633,8 @@ describe("PartSwitch", () => { permissions: { clipboardWrite: {} }, permissive: false, prefersBorder: true, + consoleErrors: ["TypeError: broken"], + blockedRequests: ["https://blocked.example.com"], }, }), ); diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/CspWorkbench.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/CspWorkbench.tsx index 8fbe83e693..70be14222e 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/CspWorkbench.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/CspWorkbench.tsx @@ -37,6 +37,8 @@ export interface RecordedWidgetPolicy { permissions?: unknown; permissive?: boolean; prefersBorder?: boolean; + consoleErrors?: string[]; + blockedRequests?: string[]; } /** Subset of the existing `sandboxInfo` prop the workbench needs. Mirrors @@ -82,6 +84,9 @@ export function CspWorkbench({ ); const [jumpToHost, setJumpToHost] = useState(null); const isRecorded = !!recordedPolicy; + const recordedErrorCount = + (recordedPolicy?.consoleErrors?.length ?? 0) + + (recordedPolicy?.blockedRequests?.length ?? 0); const input = useMemo( () => ({ @@ -135,6 +140,45 @@ export function CspWorkbench({
)} + {recordedErrorCount > 0 && ( +
+
+ {recordedErrorCount} recorded error + {recordedErrorCount === 1 ? "" : "s"} +
+ {(recordedPolicy?.consoleErrors?.length ?? 0) > 0 && ( +
+
+ Console +
+
    + {recordedPolicy?.consoleErrors?.map((error, index) => ( +
  • + {error} +
    • + ))} +
+
+ )} + {(recordedPolicy?.blockedRequests?.length ?? 0) > 0 && ( +
+
+ Blocked requests +
+
    + {recordedPolicy?.blockedRequests?.map((request, index) => ( +
  • + {request} +
    • + ))} +
+
+ )} +
+ )} { diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/__tests__/recorded-policy.test.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/__tests__/recorded-policy.test.tsx index 9764554c9f..31d70912f7 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/__tests__/recorded-policy.test.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/csp-workbench/__tests__/recorded-policy.test.tsx @@ -15,11 +15,16 @@ describe("CspWorkbench recorded policy", () => { permissions: { clipboardWrite: {} }, permissive: false, prefersBorder: true, + consoleErrors: ["TypeError: broken"], + blockedRequests: ["https://blocked.example.com"], }} />, ); expect(screen.getByText("Recorded widget policy")).toBeInTheDocument(); + expect(screen.getByText("2 recorded errors")).toBeInTheDocument(); + expect(screen.getByText("TypeError: broken")).toBeInTheDocument(); + expect(screen.getByText("https://blocked.example.com")).toBeInTheDocument(); expect(screen.queryByRole("tab", { name: "Findings" })).toBeNull(); expect(screen.getAllByText("Not recorded")).toHaveLength(2); diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx b/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx index 1bd3efde24..db8d10475f 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/part-switch.tsx @@ -570,6 +570,10 @@ export function PartSwitch({ permissions: renderOverride?.widgetPermissions, permissive: renderOverride?.widgetPermissive, prefersBorder: renderOverride?.prefersBorder, + consoleErrors: + renderOverride?.recordedWidgetErrors?.consoleErrors, + blockedRequests: + renderOverride?.recordedWidgetErrors?.blockedRequests, } : undefined } diff --git a/mcpjam-inspector/client/src/components/chat-v2/thread/tool-render-overrides.ts b/mcpjam-inspector/client/src/components/chat-v2/thread/tool-render-overrides.ts index 7de9149374..f42788ee94 100644 --- a/mcpjam-inspector/client/src/components/chat-v2/thread/tool-render-overrides.ts +++ b/mcpjam-inspector/client/src/components/chat-v2/thread/tool-render-overrides.ts @@ -16,6 +16,11 @@ export interface ToolRenderOverride { * non-interactive runs; live/record-armed surfaces leave it unset. */ frozenScreenshotUrl?: string | null; + /** Eval-only browser evidence captured with a frozen widget screenshot. */ + recordedWidgetErrors?: { + consoleErrors: string[]; + blockedRequests: string[]; + }; /** * Try the live MCP Apps fetch path before falling back to * `cachedWidgetHtmlUrl`. Used by in-flow session revisit so the widget @@ -65,7 +70,7 @@ export interface ToolRenderOverride { */ export function widgetSlotShouldRender( liveWidgetEligible: boolean, - override: Pick | undefined + override: Pick | undefined, ): boolean { return liveWidgetEligible || !!override?.frozenScreenshotUrl; } diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/frozen-screenshot-overrides.test.ts b/mcpjam-inspector/client/src/components/evals/__tests__/frozen-screenshot-overrides.test.ts index a6ee1a6fc3..f19519d590 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/frozen-screenshot-overrides.test.ts +++ b/mcpjam-inspector/client/src/components/evals/__tests__/frozen-screenshot-overrides.test.ts @@ -7,7 +7,7 @@ import type { ToolRenderOverride } from "@/components/chat-v2/thread/tool-render import { buildFrozenScreenshotOverrides } from "../frozen-screenshot-overrides"; const obs = ( - o: Partial + o: Partial, ): EvalTraceWidgetRenderObservationView => ({ toolCallId: "tc1", toolName: "search-products", @@ -19,7 +19,7 @@ const obs = ( }); const interaction = ( - o: Partial + o: Partial, ): EvalTraceBrowserInteractionStepView => ({ toolCallId: "tc1", stepIndex: 0, @@ -38,6 +38,43 @@ describe("buildFrozenScreenshotOverrides", () => { expect(out["tc1"]?.frozenScreenshotUrl).toBe("https://s/a.png"); }); + it("carries saved eval errors and the observation resource URI into the frozen override", () => { + const out = buildFrozenScreenshotOverrides({}, [ + obs({ + toolCallId: "tc1", + screenshotUrl: "https://s/a.png", + resourceUri: "ui://widget/view.html", + consoleErrors: ["TypeError: broken"], + blockedRequests: ["https://blocked.example.com"], + }), + ]); + + expect(out.tc1).toMatchObject({ + resourceUri: "ui://widget/view.html", + recordedWidgetErrors: { + consoleErrors: ["TypeError: broken"], + blockedRequests: ["https://blocked.example.com"], + }, + }); + }); + + it("keeps errors from later failed observations with the last good screenshot", () => { + const out = buildFrozenScreenshotOverrides({}, [ + obs({ screenshotUrl: "https://s/good.png", consoleErrors: ["first"] }), + obs({ + ts: 2, + status: "render_error", + screenshotUrl: undefined, + consoleErrors: ["first", "later"], + }), + ]); + + expect(out.tc1?.recordedWidgetErrors?.consoleErrors).toEqual([ + "first", + "later", + ]); + }); + it("merges onto an existing snapshot override without dropping its fields", () => { const base: Record = { tc1: { cachedWidgetHtmlUrl: "https://s/html", serverId: "srv" }, @@ -116,7 +153,7 @@ describe("buildFrozenScreenshotOverrides", () => { ts: 90, screenshotUrl: "https://s/cart-open.png", }), - ] + ], ); expect(out["tc1"]?.frozenScreenshotUrl).toBe("https://s/cart-open.png"); }); @@ -131,7 +168,7 @@ describe("buildFrozenScreenshotOverrides", () => { screenshotUrl: "https://s/initial.png", }), ], - [interaction({ toolCallId: "tc1", ts: 50, screenshotUrl: null })] + [interaction({ toolCallId: "tc1", ts: 50, screenshotUrl: null })], ); expect(out["tc1"]?.frozenScreenshotUrl).toBe("https://s/initial.png"); }); diff --git a/mcpjam-inspector/client/src/components/evals/frozen-screenshot-overrides.ts b/mcpjam-inspector/client/src/components/evals/frozen-screenshot-overrides.ts index 28ae25c10d..04fe678578 100644 --- a/mcpjam-inspector/client/src/components/evals/frozen-screenshot-overrides.ts +++ b/mcpjam-inspector/client/src/components/evals/frozen-screenshot-overrides.ts @@ -25,21 +25,41 @@ import type { ToolRenderOverride } from "@/components/chat-v2/thread/tool-render export function buildFrozenScreenshotOverrides( base: Record, observations: readonly EvalTraceWidgetRenderObservationView[], - interactionSteps: readonly EvalTraceBrowserInteractionStepView[] = [] + interactionSteps: readonly EvalTraceBrowserInteractionStepView[] = [], ): Record { - const latestByTool = new Map(); + const latestByTool = new Map< + string, + { url: string; ts: number; resourceUri?: string } + >(); + const errorsByTool = new Map< + string, + { consoleErrors: Set; blockedRequests: Set } + >(); const consider = ( toolCallId: string, url: string | null | undefined, - ts: number + ts: number, + resourceUri?: string, ) => { if (!url) return; const prev = latestByTool.get(toolCallId); - if (!prev || ts > prev.ts) latestByTool.set(toolCallId, { url, ts }); + if (!prev || ts > prev.ts) { + latestByTool.set(toolCallId, { url, ts, resourceUri }); + } }; for (const obs of observations) { + const errors = errorsByTool.get(obs.toolCallId) ?? { + consoleErrors: new Set(), + blockedRequests: new Set(), + }; + for (const error of obs.consoleErrors ?? []) + errors.consoleErrors.add(error); + for (const request of obs.blockedRequests ?? []) { + errors.blockedRequests.add(request); + } + errorsByTool.set(obs.toolCallId, errors); if (obs.status === "rendered") - consider(obs.toolCallId, obs.screenshotUrl, obs.ts); + consider(obs.toolCallId, obs.screenshotUrl, obs.ts, obs.resourceUri); } // Interaction-step captures are later than the initial render (they happen on // each click), so they win on `ts` — surfacing the post-interaction state. @@ -48,10 +68,19 @@ export function buildFrozenScreenshotOverrides( } if (latestByTool.size === 0) return base; const merged = { ...base }; - for (const [toolCallId, { url }] of latestByTool) { + for (const [toolCallId, { url, resourceUri }] of latestByTool) { + const errors = errorsByTool.get(toolCallId); + const consoleErrors = [...(errors?.consoleErrors ?? [])]; + const blockedRequests = [...(errors?.blockedRequests ?? [])]; merged[toolCallId] = { ...merged[toolCallId], frozenScreenshotUrl: url, + ...(merged[toolCallId]?.resourceUri || !resourceUri + ? {} + : { resourceUri }), + ...(consoleErrors.length || blockedRequests.length + ? { recordedWidgetErrors: { consoleErrors, blockedRequests } } + : {}), }; } return merged; From 8cd23587ab99a4e73e7dd7cdbbe94b83d3095312 Mon Sep 17 00:00:00 2001 From: Ignacio Jimenez Rocabado Date: Sat, 5 Sep 2026 12:02:40 -0700 Subject: [PATCH 7/7] Preserve recorded widget resource URI --- .../frozen-screenshot-overrides.test.ts | 26 +++++++++++++++++++ .../evals/frozen-screenshot-overrides.ts | 6 ++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/mcpjam-inspector/client/src/components/evals/__tests__/frozen-screenshot-overrides.test.ts b/mcpjam-inspector/client/src/components/evals/__tests__/frozen-screenshot-overrides.test.ts index f19519d590..3d0f5d31d9 100644 --- a/mcpjam-inspector/client/src/components/evals/__tests__/frozen-screenshot-overrides.test.ts +++ b/mcpjam-inspector/client/src/components/evals/__tests__/frozen-screenshot-overrides.test.ts @@ -158,6 +158,32 @@ describe("buildFrozenScreenshotOverrides", () => { expect(out["tc1"]?.frozenScreenshotUrl).toBe("https://s/cart-open.png"); }); + it("keeps the render resource URI when a later interaction supplies only a screenshot", () => { + const out = buildFrozenScreenshotOverrides( + {}, + [ + obs({ + toolCallId: "tc1", + ts: 1, + screenshotUrl: "https://s/initial.png", + resourceUri: "ui://widget/view.html", + }), + ], + [ + interaction({ + toolCallId: "tc1", + ts: 50, + screenshotUrl: "https://s/final.png", + }), + ], + ); + + expect(out.tc1).toMatchObject({ + frozenScreenshotUrl: "https://s/final.png", + resourceUri: "ui://widget/view.html", + }); + }); + it("falls back to the render screenshot when no interaction step has one", () => { const out = buildFrozenScreenshotOverrides( {}, diff --git a/mcpjam-inspector/client/src/components/evals/frozen-screenshot-overrides.ts b/mcpjam-inspector/client/src/components/evals/frozen-screenshot-overrides.ts index 04fe678578..0aeea0c7bd 100644 --- a/mcpjam-inspector/client/src/components/evals/frozen-screenshot-overrides.ts +++ b/mcpjam-inspector/client/src/components/evals/frozen-screenshot-overrides.ts @@ -44,7 +44,11 @@ export function buildFrozenScreenshotOverrides( if (!url) return; const prev = latestByTool.get(toolCallId); if (!prev || ts > prev.ts) { - latestByTool.set(toolCallId, { url, ts, resourceUri }); + latestByTool.set(toolCallId, { + url, + ts, + resourceUri: resourceUri ?? prev?.resourceUri, + }); } }; for (const obs of observations) {