From 10ca4c2791c27160b75d3923d4412e7315228b5e Mon Sep 17 00:00:00 2001 From: Yu Xie Date: Wed, 28 Oct 2026 22:52:31 +0100 Subject: [PATCH 1/4] fix: diagnose conversation fallbacks without exposing private state Record bounded unavailable and recovery episodes, with static Copilot identity rejection stages and unchanged HTTP/ownership behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1fad73aa-cbc5-42dd-916b-64e5198bc512 --- .../conversation-unavailable-diagnostics.md | 3 + server/connection-diagnostics.ts | 8 +- server/conversation-diagnostics.test.ts | 89 +++++++++++++++++++ server/conversation-diagnostics.ts | 43 +++++++++ server/conversation-shared.ts | 8 +- server/conversation.ts | 16 ++-- server/copilot.test.ts | 4 +- server/pane-routes.test.ts | 49 +++++++++- server/pane-routes.ts | 16 +++- 9 files changed, 218 insertions(+), 18 deletions(-) create mode 100644 changelog.d/conversation-unavailable-diagnostics.md create mode 100644 server/conversation-diagnostics.test.ts create mode 100644 server/conversation-diagnostics.ts diff --git a/changelog.d/conversation-unavailable-diagnostics.md b/changelog.d/conversation-unavailable-diagnostics.md new file mode 100644 index 0000000000..a268bd11d0 --- /dev/null +++ b/changelog.d/conversation-unavailable-diagnostics.md @@ -0,0 +1,3 @@ +### Maintenance +- Conversation fallbacks and recoveries now emit sanitized, rate-limited diagnostic episodes, + including the failed Copilot identity stage without logging session IDs or transcript paths. diff --git a/server/connection-diagnostics.ts b/server/connection-diagnostics.ts index 777763ca10..75ba9968c0 100644 --- a/server/connection-diagnostics.ts +++ b/server/connection-diagnostics.ts @@ -4,7 +4,7 @@ const REPEAT_MS = 30_000; const MAX_FAILURES = 128; const TRANSPORT_CODES = ["timeout", "connect_failed", "socket_error", "closed", "bad_frame", "server_unavailable"] as const; type TransportCode = typeof TRANSPORT_CODES[number]; -type FailureCode = TransportCode | "refresh_failed" | "connection_lost" | "action_required"; +type FailureCode = TransportCode | "refresh_failed" | "connection_lost" | "action_required" | "conversation_unavailable" | "conversation_read_pending"; export function transportFailureCode(code: string): TransportCode | null { return TRANSPORT_CODES.find((known) => known === code) ?? null; @@ -33,12 +33,12 @@ interface Failure { visible: boolean; } -/** Connection failures only: no access log, input, credentials, peer names, or exception text. */ +/** Bounded failure episodes: no access log, input, credentials, peer names, or exception text. */ export class ConnectionDiagnostics { private failures = new Map(); constructor( - private component: "herdr.rpc" | "machine.local" | "machine.remote" | "websocket", + private component: "herdr.rpc" | "machine.local" | "machine.remote" | "websocket" | "conversation", private write: (line: string) => void = (line) => console.warn(line), private now: () => number = Date.now, ) {} @@ -93,7 +93,7 @@ export class ConnectionDiagnostics { code: failure.code, failures: failure.count, suppressed: failure.suppressed, since_first_failure_ms: Math.max(0, now - failure.since), }; - if (fields.request_id && /^br-[a-z0-9]+-[0-9]+$/.test(fields.request_id)) record.request_id = fields.request_id; + if (fields.request_id && (/^br-[a-z0-9]+-[0-9]+$/.test(fields.request_id) || /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(fields.request_id))) record.request_id = fields.request_id; if (fields.machine_id && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(fields.machine_id)) record.machine_id = fields.machine_id; if (fields.phase === "connect" || fields.phase === "response") record.phase = fields.phase; for (const field of ["elapsed_ms", "connect_ms", "timeout_ms", "pending", "close_code"] as const) { diff --git a/server/conversation-diagnostics.test.ts b/server/conversation-diagnostics.test.ts new file mode 100644 index 0000000000..5c7b0f837b --- /dev/null +++ b/server/conversation-diagnostics.test.ts @@ -0,0 +1,89 @@ +import { expect, it, spyOn } from "bun:test"; +import { ConversationDiagnostics } from "./conversation-diagnostics.ts"; +import { COPILOT_UNAVAILABLE_REASONS, ConversationReadPending, ConversationUnavailable } from "./conversation-shared.ts"; + +function fixture() { + let now = 0; + const lines: string[] = []; + const diagnostics = new ConversationDiagnostics((line) => lines.push(line), () => now); + return { diagnostics, lines, records: () => lines.map((line) => JSON.parse(line)), advance: (ms: number) => { now += ms; } }; +} + +it("logs only fixed failure stages and opaque request correlation, never exception or pane contents", () => { + const { diagnostics, records, lines } = fixture(); + const secret = "/private/session-token\ntranscript"; + const error = new ConversationUnavailable(secret, undefined, "copilot_process_lookup_failed"); + Object.assign(error, { path: secret, sessionId: secret }); + diagnostics.unavailable(secret, error, "00000000-0000-4000-8000-000000000001"); + diagnostics.unavailable("other pane", new ConversationUnavailable(secret), secret); + expect(records()[0]).toMatchObject({ + event: "conversation.failed", operation: "conversation.copilot_process_lookup_failed", + code: "conversation_unavailable", request_id: "00000000-0000-4000-8000-000000000001", + }); + expect(records()[1]).toMatchObject({ operation: "conversation.unknown" }); + expect(records()[1].request_id).toBeUndefined(); + expect(lines.join("\n")).not.toContain("private"); + expect(lines.join("\n")).not.toContain("session-token"); +}); + +it("coalesces repeated reasons and rapid recoveries without losing the episode correlation", () => { + const { diagnostics, records, advance } = fixture(); + for (let i = 0; i < 100; i++) { + diagnostics.unavailable("pane", new ConversationUnavailable("no_session_path")); + diagnostics.recovered("pane"); + } + expect(records()).toHaveLength(2); + expect(records()[1]).toMatchObject({ event: "conversation.recovered", episode_id: records()[0].episode_id }); + advance(30_000); + diagnostics.unavailable("pane", new ConversationReadPending()); + expect(records()[2]).toMatchObject({ + operation: "conversation.read_pending", code: "conversation_read_pending", failures: 100, suppressed: 99, + }); +}); + +it("bounds output even when pane keys rotate past the retained episode limit", () => { + const { diagnostics, records, advance } = fixture(); + for (let i = 0; i < 300; i++) diagnostics.unavailable(`pane-${i}`, new ConversationUnavailable("no_session_id")); + expect(records()).toHaveLength(16); + advance(30_000); + diagnostics.recovered("pane-299"); + expect(records()).toHaveLength(16); + diagnostics.recovered("pane-0"); + expect(records()[16]).toMatchObject({ event: "conversation.recovered", episode_id: records()[0].episode_id }); + diagnostics.unavailable("new-pane", new ConversationUnavailable("no_session_id")); + expect(records()).toHaveLength(18); +}); + +it("keeps the logging interval monotonic across wall-clock corrections", () => { + const { diagnostics, records, advance } = fixture(); + const wall = spyOn(Date, "now").mockReturnValue(100_000); + try { + diagnostics.unavailable("pane", new ConversationUnavailable("no_session_path")); + wall.mockReturnValue(1); + advance(29_999); + diagnostics.unavailable("pane", new ConversationUnavailable("no_session_path")); + expect(records()).toHaveLength(1); + advance(1); + diagnostics.unavailable("pane", new ConversationUnavailable("no_session_path")); + expect(records()).toHaveLength(2); + } finally { wall.mockRestore(); } +}); + +it("accepts every code-owned Copilot diagnostic without accepting a forged stage", () => { + const { diagnostics, records } = fixture(); + for (const diagnostic of COPILOT_UNAVAILABLE_REASONS) { + diagnostics.unavailable(diagnostic, new ConversationUnavailable("no_session_path", undefined, diagnostic)); + } + const forged = new ConversationUnavailable("no_session_path"); + Object.defineProperty(forged, "diagnostic", { value: "private-token" }); + diagnostics.unavailable("forged", forged); + expect(records().map((record) => record.operation)).toEqual([ + ...COPILOT_UNAVAILABLE_REASONS.map((reason) => `conversation.${reason}`), "conversation.unknown", + ]); +}); + +it("does not change response handling when its writer fails", () => { + const diagnostics = new ConversationDiagnostics(() => { throw new Error("log unavailable"); }); + expect(() => diagnostics.unavailable("pane", new ConversationReadPending())).not.toThrow(); + expect(() => diagnostics.recovered("pane")).not.toThrow(); +}); diff --git a/server/conversation-diagnostics.ts b/server/conversation-diagnostics.ts new file mode 100644 index 0000000000..1ff82db4b7 --- /dev/null +++ b/server/conversation-diagnostics.ts @@ -0,0 +1,43 @@ +import { ConnectionDiagnostics } from "./connection-diagnostics.ts"; +import { COPILOT_UNAVAILABLE_REASONS, ConversationReadPending, ConversationUnavailable } from "./conversation-shared.ts"; + +const REASONS = [ + ...COPILOT_UNAVAILABLE_REASONS, "branch_unreadable", "transcript_missing", "no_session_path", + "no_session_id", "no_recognized_transcript", "pane_not_found", "session_not_written", "copilot_resume_unknown", +]; +const INTERVAL_MS = 30_000; +const MAX_LINES = 16; + +/** Keys stay private; only fixed reasons and server-generated request IDs leave the process. */ +export class ConversationDiagnostics { + private readonly episodes: ConnectionDiagnostics; + private windowAt = -Infinity; + private lines = 0; + + constructor(write: (line: string) => void = (line) => console.warn(line), private now: () => number = () => performance.now()) { + this.episodes = new ConnectionDiagnostics("conversation", (line) => { + this.lines++; + write(line); + }, () => performance.timeOrigin + now()); + } + + private available(): boolean { + const at = this.now(); + if (at - this.windowAt >= INTERVAL_MS) { this.windowAt = at; this.lines = 0; } + return this.lines < MAX_LINES; + } + + unavailable(key: string, error: ConversationUnavailable | ConversationReadPending, requestId?: string): void { + if (!this.available()) return; + const pending = error instanceof ConversationReadPending; + const reason = pending ? "read_pending" : REASONS.find((reason) => reason === (error.diagnostic ?? error.message)) ?? "unknown"; + this.episodes.failed(key, pending ? "conversation_read_pending" : "conversation_unavailable", { + operation: `conversation.${reason}`, request_id: requestId, + }); + } + + recovered(key: string, requestId?: string): void { + if (!this.available()) return; + this.episodes.recovered(key, { operation: "conversation", request_id: requestId }); + } +} diff --git a/server/conversation-shared.ts b/server/conversation-shared.ts index 4305c155e2..8011c14c9c 100644 --- a/server/conversation-shared.ts +++ b/server/conversation-shared.ts @@ -10,9 +10,15 @@ import type { ConversationMetadata, ConversationTurn } from "../shared/protocol. */ export const TRANSCRIPT_WINDOW_BYTES = 16 * 1024 * 1024; +export const COPILOT_UNAVAILABLE_REASONS = [ + "copilot_report_invalid", "copilot_process_lookup_failed", "copilot_process_set_unproved", + "copilot_peer_process_replaced", "copilot_handoff_unproved", "copilot_peer_conflict", + "copilot_foreground_unproved", "copilot_resume_unproved", +] as const; + export class ConversationUnavailable extends Error { /** `screen`: the model and effort the agent's own live status line states (start-screen.ts), for the scrollback standing in */ - constructor(reason: string, readonly screen?: ConversationMetadata) { + constructor(reason: string, readonly screen?: ConversationMetadata, readonly diagnostic?: typeof COPILOT_UNAVAILABLE_REASONS[number]) { super(reason); this.name = "ConversationUnavailable"; } diff --git a/server/conversation.ts b/server/conversation.ts index 12c7b82341..bc059acaed 100644 --- a/server/conversation.ts +++ b/server/conversation.ts @@ -1167,7 +1167,7 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[ peerInfo?.set(peer.pane_id, info ?? null); const processes = agentProcesses(info?.foreground_processes ?? [], "copilot"); // A cached peer process that has exited may have been replaced by one we never asked about. - if (cached && processes.some((entry) => typeof entry.pid === "number" && !processAlive(entry.pid))) throw new ConversationUnavailable("no_session_path"); + if (cached && processes.some((entry) => typeof entry.pid === "number" && !processAlive(entry.pid))) throw new ConversationUnavailable("no_session_path", undefined, "copilot_peer_process_replaced"); if (processes.length === 1 && typeof processes[0]?.pid === "number") { const live = copilotProcessSession(processEnviron(processes[0].pid), processes[0].argv ?? [], processes[0].pid); if (live?.logged) { @@ -1182,18 +1182,19 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[ const claimedByPeer = async (session: string): Promise => (await (peerClaims ??= gatherPeerClaims())).has(session); if (pane.agent !== "copilot" || (report != null && (report.agent !== "copilot" || report.kind !== "id" || report.source !== "herdr:copilot" || typeof report.value !== "string" || !SESSION_ID.test(report.value)))) { - throw new ConversationUnavailable("no_session_id"); + throw new ConversationUnavailable("no_session_id", undefined, "copilot_report_invalid"); } // A failed RPC does not prove an ownership change. Retry that read once; an answered // empty/conflicting process set still fails closed, as does a failed retry. const info = await paneProcessInfo(pane.pane_id).catch(() => paneProcessInfo(pane.pane_id)).catch(() => null); + if (info === null) throw new ConversationUnavailable("no_session_path", undefined, "copilot_process_lookup_failed"); const processes = agentProcesses(info?.foreground_processes ?? [], "copilot"); const foreground = processes[0]; if (process.platform !== "linux" || ![1, 2].includes(processes.length) || typeof foreground?.pid !== "number") { - throw new ConversationUnavailable("no_session_path"); + throw new ConversationUnavailable("no_session_path", undefined, "copilot_process_set_unproved"); } if (processes.length === 2) { - if (report == null) throw new ConversationUnavailable("no_session_id"); + if (report == null) throw new ConversationUnavailable("no_session_id", undefined, "copilot_report_invalid"); const parent = processes[0]!; const child = processes[1]!; const evidence = (entry: typeof parent) => ({ pid: entry.pid, argv: entry.argv ?? [] }); @@ -1206,7 +1207,7 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[ if (path !== null && nodePath.basename(nodePath.dirname(path)) === report.value) return { path, binding: selected.generation, peerClaims: await peerClaims! }; } - throw new ConversationUnavailable("no_session_path"); + throw new ConversationUnavailable("no_session_path", undefined, "copilot_handoff_unproved"); } const environment = processEnviron(foreground.pid); const argv = foreground.argv ?? []; @@ -1218,7 +1219,7 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[ return { path, binding: `${foreground.pid}:${logged.generation}`, peerClaims: await peerClaims! }; } if (report != null) { - if (await claimedByPeer(report.value)) throw new ConversationUnavailable("no_session_id"); + if (await claimedByPeer(report.value)) throw new ConversationUnavailable("no_session_id", undefined, "copilot_peer_conflict"); if (logged === null) { const path = copilotTranscriptFile(report.value, selected); if (path !== null && nodePath.basename(nodePath.dirname(path)) === report.value) @@ -1230,7 +1231,8 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[ && !(await claimedByPeer(pending.sessionId))) { throw new ConversationNotStarted(pending.sessionId, "copilot-transcript", pending.path); } - throw new ConversationUnavailable(copilotExplicitResume(argv) ? "copilot_resume_unknown" : report == null ? "no_session_id" : "no_session_path"); + throw new ConversationUnavailable(copilotExplicitResume(argv) ? "copilot_resume_unknown" : report == null ? "no_session_id" : "no_session_path", + undefined, copilotExplicitResume(argv) ? "copilot_resume_unproved" : "copilot_foreground_unproved"); } /** Where a pane's conversation is: a transcript file, or a session in OpenCode's database. */ diff --git a/server/copilot.test.ts b/server/copilot.test.ts index 42ff69212f..f2f7cf51e1 100644 --- a/server/copilot.test.ts +++ b/server/copilot.test.ts @@ -535,12 +535,12 @@ it.skipIf(process.platform !== "linux")("retries a failed Copilot process lookup expect(changed.turns[0]?.parts[0]).toEqual({ kind: "text", text: "changed history" }); unavailable = true; ownCalls = 0; - await expect(paneConversation("own")).rejects.toBeInstanceOf(ConversationUnavailable); + await expect(paneConversation("own")).rejects.toMatchObject({ diagnostic: "copilot_process_lookup_failed", message: "no_session_path" }); expect(ownCalls).toBe(2); unavailable = false; empty = true; ownCalls = 0; - await expect(paneConversation("own")).rejects.toBeInstanceOf(ConversationUnavailable); + await expect(paneConversation("own")).rejects.toMatchObject({ diagnostic: "copilot_process_set_unproved", message: "no_session_path" }); expect(ownCalls).toBe(1); } finally { info.mockRestore(); environment.mockRestore(); rpc.mockRestore(); snapshots.mockRestore(); diff --git a/server/pane-routes.test.ts b/server/pane-routes.test.ts index 4af67c0bdc..b1ebdabc54 100644 --- a/server/pane-routes.test.ts +++ b/server/pane-routes.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, expect, it } from "bun:test"; +import { afterEach, beforeEach, expect, it, spyOn } from "bun:test"; import { Database } from "bun:sqlite"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -10,6 +10,8 @@ import { findLastIndex } from "./dialogs/shared.ts"; import { handlePaneRequest } from "./pane-routes.ts"; import { paneAgentOf, resolveAgent } from "./pane-context.ts"; import { openCodeServiceOptions } from "./opencode-service.ts"; +import * as conversation from "./conversation.ts"; +import { requestIdContext } from "./http.ts"; // The real RPC client against a unix socket this test owns; only herdr's answers are synthetic. let root: string; @@ -58,6 +60,51 @@ afterEach(() => { const OMO_PROCESS = { pid: 2147483647, argv: ["omo"], cwd: "" }; +it("diagnoses fallback and recovery without changing the conversation or 304 response", async () => { + const id = crypto.randomUUID(); + const pane = `private-pane-${crypto.randomUUID()}`; + const url = new URL(`http://x/api/pane/conversation?pane_id=${pane}`); + const lines: string[] = []; + const warn = spyOn(console, "warn").mockImplementation((line: unknown) => { lines.push(String(line)); }); + const read = spyOn(conversation, "paneConversation").mockRejectedValueOnce( + new conversation.ConversationUnavailable("no_session_path", undefined, "copilot_process_lookup_failed"), + ).mockResolvedValue({ + source: "copilot-transcript", turns: [], metadata: { model: null, reasoning_effort: null }, + cursor: null, history_id: "private-history", version: "fixture", + }); + const ask = (headers?: HeadersInit) => requestIdContext.run({ id, route: "unmatched" }, () => + handlePaneRequest(new Request(url, { headers }), url, "/api/pane/conversation", {})); + try { + const fallback = (await ask())!; + expect(fallback.status).toBe(200); + expect(await fallback.json()).toEqual({ source: "scrollback", turns: [] }); + const recovered = (await ask({ "if-none-match": 'W/"fixture"' }))!; + expect(recovered.status).toBe(304); + expect(recovered.headers.get("etag")).toBe('W/"fixture"'); + const records = lines.map((line) => JSON.parse(line)); + expect(records[0]).toMatchObject({ event: "conversation.failed", operation: "conversation.copilot_process_lookup_failed", request_id: id }); + expect(records[1]).toMatchObject({ event: "conversation.recovered", episode_id: records[0].episode_id, request_id: id }); + expect(lines.join("\n")).not.toContain(pane); + expect(lines.join("\n")).not.toContain("private-history"); + } finally { read.mockRestore(); warn.mockRestore(); } +}); + +it("diagnoses pending reads while preserving the retry response", async () => { + const url = new URL(`http://x/api/pane/conversation?pane_id=${crypto.randomUUID()}`); + const lines: string[] = []; + const warn = spyOn(console, "warn").mockImplementation((line: unknown) => { lines.push(String(line)); }); + const read = spyOn(conversation, "paneConversation").mockRejectedValue(new conversation.ConversationReadPending()); + try { + const response = (await handlePaneRequest(new Request(url), url, "/api/pane/conversation", {}))!; + expect(response.status).toBe(503); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(await response.json()).toEqual({ error: { + code: "conversation_read_pending", message: "The conversation could not be read; try again shortly.", + } }); + expect(JSON.parse(lines[0]!)).toMatchObject({ event: "conversation.failed", operation: "conversation.read_pending" }); + } finally { read.mockRestore(); warn.mockRestore(); } +}); + it("names an omo pane that herdr labels pi omo in the commands route, and types into it as omo only once omo holds a session", async () => { panes["w:p1"] = { agent: "pi", process: [{ ...OMO_PROCESS, cwd: root }] }; panes["w:p2"] = { agent: "pi", process: [{ pid: 2147483646, argv: ["/usr/bin/pi"], cwd: root }] }; diff --git a/server/pane-routes.ts b/server/pane-routes.ts index 12f86f47be..7c72ad5164 100644 --- a/server/pane-routes.ts +++ b/server/pane-routes.ts @@ -8,7 +8,7 @@ import type { AgentRenameRequest, PaneDirection, PaneResized, PaneSplit, PaneSwa import { mergeCatalogSources } from "./catalog-sources.ts"; import { paneCommands, rememberPaneCommands } from "./commands.ts"; import { paneFiles } from "./files.ts"; -import { badRequest, conflictResponse, errorResponse, isCount, isJsonObject, jsonResponse, readJsonObject, setRequestRoute } from "./http.ts"; +import { badRequest, conflictResponse, errorResponse, isCount, isJsonObject, jsonResponse, readJsonObject, requestIdContext, setRequestRoute } from "./http.ts"; import { conversationImage, ConversationReadPending, ConversationUnavailable, forgetPaneTranscriptState, HistoryChanged, paneConversation, paneProcessRoots, paneTranscript, toolOutput } from "./conversation.ts"; import { OPENCODE_TOOL_REF } from "./opencode.ts"; import { openCodeInstalledVersion } from "./opencode-service.ts"; @@ -19,6 +19,9 @@ import { PasteImageError, readImageBody, savePaneImage } from "./paste.ts"; import { CLAUDE_REWIND_OPEN_MESSAGE } from "./dialogs/claude.ts"; import { claudeRewindOpen } from "./prompt.ts"; import { expandedDirectory } from "./workspace-routes.ts"; +import { ConversationDiagnostics } from "./conversation-diagnostics.ts"; + +const conversationDiagnostics = new ConversationDiagnostics(); /** the native stores createServer was given; unset, each reader finds its own */ export type PaneStores = { codexHome?: string; grokHome?: string; museData?: string; opencodeDb?: string; devinDbPath?: string; hermesHome?: string; openCodeStateDir?: string }; @@ -260,6 +263,7 @@ export async function handlePaneRequest(request: Request, url: URL, pathname: st }; try { const { version, ...conversation } = await paneConversation(paneId, options.codexHome, page, options.devinDbPath, options.opencodeDb, options.hermesHome, options.grokHome, options.museData); + conversationDiagnostics.recovered(paneId, requestIdContext.getStore()?.id); // no-store keeps the browser's own cache out of it, so the chat sees the 304. // weak: the same answer goes out gzipped or plain (compress.ts), which are not the same bytes const etag = `W/"${version}"`; @@ -268,11 +272,17 @@ export async function handlePaneRequest(request: Request, url: URL, pathname: st return jsonResponse(conversation, 200, headers); } catch (error) { if (error instanceof HistoryChanged || error instanceof DevinHistoryChanged) return jsonResponse({ error: { code: "history_changed", message: error.message } }, 409); - if (error instanceof ConversationReadPending) return jsonResponse({ error: { code: "conversation_read_pending", message: error.message } }, 503, { "cache-control": "no-store" }); + if (error instanceof ConversationReadPending) { + conversationDiagnostics.unavailable(paneId, error, requestIdContext.getStore()?.id); + return jsonResponse({ error: { code: "conversation_read_pending", message: error.message } }, 503, { "cache-control": "no-store" }); + } // an unrecognized pane is not an error: the client falls back to the // scrollback transcript, exactly like chatmux's terminal fallback. Its metadata is only // ever what the agent's own status line states - if (error instanceof ConversationUnavailable) return jsonResponse({ source: "scrollback", turns: [], ...(error.screen ? { metadata: error.screen, screen: true } : {}) }); + if (error instanceof ConversationUnavailable) { + conversationDiagnostics.unavailable(paneId, error, requestIdContext.getStore()?.id); + return jsonResponse({ source: "scrollback", turns: [], ...(error.screen ? { metadata: error.screen, screen: true } : {}) }); + } return errorResponse(error); } } From edb82ec07d8000cf5bf277ffaf8a4186ec2841ac Mon Sep 17 00:00:00 2001 From: devswha Date: Thu, 29 Oct 2026 07:03:46 +0900 Subject: [PATCH 2/4] fix(diagnostics): keep shell panes out of the failure log and episodes intact under the line cap --- server/connection-diagnostics.ts | 13 ++++++++---- server/conversation-diagnostics.test.ts | 27 +++++++++++++++++++++---- server/conversation-diagnostics.ts | 11 ++++++++-- server/pane-routes.test.ts | 16 ++++++++++++++- 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/server/connection-diagnostics.ts b/server/connection-diagnostics.ts index 75ba9968c0..f6a29b17d2 100644 --- a/server/connection-diagnostics.ts +++ b/server/connection-diagnostics.ts @@ -39,7 +39,8 @@ export class ConnectionDiagnostics { constructor( private component: "herdr.rpc" | "machine.local" | "machine.remote" | "websocket" | "conversation", - private write: (line: string) => void = (line) => console.warn(line), + /** a budgeted sink returns false to decline a line: the failure then stays unseen */ + private write: (line: string) => unknown = (line) => console.warn(line), private now: () => number = Date.now, ) {} @@ -66,9 +67,13 @@ export class ConnectionDiagnostics { failure.suppressed++; return; } + // A declined line shows nothing, so no recovery may follow it; the next failure tries again. + if (!this.emit("failed", failure, fields, now)) { + failure.suppressed++; + return; + } failure.loggedAt = now; failure.visible = true; - this.emit("failed", failure, fields, now); failure.suppressed = 0; } @@ -85,7 +90,7 @@ export class ConnectionDiagnostics { failure.suppressed = 0; } - private emit(event: "failed" | "recovered", failure: Failure, fields: Fields, now: number): void { + private emit(event: "failed" | "recovered", failure: Failure, fields: Fields, now: number): boolean { // Pick fields explicitly. Never spread an exception, request, response, or caller's object. const record: Record = { time: new Date(now).toISOString(), event: `${this.component}.${event}`, episode_id: failure.id, @@ -100,6 +105,6 @@ export class ConnectionDiagnostics { const value = fields[field]; if (typeof value === "number" && Number.isFinite(value) && value >= 0) record[field] = Math.round(value); } - try { this.write(JSON.stringify(record)); } catch { /* diagnostics must not change delivery or cleanup */ } + try { return this.write(JSON.stringify(record)) !== false; } catch { /* diagnostics must not change delivery or cleanup */ return true; } } } diff --git a/server/conversation-diagnostics.test.ts b/server/conversation-diagnostics.test.ts index 5c7b0f837b..ed5cc4b0df 100644 --- a/server/conversation-diagnostics.test.ts +++ b/server/conversation-diagnostics.test.ts @@ -46,12 +46,31 @@ it("bounds output even when pane keys rotate past the retained episode limit", ( for (let i = 0; i < 300; i++) diagnostics.unavailable(`pane-${i}`, new ConversationUnavailable("no_session_id")); expect(records()).toHaveLength(16); advance(30_000); - diagnostics.recovered("pane-299"); + diagnostics.recovered("pane-299"); // its failure was never shown: no orphan recovery + diagnostics.recovered("pane-0"); // retired with the oldest keys: nothing left to close expect(records()).toHaveLength(16); - diagnostics.recovered("pane-0"); - expect(records()[16]).toMatchObject({ event: "conversation.recovered", episode_id: records()[0].episode_id }); diagnostics.unavailable("new-pane", new ConversationUnavailable("no_session_id")); - expect(records()).toHaveLength(18); + expect(records()).toHaveLength(17); +}); + +it("keeps episodes apart while the line budget is spent", () => { + const { diagnostics, records, advance } = fixture(); + for (let i = 0; i < 16; i++) diagnostics.unavailable(`pane-${i}`, new ConversationUnavailable("no_session_path")); + diagnostics.recovered("pane-0"); // no line left in this window, but the episode still ends + expect(records()).toHaveLength(16); + advance(30_000); + diagnostics.unavailable("pane-0", new ConversationUnavailable("no_session_path")); + expect(records()[16]).toMatchObject({ event: "conversation.failed", failures: 1 }); + expect(records()[16].episode_id).not.toBe(records()[0].episode_id); +}); + +it("does not count a pane without an agent conversation as a failure", () => { + const { diagnostics, records } = fixture(); + for (let i = 0; i < 50; i++) diagnostics.unavailable(`shell-${i}`, new ConversationUnavailable("no_recognized_transcript")); + expect(records()).toEqual([]); + diagnostics.unavailable("copilot", new ConversationUnavailable("no_session_path", undefined, "copilot_process_lookup_failed")); + diagnostics.unavailable("copilot", new ConversationUnavailable("no_recognized_transcript")); // its agent left: the episode ends + expect(records().map((record) => record.event)).toEqual(["conversation.failed", "conversation.recovered"]); }); it("keeps the logging interval monotonic across wall-clock corrections", () => { diff --git a/server/conversation-diagnostics.ts b/server/conversation-diagnostics.ts index 1ff82db4b7..93f06eed68 100644 --- a/server/conversation-diagnostics.ts +++ b/server/conversation-diagnostics.ts @@ -5,6 +5,8 @@ const REASONS = [ ...COPILOT_UNAVAILABLE_REASONS, "branch_unreadable", "transcript_missing", "no_session_path", "no_session_id", "no_recognized_transcript", "pane_not_found", "session_not_written", "copilot_resume_unknown", ]; +/** A pane with no agent conversation (a plain shell): its scrollback is the expected view, not a failure. */ +const EXPECTED = new Set(["no_recognized_transcript"]); const INTERVAL_MS = 30_000; const MAX_LINES = 16; @@ -15,9 +17,12 @@ export class ConversationDiagnostics { private lines = 0; constructor(write: (line: string) => void = (line) => console.warn(line), private now: () => number = () => performance.now()) { + // The budget bounds the output only: episodes still open and close while it is spent. this.episodes = new ConnectionDiagnostics("conversation", (line) => { + if (!this.available()) return false; this.lines++; write(line); + return true; }, () => performance.timeOrigin + now()); } @@ -28,7 +33,10 @@ export class ConversationDiagnostics { } unavailable(key: string, error: ConversationUnavailable | ConversationReadPending, requestId?: string): void { - if (!this.available()) return; + if (error instanceof ConversationUnavailable && error.diagnostic === undefined && EXPECTED.has(error.message)) { + this.recovered(key, requestId); + return; + } const pending = error instanceof ConversationReadPending; const reason = pending ? "read_pending" : REASONS.find((reason) => reason === (error.diagnostic ?? error.message)) ?? "unknown"; this.episodes.failed(key, pending ? "conversation_read_pending" : "conversation_unavailable", { @@ -37,7 +45,6 @@ export class ConversationDiagnostics { } recovered(key: string, requestId?: string): void { - if (!this.available()) return; this.episodes.recovered(key, { operation: "conversation", request_id: requestId }); } } diff --git a/server/pane-routes.test.ts b/server/pane-routes.test.ts index b1ebdabc54..5bc922a964 100644 --- a/server/pane-routes.test.ts +++ b/server/pane-routes.test.ts @@ -99,12 +99,26 @@ it("diagnoses pending reads while preserving the retry response", async () => { expect(response.status).toBe(503); expect(response.headers.get("cache-control")).toBe("no-store"); expect(await response.json()).toEqual({ error: { - code: "conversation_read_pending", message: "The conversation could not be read; try again shortly.", + code: "conversation_read_pending", message: new conversation.ConversationReadPending().message, } }); expect(JSON.parse(lines[0]!)).toMatchObject({ event: "conversation.failed", operation: "conversation.read_pending" }); } finally { read.mockRestore(); warn.mockRestore(); } }); +it("does not report a plain shell's scrollback view as a conversation failure", async () => { + panes["w:shell"] = { process: [{ pid: 2147483645, argv: ["/bin/bash"], cwd: root }] }; + const url = new URL("http://x/api/pane/conversation?pane_id=w:shell"); + const lines: string[] = []; + const warn = spyOn(console, "warn").mockImplementation((line: unknown) => { lines.push(String(line)); }); + try { + for (let i = 0; i < 3; i++) { + const response = (await handlePaneRequest(new Request(url), url, "/api/pane/conversation", {}))!; + expect(await response.json()).toMatchObject({ source: "scrollback", turns: [] }); + } + expect(lines.filter((line) => line.includes("conversation."))).toEqual([]); + } finally { warn.mockRestore(); } +}); + it("names an omo pane that herdr labels pi omo in the commands route, and types into it as omo only once omo holds a session", async () => { panes["w:p1"] = { agent: "pi", process: [{ ...OMO_PROCESS, cwd: root }] }; panes["w:p2"] = { agent: "pi", process: [{ pid: 2147483646, argv: ["/usr/bin/pi"], cwd: root }] }; From 44ef36f1c6dc4248b29c274705321e5c8f015ec8 Mon Sep 17 00:00:00 2001 From: Yu Xie Date: Wed, 28 Oct 2026 23:06:40 +0100 Subject: [PATCH 3/4] fix(chat): distinguish peer conflicts in Copilot handoffs Preserve the existing rejection and HTTP behavior while recording the proven conflict stage. Add a contract assertion that failed before the correction. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1fad73aa-cbc5-42dd-916b-64e5198bc512 --- server/conversation.ts | 3 ++- server/copilot.contract.test.ts | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/conversation.ts b/server/conversation.ts index bc059acaed..e79748eca5 100644 --- a/server/conversation.ts +++ b/server/conversation.ts @@ -1202,7 +1202,8 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[ report.value, info?.foreground_process_group_id, info?.tty, processEnviron) ?? copilotHandoffSession(evidence(child), evidence(parent), report.value, info?.foreground_process_group_id, info?.tty, processEnviron); - if (selected?.logged && !(await claimedByPeer(selected.sessionId))) { + if (selected?.logged) { + if (await claimedByPeer(selected.sessionId)) throw new ConversationUnavailable("no_session_path", undefined, "copilot_peer_conflict"); const path = copilotTranscriptFile(report.value, selected); if (path !== null && nodePath.basename(nodePath.dirname(path)) === report.value) return { path, binding: selected.generation, peerClaims: await peerClaims! }; diff --git a/server/copilot.contract.test.ts b/server/copilot.contract.test.ts index 125e185106..d9f2ddf496 100644 --- a/server/copilot.contract.test.ts +++ b/server/copilot.contract.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { createServer } from "./index.ts"; import { herdrRpc, paneProcessInfo, sessionSnapshot, workspaceClose, workspaceCreate } from "./herdr/client.ts"; -import { forgetTranscriptState } from "./conversation.ts"; +import { forgetTranscriptState, paneConversation } from "./conversation.ts"; import { copilotHandoffSession, copilotProcessSession } from "./copilot-store.ts"; import { processEnviron } from "./omo.ts"; import * as processStart from "./process-start.ts"; @@ -195,6 +195,9 @@ const handoffTest = async () => { await startFixture(duplicate.root_pane.pane_id, handoffId); await herdrRpc("pane.report_agent_session", { pane_id: duplicate.root_pane.pane_id, agent: "copilot", source: "herdr:copilot", seq: Date.now() * 1000 + 1, agent_session_id: handoffId }); + await expect(paneConversation(current)).rejects.toMatchObject({ + message: "no_session_path", diagnostic: "copilot_peer_conflict", + }); expect((await read()).source).toBe("scrollback"); } finally { await workspaceClose(duplicate.workspace.workspace_id); } expect((await read()).turns[0]?.parts[0]?.text).toBe("handoff history"); From 2ddad703269a9a62e95bee69bba0f2133291e10e Mon Sep 17 00:00:00 2001 From: devswha Date: Thu, 29 Oct 2026 07:10:32 +0900 Subject: [PATCH 4/4] fix(diagnostics): report an agent pane without a folder instead of treating it as a shell --- server/conversation-diagnostics.test.ts | 10 ++++++++++ server/conversation-diagnostics.ts | 4 ++-- server/conversation-shared.ts | 4 +++- server/conversation.ts | 3 ++- server/pane-routes.test.ts | 17 +++++++++++++++-- 5 files changed, 32 insertions(+), 6 deletions(-) diff --git a/server/conversation-diagnostics.test.ts b/server/conversation-diagnostics.test.ts index ed5cc4b0df..318470fbaf 100644 --- a/server/conversation-diagnostics.test.ts +++ b/server/conversation-diagnostics.test.ts @@ -64,6 +64,16 @@ it("keeps episodes apart while the line budget is spent", () => { expect(records()[16].episode_id).not.toBe(records()[0].episode_id); }); +it("keeps an agent pane without a folder in its failure episode", () => { + const { diagnostics, records, advance } = fixture(); + diagnostics.unavailable("copilot", new ConversationUnavailable("no_session_path", undefined, "copilot_process_lookup_failed")); + diagnostics.unavailable("copilot", new ConversationUnavailable("no_recognized_transcript", undefined, "agent_cwd_missing")); + advance(30_000); + diagnostics.unavailable("copilot", new ConversationUnavailable("no_recognized_transcript", undefined, "agent_cwd_missing")); + expect(records().map((record) => record.event)).toEqual(["conversation.failed", "conversation.failed"]); + expect(records()[1]).toMatchObject({ operation: "conversation.agent_cwd_missing", episode_id: records()[0].episode_id, failures: 3 }); +}); + it("does not count a pane without an agent conversation as a failure", () => { const { diagnostics, records } = fixture(); for (let i = 0; i < 50; i++) diagnostics.unavailable(`shell-${i}`, new ConversationUnavailable("no_recognized_transcript")); diff --git a/server/conversation-diagnostics.ts b/server/conversation-diagnostics.ts index 93f06eed68..cec2a99c93 100644 --- a/server/conversation-diagnostics.ts +++ b/server/conversation-diagnostics.ts @@ -1,8 +1,8 @@ import { ConnectionDiagnostics } from "./connection-diagnostics.ts"; -import { COPILOT_UNAVAILABLE_REASONS, ConversationReadPending, ConversationUnavailable } from "./conversation-shared.ts"; +import { UNAVAILABLE_DIAGNOSTICS, ConversationReadPending, ConversationUnavailable } from "./conversation-shared.ts"; const REASONS = [ - ...COPILOT_UNAVAILABLE_REASONS, "branch_unreadable", "transcript_missing", "no_session_path", + ...UNAVAILABLE_DIAGNOSTICS, "branch_unreadable", "transcript_missing", "no_session_path", "no_session_id", "no_recognized_transcript", "pane_not_found", "session_not_written", "copilot_resume_unknown", ]; /** A pane with no agent conversation (a plain shell): its scrollback is the expected view, not a failure. */ diff --git a/server/conversation-shared.ts b/server/conversation-shared.ts index 8011c14c9c..56fa946c52 100644 --- a/server/conversation-shared.ts +++ b/server/conversation-shared.ts @@ -15,10 +15,12 @@ export const COPILOT_UNAVAILABLE_REASONS = [ "copilot_peer_process_replaced", "copilot_handoff_unproved", "copilot_peer_conflict", "copilot_foreground_unproved", "copilot_resume_unproved", ] as const; +/** Fixed stages a diagnostic may name: Copilot's rejections, and an agent pane herdr gives no folder. */ +export const UNAVAILABLE_DIAGNOSTICS = [...COPILOT_UNAVAILABLE_REASONS, "agent_cwd_missing"] as const; export class ConversationUnavailable extends Error { /** `screen`: the model and effort the agent's own live status line states (start-screen.ts), for the scrollback standing in */ - constructor(reason: string, readonly screen?: ConversationMetadata, readonly diagnostic?: typeof COPILOT_UNAVAILABLE_REASONS[number]) { + constructor(reason: string, readonly screen?: ConversationMetadata, readonly diagnostic?: typeof UNAVAILABLE_DIAGNOSTICS[number]) { super(reason); this.name = "ConversationUnavailable"; } diff --git a/server/conversation.ts b/server/conversation.ts index bc059acaed..47b6b8e255 100644 --- a/server/conversation.ts +++ b/server/conversation.ts @@ -1370,7 +1370,8 @@ export async function paneTranscript(paneId: string, codexHome?: string, snapsho const panes = (snapshot ?? await sessionSnapshot()).panes; const pane = panes.find((candidate) => candidate.pane_id === paneId); if (pane === undefined) throw new ConversationUnavailable("pane_not_found"); - if (typeof pane.cwd !== "string" || pane.cwd.length === 0) throw new ConversationUnavailable("no_recognized_transcript"); + // an agent pane without a folder is a failure to diagnose, not a shell's expected scrollback + if (typeof pane.cwd !== "string" || pane.cwd.length === 0) throw new ConversationUnavailable("no_recognized_transcript", undefined, pane.agent ? "agent_cwd_missing" : undefined); if (pane.agent === "devin" && !(await paneRunsOmo(paneId, known))) { const cwd = pane.foreground_cwd || pane.cwd; diff --git a/server/pane-routes.test.ts b/server/pane-routes.test.ts index 5bc922a964..d74a28cbcd 100644 --- a/server/pane-routes.test.ts +++ b/server/pane-routes.test.ts @@ -17,10 +17,10 @@ import { requestIdContext } from "./http.ts"; let root: string; let listener: ReturnType; let socketBefore: string | undefined; -let panes: Record; +let panes: Record; const paneInfo = (id: string) => ({ - pane_id: id, workspace_id: "w", tab_id: "w:t1", terminal_id: id, cwd: root, focused: false, revision: 0, + pane_id: id, workspace_id: "w", tab_id: "w:t1", terminal_id: id, cwd: panes[id]?.cwd ?? root, focused: false, revision: 0, ...(panes[id]?.agent ? { agent: panes[id]!.agent } : {}), ...(panes[id]?.session ? { agent_session: { agent: "opencode", source: "herdr:opencode", kind: "id", value: panes[id]!.session } } : {}), }); @@ -119,6 +119,19 @@ it("does not report a plain shell's scrollback view as a conversation failure", } finally { warn.mockRestore(); } }); +it("still reports an agent pane herdr gives no folder, and does not count it as a recovery", async () => { + panes["w:nocwd"] = { agent: "copilot", cwd: "", process: [] }; + const url = new URL("http://x/api/pane/conversation?pane_id=w:nocwd"); + const lines: string[] = []; + const warn = spyOn(console, "warn").mockImplementation((line: unknown) => { lines.push(String(line)); }); + try { + const response = (await handlePaneRequest(new Request(url), url, "/api/pane/conversation", {}))!; + expect(await response.json()).toMatchObject({ source: "scrollback", turns: [] }); + const records = lines.filter((line) => line.includes("conversation.")).map((line) => JSON.parse(line)); + expect(records).toEqual([expect.objectContaining({ event: "conversation.failed", operation: "conversation.agent_cwd_missing" })]); + } finally { warn.mockRestore(); } +}); + it("names an omo pane that herdr labels pi omo in the commands route, and types into it as omo only once omo holds a session", async () => { panes["w:p1"] = { agent: "pi", process: [{ ...OMO_PROCESS, cwd: root }] }; panes["w:p2"] = { agent: "pi", process: [{ pid: 2147483646, argv: ["/usr/bin/pi"], cwd: root }] };