fix(chat): add bounded diagnostics for conversation fallbacks

changelog.d/conversation-unavailable-diagnostics.md

@@ -0,0 +1,3 @@
+### Maintenance
+- Conversation fallbacks and recoveries now emit sanitized, rate-limited diagnostic episodes,
+  including the failed Copilot identity stage without logging session IDs or transcript paths.

server/connection-diagnostics.ts

@@ -4,7 +4,7 @@ const REPEAT_MS = 30_000;
 const MAX_FAILURES = 128;
 const TRANSPORT_CODES = ["timeout", "connect_failed", "socket_error", "closed", "bad_frame", "server_unavailable"] as const;
 type TransportCode = typeof TRANSPORT_CODES[number];
-type FailureCode = TransportCode | "refresh_failed" | "connection_lost" | "action_required";
+type FailureCode = TransportCode | "refresh_failed" | "connection_lost" | "action_required" | "conversation_unavailable" | "conversation_read_pending";
 
 export function transportFailureCode(code: string): TransportCode | null {
   return TRANSPORT_CODES.find((known) => known === code) ?? null;
@@ -33,13 +33,14 @@ interface Failure {
   visible: boolean;
 }
 
-/** Connection failures only: no access log, input, credentials, peer names, or exception text. */
+/** Bounded failure episodes: no access log, input, credentials, peer names, or exception text. */
 export class ConnectionDiagnostics {
   private failures = new Map<string, Failure>();
 
   constructor(
-    private component: "herdr.rpc" | "machine.local" | "machine.remote" | "websocket",
-    private write: (line: string) => void = (line) => console.warn(line),
+    private component: "herdr.rpc" | "machine.local" | "machine.remote" | "websocket" | "conversation",
+    /** a budgeted sink returns false to decline a line: the failure then stays unseen */
+    private write: (line: string) => unknown = (line) => console.warn(line),
     private now: () => number = Date.now,
   ) {}
 
@@ -66,9 +67,13 @@ export class ConnectionDiagnostics {
       failure.suppressed++;
       return;
     }
+    // A declined line shows nothing, so no recovery may follow it; the next failure tries again.
+    if (!this.emit("failed", failure, fields, now)) {
+      failure.suppressed++;
+      return;
+    }
     failure.loggedAt = now;
     failure.visible = true;
-    this.emit("failed", failure, fields, now);
     failure.suppressed = 0;
   }
 
@@ -85,21 +90,21 @@ export class ConnectionDiagnostics {
     failure.suppressed = 0;
   }
 
-  private emit(event: "failed" | "recovered", failure: Failure, fields: Fields, now: number): void {
+  private emit(event: "failed" | "recovered", failure: Failure, fields: Fields, now: number): boolean {
     // Pick fields explicitly. Never spread an exception, request, response, or caller's object.
     const record: Record<string, string | number> = {
       time: new Date(now).toISOString(), event: `${this.component}.${event}`, episode_id: failure.id,
       operation: /^[a-z][a-z0-9_.]{0,95}$/.test(fields.operation) ? fields.operation : "unknown",
       code: failure.code, failures: failure.count, suppressed: failure.suppressed,
       since_first_failure_ms: Math.max(0, now - failure.since),
     };
-    if (fields.request_id && /^br-[a-z0-9]+-[0-9]+$/.test(fields.request_id)) record.request_id = fields.request_id;
+    if (fields.request_id && (/^br-[a-z0-9]+-[0-9]+$/.test(fields.request_id) || /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(fields.request_id))) record.request_id = fields.request_id;
     if (fields.machine_id && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(fields.machine_id)) record.machine_id = fields.machine_id;
     if (fields.phase === "connect" || fields.phase === "response") record.phase = fields.phase;
     for (const field of ["elapsed_ms", "connect_ms", "timeout_ms", "pending", "close_code"] as const) {
       const value = fields[field];
       if (typeof value === "number" && Number.isFinite(value) && value >= 0) record[field] = Math.round(value);
     }
-    try { this.write(JSON.stringify(record)); } catch { /* diagnostics must not change delivery or cleanup */ }
+    try { return this.write(JSON.stringify(record)) !== false; } catch { /* diagnostics must not change delivery or cleanup */ return true; }
   }
 }

server/conversation-diagnostics.test.ts

@@ -0,0 +1,118 @@
+import { expect, it, spyOn } from "bun:test";
+import { ConversationDiagnostics } from "./conversation-diagnostics.ts";
+import { COPILOT_UNAVAILABLE_REASONS, ConversationReadPending, ConversationUnavailable } from "./conversation-shared.ts";
+
+function fixture() {
+  let now = 0;
+  const lines: string[] = [];
+  const diagnostics = new ConversationDiagnostics((line) => lines.push(line), () => now);
+  return { diagnostics, lines, records: () => lines.map((line) => JSON.parse(line)), advance: (ms: number) => { now += ms; } };
+}
+
+it("logs only fixed failure stages and opaque request correlation, never exception or pane contents", () => {
+  const { diagnostics, records, lines } = fixture();
+  const secret = "/private/session-token\ntranscript";
+  const error = new ConversationUnavailable(secret, undefined, "copilot_process_lookup_failed");
+  Object.assign(error, { path: secret, sessionId: secret });
+  diagnostics.unavailable(secret, error, "00000000-0000-4000-8000-000000000001");
+  diagnostics.unavailable("other pane", new ConversationUnavailable(secret), secret);
+  expect(records()[0]).toMatchObject({
+    event: "conversation.failed", operation: "conversation.copilot_process_lookup_failed",
+    code: "conversation_unavailable", request_id: "00000000-0000-4000-8000-000000000001",
+  });
+  expect(records()[1]).toMatchObject({ operation: "conversation.unknown" });
+  expect(records()[1].request_id).toBeUndefined();
+  expect(lines.join("\n")).not.toContain("private");
+  expect(lines.join("\n")).not.toContain("session-token");
+});
+
+it("coalesces repeated reasons and rapid recoveries without losing the episode correlation", () => {
+  const { diagnostics, records, advance } = fixture();
+  for (let i = 0; i < 100; i++) {
+    diagnostics.unavailable("pane", new ConversationUnavailable("no_session_path"));
+    diagnostics.recovered("pane");
+  }
+  expect(records()).toHaveLength(2);
+  expect(records()[1]).toMatchObject({ event: "conversation.recovered", episode_id: records()[0].episode_id });
+  advance(30_000);
+  diagnostics.unavailable("pane", new ConversationReadPending());
+  expect(records()[2]).toMatchObject({
+    operation: "conversation.read_pending", code: "conversation_read_pending", failures: 100, suppressed: 99,
+  });
+});
+
+it("bounds output even when pane keys rotate past the retained episode limit", () => {
+  const { diagnostics, records, advance } = fixture();
+  for (let i = 0; i < 300; i++) diagnostics.unavailable(`pane-${i}`, new ConversationUnavailable("no_session_id"));
+  expect(records()).toHaveLength(16);
+  advance(30_000);
+  diagnostics.recovered("pane-299"); // its failure was never shown: no orphan recovery
+  diagnostics.recovered("pane-0"); // retired with the oldest keys: nothing left to close
+  expect(records()).toHaveLength(16);
+  diagnostics.unavailable("new-pane", new ConversationUnavailable("no_session_id"));
+  expect(records()).toHaveLength(17);
+});
+
+it("keeps episodes apart while the line budget is spent", () => {
+  const { diagnostics, records, advance } = fixture();
+  for (let i = 0; i < 16; i++) diagnostics.unavailable(`pane-${i}`, new ConversationUnavailable("no_session_path"));
+  diagnostics.recovered("pane-0"); // no line left in this window, but the episode still ends
+  expect(records()).toHaveLength(16);
+  advance(30_000);
+  diagnostics.unavailable("pane-0", new ConversationUnavailable("no_session_path"));
+  expect(records()[16]).toMatchObject({ event: "conversation.failed", failures: 1 });
+  expect(records()[16].episode_id).not.toBe(records()[0].episode_id);
+});
+
+it("keeps an agent pane without a folder in its failure episode", () => {
+  const { diagnostics, records, advance } = fixture();
+  diagnostics.unavailable("copilot", new ConversationUnavailable("no_session_path", undefined, "copilot_process_lookup_failed"));
+  diagnostics.unavailable("copilot", new ConversationUnavailable("no_recognized_transcript", undefined, "agent_cwd_missing"));
+  advance(30_000);
+  diagnostics.unavailable("copilot", new ConversationUnavailable("no_recognized_transcript", undefined, "agent_cwd_missing"));
+  expect(records().map((record) => record.event)).toEqual(["conversation.failed", "conversation.failed"]);
+  expect(records()[1]).toMatchObject({ operation: "conversation.agent_cwd_missing", episode_id: records()[0].episode_id, failures: 3 });
+});
+
+it("does not count a pane without an agent conversation as a failure", () => {
+  const { diagnostics, records } = fixture();
+  for (let i = 0; i < 50; i++) diagnostics.unavailable(`shell-${i}`, new ConversationUnavailable("no_recognized_transcript"));
+  expect(records()).toEqual([]);
+  diagnostics.unavailable("copilot", new ConversationUnavailable("no_session_path", undefined, "copilot_process_lookup_failed"));
+  diagnostics.unavailable("copilot", new ConversationUnavailable("no_recognized_transcript")); // its agent left: the episode ends
+  expect(records().map((record) => record.event)).toEqual(["conversation.failed", "conversation.recovered"]);
+});
+
+it("keeps the logging interval monotonic across wall-clock corrections", () => {
+  const { diagnostics, records, advance } = fixture();
+  const wall = spyOn(Date, "now").mockReturnValue(100_000);
+  try {
+    diagnostics.unavailable("pane", new ConversationUnavailable("no_session_path"));
+    wall.mockReturnValue(1);
+    advance(29_999);
+    diagnostics.unavailable("pane", new ConversationUnavailable("no_session_path"));
+    expect(records()).toHaveLength(1);
+    advance(1);
+    diagnostics.unavailable("pane", new ConversationUnavailable("no_session_path"));
+    expect(records()).toHaveLength(2);
+  } finally { wall.mockRestore(); }
+});
+
+it("accepts every code-owned Copilot diagnostic without accepting a forged stage", () => {
+  const { diagnostics, records } = fixture();
+  for (const diagnostic of COPILOT_UNAVAILABLE_REASONS) {
+    diagnostics.unavailable(diagnostic, new ConversationUnavailable("no_session_path", undefined, diagnostic));
+  }
+  const forged = new ConversationUnavailable("no_session_path");
+  Object.defineProperty(forged, "diagnostic", { value: "private-token" });
+  diagnostics.unavailable("forged", forged);
+  expect(records().map((record) => record.operation)).toEqual([
+    ...COPILOT_UNAVAILABLE_REASONS.map((reason) => `conversation.${reason}`), "conversation.unknown",
+  ]);
+});
+
+it("does not change response handling when its writer fails", () => {
+  const diagnostics = new ConversationDiagnostics(() => { throw new Error("log unavailable"); });
+  expect(() => diagnostics.unavailable("pane", new ConversationReadPending())).not.toThrow();
+  expect(() => diagnostics.recovered("pane")).not.toThrow();
+});

server/conversation-diagnostics.ts

@@ -0,0 +1,50 @@
+import { ConnectionDiagnostics } from "./connection-diagnostics.ts";
+import { UNAVAILABLE_DIAGNOSTICS, ConversationReadPending, ConversationUnavailable } from "./conversation-shared.ts";
+
+const REASONS = [
+  ...UNAVAILABLE_DIAGNOSTICS, "branch_unreadable", "transcript_missing", "no_session_path",
+  "no_session_id", "no_recognized_transcript", "pane_not_found", "session_not_written", "copilot_resume_unknown",
+];
+/** A pane with no agent conversation (a plain shell): its scrollback is the expected view, not a failure. */
+const EXPECTED = new Set(["no_recognized_transcript"]);
+const INTERVAL_MS = 30_000;
+const MAX_LINES = 16;
+
+/** Keys stay private; only fixed reasons and server-generated request IDs leave the process. */
+export class ConversationDiagnostics {
+  private readonly episodes: ConnectionDiagnostics;
+  private windowAt = -Infinity;
+  private lines = 0;
+
+  constructor(write: (line: string) => void = (line) => console.warn(line), private now: () => number = () => performance.now()) {
+    // The budget bounds the output only: episodes still open and close while it is spent.
+    this.episodes = new ConnectionDiagnostics("conversation", (line) => {
+      if (!this.available()) return false;
+      this.lines++;
+      write(line);
+      return true;
+    }, () => performance.timeOrigin + now());
+  }
+
+  private available(): boolean {
+    const at = this.now();
+    if (at - this.windowAt >= INTERVAL_MS) { this.windowAt = at; this.lines = 0; }
+    return this.lines < MAX_LINES;
+  }
+
+  unavailable(key: string, error: ConversationUnavailable | ConversationReadPending, requestId?: string): void {
+    if (error instanceof ConversationUnavailable && error.diagnostic === undefined && EXPECTED.has(error.message)) {
+      this.recovered(key, requestId);
+      return;
+    }
+    const pending = error instanceof ConversationReadPending;
+    const reason = pending ? "read_pending" : REASONS.find((reason) => reason === (error.diagnostic ?? error.message)) ?? "unknown";
+    this.episodes.failed(key, pending ? "conversation_read_pending" : "conversation_unavailable", {
+      operation: `conversation.${reason}`, request_id: requestId,
+    });
+  }
+
+  recovered(key: string, requestId?: string): void {
+    this.episodes.recovered(key, { operation: "conversation", request_id: requestId });
+  }
+}

server/conversation-shared.ts

@@ -10,9 +10,17 @@ import type { ConversationMetadata, ConversationTurn } from "../shared/protocol.
  */
 export const TRANSCRIPT_WINDOW_BYTES = 16 * 1024 * 1024;
 
+export const COPILOT_UNAVAILABLE_REASONS = [
+  "copilot_report_invalid", "copilot_process_lookup_failed", "copilot_process_set_unproved",
+  "copilot_peer_process_replaced", "copilot_handoff_unproved", "copilot_peer_conflict",
+  "copilot_foreground_unproved", "copilot_resume_unproved",
+] as const;
+/** Fixed stages a diagnostic may name: Copilot's rejections, and an agent pane herdr gives no folder. */
+export const UNAVAILABLE_DIAGNOSTICS = [...COPILOT_UNAVAILABLE_REASONS, "agent_cwd_missing"] as const;
+
 export class ConversationUnavailable extends Error {
   /** `screen`: the model and effort the agent's own live status line states (start-screen.ts), for the scrollback standing in */
-  constructor(reason: string, readonly screen?: ConversationMetadata) {
+  constructor(reason: string, readonly screen?: ConversationMetadata, readonly diagnostic?: typeof UNAVAILABLE_DIAGNOSTICS[number]) {
     super(reason);
     this.name = "ConversationUnavailable";
   }

server/conversation.ts

@@ -1167,7 +1167,7 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[
       peerInfo?.set(peer.pane_id, info ?? null);
       const processes = agentProcesses(info?.foreground_processes ?? [], "copilot");
       // A cached peer process that has exited may have been replaced by one we never asked about.
-      if (cached && processes.some((entry) => typeof entry.pid === "number" && !processAlive(entry.pid))) throw new ConversationUnavailable("no_session_path");
+      if (cached && processes.some((entry) => typeof entry.pid === "number" && !processAlive(entry.pid))) throw new ConversationUnavailable("no_session_path", undefined, "copilot_peer_process_replaced");
       if (processes.length === 1 && typeof processes[0]?.pid === "number") {
         const live = copilotProcessSession(processEnviron(processes[0].pid), processes[0].argv ?? [], processes[0].pid);
         if (live?.logged) {
@@ -1182,31 +1182,33 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[
   const claimedByPeer = async (session: string): Promise<boolean> => (await (peerClaims ??= gatherPeerClaims())).has(session);
   if (pane.agent !== "copilot" || (report != null && (report.agent !== "copilot" || report.kind !== "id"
     || report.source !== "herdr:copilot" || typeof report.value !== "string" || !SESSION_ID.test(report.value)))) {
-    throw new ConversationUnavailable("no_session_id");
+    throw new ConversationUnavailable("no_session_id", undefined, "copilot_report_invalid");
   }
   // A failed RPC does not prove an ownership change. Retry that read once; an answered
   // empty/conflicting process set still fails closed, as does a failed retry.
   const info = await paneProcessInfo(pane.pane_id).catch(() => paneProcessInfo(pane.pane_id)).catch(() => null);
+  if (info === null) throw new ConversationUnavailable("no_session_path", undefined, "copilot_process_lookup_failed");
   const processes = agentProcesses(info?.foreground_processes ?? [], "copilot");
   const foreground = processes[0];
   if (process.platform !== "linux" || ![1, 2].includes(processes.length) || typeof foreground?.pid !== "number") {
-    throw new ConversationUnavailable("no_session_path");
+    throw new ConversationUnavailable("no_session_path", undefined, "copilot_process_set_unproved");
   }
   if (processes.length === 2) {
-    if (report == null) throw new ConversationUnavailable("no_session_id");
+    if (report == null) throw new ConversationUnavailable("no_session_id", undefined, "copilot_report_invalid");
     const parent = processes[0]!;
     const child = processes[1]!;
     const evidence = (entry: typeof parent) => ({ pid: entry.pid, argv: entry.argv ?? [] });
     const selected = copilotHandoffSession(evidence(parent), evidence(child),
       report.value, info?.foreground_process_group_id, info?.tty, processEnviron)
       ?? copilotHandoffSession(evidence(child), evidence(parent),
         report.value, info?.foreground_process_group_id, info?.tty, processEnviron);
-    if (selected?.logged && !(await claimedByPeer(selected.sessionId))) {
+    if (selected?.logged) {
+      if (await claimedByPeer(selected.sessionId)) throw new ConversationUnavailable("no_session_path", undefined, "copilot_peer_conflict");
       const path = copilotTranscriptFile(report.value, selected);
       if (path !== null && nodePath.basename(nodePath.dirname(path)) === report.value)
         return { path, binding: selected.generation, peerClaims: await peerClaims! };
     }
-    throw new ConversationUnavailable("no_session_path");
+    throw new ConversationUnavailable("no_session_path", undefined, "copilot_handoff_unproved");
   }
   const environment = processEnviron(foreground.pid);
   const argv = foreground.argv ?? [];
@@ -1218,7 +1220,7 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[
       return { path, binding: `${foreground.pid}:${logged.generation}`, peerClaims: await peerClaims! };
   }
   if (report != null) {
-    if (await claimedByPeer(report.value)) throw new ConversationUnavailable("no_session_id");
+    if (await claimedByPeer(report.value)) throw new ConversationUnavailable("no_session_id", undefined, "copilot_peer_conflict");
     if (logged === null) {
       const path = copilotTranscriptFile(report.value, selected);
       if (path !== null && nodePath.basename(nodePath.dirname(path)) === report.value)
@@ -1230,7 +1232,8 @@ async function copilotTranscriptPath(pane: HerdrPane, panes: readonly HerdrPane[
     && !(await claimedByPeer(pending.sessionId))) {
     throw new ConversationNotStarted(pending.sessionId, "copilot-transcript", pending.path);
   }
-  throw new ConversationUnavailable(copilotExplicitResume(argv) ? "copilot_resume_unknown" : report == null ? "no_session_id" : "no_session_path");
+  throw new ConversationUnavailable(copilotExplicitResume(argv) ? "copilot_resume_unknown" : report == null ? "no_session_id" : "no_session_path",
+    undefined, copilotExplicitResume(argv) ? "copilot_resume_unproved" : "copilot_foreground_unproved");
 }
 
 /** Where a pane's conversation is: a transcript file, or a session in OpenCode's database. */
@@ -1368,7 +1371,8 @@ export async function paneTranscript(paneId: string, codexHome?: string, snapsho
   const panes = (snapshot ?? await sessionSnapshot()).panes;
   const pane = panes.find((candidate) => candidate.pane_id === paneId);
   if (pane === undefined) throw new ConversationUnavailable("pane_not_found");
-  if (typeof pane.cwd !== "string" || pane.cwd.length === 0) throw new ConversationUnavailable("no_recognized_transcript");
+  // an agent pane without a folder is a failure to diagnose, not a shell's expected scrollback
+  if (typeof pane.cwd !== "string" || pane.cwd.length === 0) throw new ConversationUnavailable("no_recognized_transcript", undefined, pane.agent ? "agent_cwd_missing" : undefined);
 
   if (pane.agent === "devin" && !(await paneRunsOmo(paneId, known))) {
     const cwd = pane.foreground_cwd || pane.cwd;