diff --git a/src/platform/telemetry/TelemetryRegistry.ts b/src/platform/telemetry/TelemetryRegistry.ts index f25f5190e50..d4c194bfc92 100644 --- a/src/platform/telemetry/TelemetryRegistry.ts +++ b/src/platform/telemetry/TelemetryRegistry.ts @@ -8,6 +8,7 @@ import type { AgentPanelOpenedMetadata, AgentReconnectFailedMetadata, AgentReconnectStartedMetadata, + AgentReconnectSucceededMetadata, AgentWorkflowAppliedMetadata, AuthErrorMetadata, AuthMetadata, @@ -390,6 +391,14 @@ export class TelemetryRegistry implements TelemetryDispatcher { this.dispatch((provider) => provider.trackAgentReconnectStarted?.(metadata)) } + trackAgentReconnectSucceeded( + metadata: AgentReconnectSucceededMetadata + ): void { + this.dispatch((provider) => + provider.trackAgentReconnectSucceeded?.(metadata) + ) + } + trackWidgetFavoriteToggled(metadata: WidgetFavoriteToggledMetadata): void { this.dispatch((provider) => provider.trackWidgetFavoriteToggled?.(metadata)) } diff --git a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts index d6027a96bdc..9ab214de528 100644 --- a/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts +++ b/src/platform/telemetry/providers/cloud/PostHogTelemetryProvider.ts @@ -20,6 +20,7 @@ import type { AgentPanelOpenedMetadata, AgentReconnectFailedMetadata, AgentReconnectStartedMetadata, + AgentReconnectSucceededMetadata, AgentWorkflowAppliedMetadata, AuthErrorMetadata, AuthMetadata, @@ -712,6 +713,12 @@ export class PostHogTelemetryProvider implements TelemetryProvider { this.trackEvent(TelemetryEvents.AGENT_RECONNECT_STARTED, metadata) } + trackAgentReconnectSucceeded( + metadata: AgentReconnectSucceededMetadata + ): void { + this.trackEvent(TelemetryEvents.AGENT_RECONNECT_SUCCEEDED, metadata) + } + trackWidgetFavoriteToggled(metadata: WidgetFavoriteToggledMetadata): void { this.trackEvent(TelemetryEvents.WIDGET_FAVORITE_TOGGLED, metadata) } diff --git a/src/platform/telemetry/types.ts b/src/platform/telemetry/types.ts index 1ae847b82a1..7682a777b9b 100644 --- a/src/platform/telemetry/types.ts +++ b/src/platform/telemetry/types.ts @@ -593,6 +593,17 @@ export interface AgentReconnectStartedMetadata extends Record { offline_duration_ms: number | null } +export interface AgentReconnectSucceededMetadata extends Record< + string, + unknown +> { + attempt: number + reconnect_duration_ms: number + replayed_bytes: number + from_version: number + to_version: number +} + /** * Widget (input/parameter) favorite toggle tracking metadata. * Used to measure discoverability of the right side panel favoriting feature. @@ -1107,6 +1118,7 @@ export interface TelemetryProvider { trackAgentWorkflowApplied?(metadata: AgentWorkflowAppliedMetadata): void trackAgentReconnectFailed?(metadata: AgentReconnectFailedMetadata): void trackAgentReconnectStarted?(metadata: AgentReconnectStartedMetadata): void + trackAgentReconnectSucceeded?(metadata: AgentReconnectSucceededMetadata): void // Right side panel widget favorite events trackWidgetFavoriteToggled?(metadata: WidgetFavoriteToggledMetadata): void @@ -1275,6 +1287,7 @@ export const TelemetryEvents = { AGENT_WORKFLOW_APPLIED: 'app:agent_workflow_applied', AGENT_RECONNECT_FAILED: 'app:agent_reconnect_failed', AGENT_RECONNECT_STARTED: 'app:agent_reconnect_started', + AGENT_RECONNECT_SUCCEEDED: 'app:agent_reconnect_succeeded', // Right Side Panel Widget Favorites WIDGET_FAVORITE_TOGGLED: 'app:widget_favorite_toggled', diff --git a/src/workbench/extensions/agent/crdt/reconnectReport.ts b/src/workbench/extensions/agent/crdt/reconnectReport.ts new file mode 100644 index 00000000000..9ac24a3da8d --- /dev/null +++ b/src/workbench/extensions/agent/crdt/reconnectReport.ts @@ -0,0 +1,149 @@ +import type { AgentReconnectSucceededMetadata } from '@/platform/telemetry/types' + +/** + * TEL-10's recovery bookkeeping as one state rather than a bag of fields. + * + * A recovery has exactly three phases: nothing in flight, a disconnect signal + * seen but not yet answered, and an answered subscribe whose report is waiting + * for the catch-up frame that proves how much was replayed. The report is not + * emitted at the ack because the relay sends the ack first and the catch-up + * `doc_update` (same seq as the ack) after it. + * + * `fromVersion` is supplied by the caller, never read off the transport: the + * bridge clears its sequence bookkeeping the moment a subscribe frame leaves, + * so by the time a refusal comes back it no longer knows where the follower + * was. + */ +export type ReconnectState = + | { phase: 'idle' } + | { + phase: 'inFlight' + startedAt: number + fromVersion: number + /** + * The relay joins the fanout before it acks, so frames can arrive ahead + * of the ack that says which of them are catch-up. They are held here + * until the ack seq can classify them. + */ + preAck: readonly { seq: number; bytes: number }[] + } + | { + phase: 'pending' + ackSeq: number + report: AgentReconnectSucceededMetadata + } + +export type ReconnectEvent = + /** A socket drop or a server refusal, i.e. the recovery clock starts. */ + | { type: 'disconnected'; at: number; fromVersion: number } + | { type: 'confirmed'; at: number; ackSeq: number | null; attempt: number } + | { type: 'update'; seq: number; bytes: number } + /** The binding is going away; report whatever succeeded, drop the rest. */ + | { type: 'flushed' } + /** The binding is going away and nothing about it is reportable. */ + | { type: 'abandoned' } + +export interface ReconnectTransition { + state: ReconnectState + report: AgentReconnectSucceededMetadata | null +} + +export const initialReconnectState: ReconnectState = { phase: 'idle' } + +const idle = ( + report: AgentReconnectSucceededMetadata | null = null +): ReconnectTransition => ({ state: initialReconnectState, report }) + +const unchanged = (state: ReconnectState): ReconnectTransition => ({ + state, + report: null +}) + +const isReplay = (seq: number, fromVersion: number, ackSeq: number): boolean => + seq > fromVersion && seq <= ackSeq + +export function reduceReconnect( + state: ReconnectState, + event: ReconnectEvent +): ReconnectTransition { + switch (event.type) { + case 'disconnected': { + // A disconnect while a report is pending is a NEW recovery, so the + // pending one is reported before the clock restarts. A second signal + // mid-recovery must not reset the clock or the baseline. + if (state.phase === 'inFlight') return unchanged(state) + return { + state: { + phase: 'inFlight', + startedAt: event.at, + fromVersion: event.fromVersion, + preAck: [] + }, + report: state.phase === 'pending' ? state.report : null + } + } + case 'confirmed': { + // A second ok ack while pending (the stale probe's resubscribe) proves + // no catch-up is coming. + if (state.phase === 'pending') return idle(state.report) + if (state.phase === 'idle') return unchanged(state) + const report: AgentReconnectSucceededMetadata = { + attempt: event.attempt, + reconnect_duration_ms: Math.max( + 0, + Math.round(event.at - state.startedAt) + ), + replayed_bytes: 0, + from_version: state.fromVersion, + // An ack without a seq says nothing about where the doc now is; + // reporting 0 would read as a version that moved backwards. + to_version: state.fromVersion + } + if (event.ackSeq === null) return idle(report) + const ackSeq = event.ackSeq + const replayed = state.preAck.filter((frame) => + isReplay(frame.seq, state.fromVersion, ackSeq) + ) + const confirmed = { + ...report, + to_version: ackSeq, + replayed_bytes: replayed.reduce( + (total, frame) => total + frame.bytes, + 0 + ) + } + // The catch-up frame overtook its own ack, so there is nothing left to + // wait for. + const catchUpAlreadyLanded = replayed.some( + (frame) => frame.seq === ackSeq + ) + return catchUpAlreadyLanded + ? idle(confirmed) + : unchanged({ phase: 'pending', ackSeq, report: confirmed }) + } + case 'update': { + if (state.phase === 'idle') return unchanged(state) + if (state.phase === 'inFlight') + return unchanged({ + ...state, + preAck: [...state.preAck, { seq: event.seq, bytes: event.bytes }] + }) + // A frame above the ack seq is live traffic: it proves the catch-up was + // empty (or already counted) and is not part of it. + if (event.seq > state.ackSeq) return idle(state.report) + if (!isReplay(event.seq, state.report.from_version, state.ackSeq)) + return unchanged(state) + const report = { + ...state.report, + replayed_bytes: state.report.replayed_bytes + event.bytes + } + return event.seq === state.ackSeq + ? idle(report) + : unchanged({ ...state, report }) + } + case 'flushed': + return state.phase === 'pending' ? idle(state.report) : unchanged(state) + case 'abandoned': + return idle() + } +} diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index 0dc6014cab8..189448a6093 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -21,9 +21,17 @@ import type { MaterializableGraph } from './agentNodeMaterializer' const bridgeState = vi.hoisted(() => { class FakeBridge extends EventTarget { - subscribe = vi.fn() + // Mirrors the real bridge: `reconcile()` nulls `lastSeq`/`ackSeq` as soon + // as a subscribe frame leaves, so `lastSequence` reads 0 from the moment a + // subscribe is attempted until the next ack. Anything deriving a + // reconnect's `from_version` from the bridge would therefore report 0. + subscribe = vi.fn((_workflowId: string): void => { + this.lastSequence = 0 + }) unsubscribe = vi.fn() - resubscribe = vi.fn() + resubscribe = vi.fn(() => { + this.lastSequence = 0 + }) reconcile = vi.fn() destroy = vi.fn() sendHumanOps = vi.fn() @@ -55,7 +63,8 @@ const adapterState = vi.hoisted(() => ({ const telemetryState = vi.hoisted(() => ({ trackAgentReconnectFailed: vi.fn(), - trackAgentReconnectStarted: vi.fn() + trackAgentReconnectStarted: vi.fn(), + trackAgentReconnectSucceeded: vi.fn() })) const materializerState = vi.hoisted(() => ({ @@ -255,6 +264,405 @@ describe('useAgentCrdtFollower', () => { unmount() }) + it('TEL-10: reports reconnect success once the catch-up frame lands', () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + // A catch-up frame can overtake its own ack; it is still replay. + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 42, + update: new Uint8Array(10) + }) + dispatchFrame('doc_subscribed', { ok: true, seq: 43 }) + // The relay sends the catch-up AFTER the ack, so the ack alone is not the + // restore point: nothing is reported yet. + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + + // The wait for the catch-up must not stretch the duration. + vi.advanceTimersByTime(250) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 43, + update: new Uint8Array(20) + }) + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledOnce() + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledWith({ + attempt: 1, + reconnect_duration_ms: 500, + replayed_bytes: 30, + from_version: 41, + to_version: 43 + }) + + // Ordinary live traffic after the report must not re-report. + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 44, + update: new Uint8Array(5) + }) + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledOnce() + + // A second recovery is its own report, baselined on the live frame above. + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_subscribed', { ok: true, seq: 45 }) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 45, + update: new Uint8Array(7) + }) + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledTimes(2) + expect( + telemetryState.trackAgentReconnectSucceeded + ).toHaveBeenLastCalledWith({ + attempt: 1, + reconnect_duration_ms: 500, + replayed_bytes: 7, + from_version: 44, + to_version: 45 + }) + unmount() + }) + + it('TEL-10: reports zero replayed bytes when the first post-ack frame is live', () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + + // No catch-up was needed (ack seq equals the follower's own seq); the + // first live frame above the ack seq proves that and is NOT counted. + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 42, + update: new Uint8Array(99) + }) + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledOnce() + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledWith({ + attempt: 1, + reconnect_duration_ms: 500, + replayed_bytes: 0, + from_version: 41, + to_version: 41 + }) + unmount() + }) + + it('TEL-10: a quiet channel is reported by the stale probe re-ack', () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + + bridge().resubscribe.mockClear() + vi.advanceTimersByTime(STALE_AFTER_MS) + expect(bridge().resubscribe).toHaveBeenCalledOnce() + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledOnce() + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledWith({ + attempt: 1, + reconnect_duration_ms: 500, + replayed_bytes: 0, + from_version: 41, + to_version: 41 + }) + unmount() + }) + + it('TEL-10: the socket-drop path reports from the pre-drop version', () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 7 }) + + apiState.target.dispatchEvent(new Event('reconnected')) + vi.advanceTimersByTime(1_200) + dispatchFrame('doc_subscribed', { ok: true, seq: 9 }) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 9, + update: new Uint8Array(12) + }) + + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledOnce() + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledWith({ + attempt: 1, + reconnect_duration_ms: 1_200, + replayed_bytes: 12, + from_version: 7, + to_version: 9 + }) + unmount() + }) + + it('TEL-10/TEL-9: a socket drop followed by refusals reports every attempt', () => { + // Both disconnect signals can fire in one recovery: the socket drop + // resubscribes immediately, the server refuses it, and the refusal retry + // is what finally lands. The success report must count both legs, and + // TEL-9's live counter must survive the snapshot the report takes. + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 7 }) + + apiState.target.dispatchEvent(new Event('reconnected')) + expect(telemetryState.trackAgentReconnectStarted).toHaveBeenCalledWith( + expect.objectContaining({ attempt: 1 }) + ) + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + expect(bridge().resubscribe).toHaveBeenCalled() + dispatchFrame('doc_subscribed', { ok: true, seq: 9 }) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 9, + update: new Uint8Array(6) + }) + + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledOnce() + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledWith({ + attempt: 2, + reconnect_duration_ms: 500, + replayed_bytes: 6, + from_version: 7, + to_version: 9 + }) + + // The report snapshot must not have consumed TEL-9's counter: the next + // socket drop is attempt 1 again because the confirm above reset it. + apiState.target.dispatchEvent(new Event('reconnected')) + expect(telemetryState.trackAgentReconnectStarted).toHaveBeenLastCalledWith( + expect.objectContaining({ attempt: 1 }) + ) + unmount() + }) + + it('TEL-10: a workflow switch mid-reconnect does not report on the next bind', async () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount, workflowId } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + // Armed but never confirmed: the switch drops the bookkeeping silently. + apiState.target.dispatchEvent(new Event('reconnected')) + workflowId.value = 'wf-2' + await Promise.resolve() + await Promise.resolve() + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + + dispatchFrame('doc_subscribed', { ok: true, seq: 1 }) + dispatchFrame('doc_update', { + workflowId: 'wf-2', + seq: 1, + update: new Uint8Array(3) + }) + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + unmount() + }) + + it('TEL-10: a workflow switch while the report is pending flushes it', async () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount, workflowId } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + // Confirmed, catch-up not yet delivered: the reconnect DID succeed. + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_subscribed', { ok: true, seq: 43 }) + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + + workflowId.value = 'wf-2' + await Promise.resolve() + await Promise.resolve() + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledOnce() + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledWith({ + attempt: 1, + reconnect_duration_ms: 500, + replayed_bytes: 0, + from_version: 41, + to_version: 43 + }) + unmount() + }) + + it('TEL-10: does not report reconnect success on the first-ever bind', () => { + vi.useFakeTimers() + mountFollower('wf-1') + + dispatchFrame('doc_subscribed', { ok: true, seq: 1 }) + + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + }) + + it('TEL-10: does not report reconnect success on a first-ever bind that is refused first', () => { + // FE-1901's documented cold start: the subscribe races the doc-host, is + // refused, and the retry lands. Nothing was ever connected, so there is no + // recovery to report - counting it would inflate the very rate this metric + // exists to measure. + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + const { unmount } = mountFollower('wf-1') + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_subscribed', { ok: true, seq: 1 }) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 1, + update: new Uint8Array(4) + }) + + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + unmount() + }) + + it('TEL-10: does not report reconnect success when the socket reconnects before any binding was confirmed', () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + const { unmount } = mountFollower('wf-1') + + apiState.target.dispatchEvent(new Event('reconnected')) + dispatchFrame('doc_subscribed', { ok: true, seq: 3 }) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 3, + update: new Uint8Array(4) + }) + + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + unmount() + }) + + it('TEL-10: an ack without a seq reports the baseline rather than a version that moved backwards', () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_subscribed', { ok: true }) + + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledWith({ + attempt: 1, + reconnect_duration_ms: 500, + replayed_bytes: 0, + from_version: 41, + to_version: 41 + }) + unmount() + }) + + it('TEL-10: a pre-ack frame above the ack seq is live traffic, not replay', () => { + // The relay joins the fanout before it acks, so a frame can arrive ahead + // of the ack that says where the catch-up ends. Only frames the ack + // actually covers are replay. + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 42, + update: new Uint8Array(10) + }) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 43, + update: new Uint8Array(90) + }) + dispatchFrame('doc_subscribed', { ok: true, seq: 42 }) + + // The catch-up (seq 42) already landed, so the report needs nothing more. + expect(telemetryState.trackAgentReconnectSucceeded).toHaveBeenCalledWith({ + attempt: 1, + reconnect_duration_ms: 500, + replayed_bytes: 10, + from_version: 41, + to_version: 42 + }) + unmount() + }) + + it('TEL-10: a lineage break drops the reconnect bookkeeping instead of carrying it into the new doc', () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_subscribed', { ok: true, seq: 43 }) + dispatchFrame('doc_reset', { workflowId: 'wf-1', seq: 0 }) + + // The new lineage restarts at seq 1; without dropping the bookkeeping its + // frames would be counted as the old doc's catch-up. + dispatchFrame('doc_subscribed', { ok: true, seq: 1 }) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 1, + update: new Uint8Array(8) + }) + + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + unmount() + }) + + it('TEL-10: a schema error drops the pending report instead of leaving it to a later frame', () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + vi.setSystemTime(1_000) + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + dispatchFrame('doc_subscribed', { ok: false }) + vi.advanceTimersByTime(500) + dispatchFrame('doc_subscribed', { ok: true, seq: 43 }) + dispatchFrame('schema_error', { workflowId: 'wf-1' }) + dispatchFrame('doc_update', { + workflowId: 'wf-1', + seq: 43, + update: new Uint8Array(8) + }) + + // Nothing was projected, so the recovery never completed. + expect(telemetryState.trackAgentReconnectSucceeded).not.toHaveBeenCalled() + unmount() + }) + + it('TEL-9/TEL-10: a workflow switch resets the reconnect attempt counter', async () => { + vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) + const { unmount, workflowId } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: true, seq: 41 }) + + apiState.target.dispatchEvent(new Event('reconnected')) + workflowId.value = 'wf-2' + await nextTick() + + apiState.target.dispatchEvent(new Event('reconnected')) + expect(telemetryState.trackAgentReconnectStarted).toHaveBeenLastCalledWith( + expect.objectContaining({ attempt: 1 }) + ) + unmount() + }) + it('reports retry exhaustion exactly once with normalized metadata', () => { vi.useFakeTimers({ toFake: ['performance', 'setTimeout', 'clearTimeout'] }) vi.setSystemTime(1_000) @@ -515,6 +923,7 @@ describe('useAgentCrdtFollower', () => { vi.setSystemTime(1_000) const { unmount } = mountFollower('wf-1') dispatchFrame('doc_subscribed', { ok: true }) + bridge().lastSequence = 41 vi.advanceTimersByTime(5_000) apiState.target.dispatchEvent(new Event('reconnected')) diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 18a3e01b875..fd97bbda016 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -20,6 +20,8 @@ import type { GraphOperation } from './graphOperations' import { LayoutFollowerBridge } from './layoutFollowerBridge' import type { OpsResultView } from './opSender' import { createOpSender } from './opSender' +import type { ReconnectEvent } from './reconnectReport' +import { initialReconnectState, reduceReconnect } from './reconnectReport' // FE-1902: the doc id is otherwise held only in memory (set on turn ack), so a // panel remount loses the binding until the NEXT turn ack. Persist it per-tab @@ -296,6 +298,17 @@ export function useAgentCrdtFollower( let subscribeRetryStartedAt: number | null = null let subscribeRetryFailureReported = false + // TEL-10: mirrors the retry-exhaustion bookkeeping above, but for the + // recovery path — armed by the SAME two disconnect signals + // (`onReconnected`'s socket drop and a server `doc_subscribed{ok:false}` + // refusal). See `reconnectReport.ts` for the phases and the transition. + let reconnect = initialReconnectState + // The seq this binding is known to have reached, and the proof that it was + // ever bound at all: `null` until a subscribe for it is CONFIRMED. A refusal + // before that is FE-1901's cold-start race, not a recovery, so nothing is + // armed and no "reconnect" is reported for a connection that never was. + let confirmedVersion: number | null = null + // The recency heartbeat: armed only while a subscribe is CONFIRMED (bound + // healthy by definition), slid forward by every doc-scoped frame, cancelled // by the same lifecycle exits as the subscribe retry. The probe is @@ -357,6 +370,50 @@ export function useAgentCrdtFollower( subscribeRetryFailureReported = false } + const dispatchReconnect = (event: ReconnectEvent): void => { + const { state, report } = reduceReconnect(reconnect, event) + reconnect = state + if (report !== null) useTelemetry()?.trackAgentReconnectSucceeded(report) + } + + const clearReconnectTracking = (): void => { + dispatchReconnect({ type: 'abandoned' }) + } + const flushPendingReconnectReport = (): void => { + dispatchReconnect({ type: 'flushed' }) + } + + const armReconnectTracking = (): void => { + if (confirmedVersion === null) return + dispatchReconnect({ + type: 'disconnected', + at: performance.now(), + fromVersion: confirmedVersion + }) + } + + const onReconnectConfirmed = (ackSeq: number | null): void => { + dispatchReconnect({ + type: 'confirmed', + at: performance.now(), + ackSeq, + // Both disconnect signals produce subscribe attempts and both counters + // reset on this same confirmed subscribe, so the recovery cost is their + // sum: `onReconnected`'s immediate resubscribe plus every refusal-driven + // retry. Floored at 1 because a socket-drop recovery that never retried + // still took one attempt. + attempt: Math.max(1, reconnectAttempt + subscribeRetryAttempt) + }) + } + + const trackReconnectUpdate = (update: DocUpdate): void => { + dispatchReconnect({ + type: 'update', + seq: update.seq, + bytes: update.update instanceof Uint8Array ? update.update.length : 0 + }) + } + const reportSubscribeRetryExhausted = (): void => { if ( subscribeRetryFailureReported || @@ -393,6 +450,7 @@ export function useAgentCrdtFollower( const target = subscribedWorkflowId.value if (target === null) return subscribeRetryStartedAt ??= performance.now() + armReconnectTracking() const delay = SUBSCRIBE_RETRY_BASE_MS * 2 ** subscribeRetryAttempt subscribeRetryAttempt += 1 subscribeRetryTimer = setTimeout(() => { @@ -427,6 +485,15 @@ export function useAgentCrdtFollower( lastFrameType.value = event.type recordDevEvent('doc_subscribed', event.detail ?? null) if (ok) { + // Snapshot before clearSubscribeRetry() resets the attempt counter. The + // confirm is only a RECOVERY if a disconnect signal armed one; on the + // panel's first-ever bind it merely establishes the baseline below. + const seq = event.detail?.seq + onReconnectConfirmed(typeof seq === 'number' ? seq : null) + confirmedVersion = + typeof seq === 'number' + ? Math.max(confirmedVersion ?? 0, seq) + : (confirmedVersion ?? 0) clearSubscribeRetry() armStaleProbe() reconnectAttempt = 0 @@ -461,6 +528,9 @@ export function useAgentCrdtFollower( } return } + trackReconnectUpdate(update) + if (confirmedVersion !== null) + confirmedVersion = Math.max(confirmedVersion, update.seq) if (staleProbeTimer !== null) armStaleProbe() markActivity() refreshPersistedDocId() @@ -526,6 +596,11 @@ export function useAgentCrdtFollower( updatesApplied.value = 0 lastFrameType.value = event.type clearStaleProbe() + // The seq space restarts with the new lineage, so a recovery armed against + // the old one can never be confirmed against a comparable version. Neither + // is a doc_reset itself a reconnect success: drop it without a report. + clearReconnectTracking() + confirmedVersion = null knownDocNodeIds = new Set() recordDevEvent( 'doc_reset', @@ -566,6 +641,7 @@ export function useAgentCrdtFollower( connected.value = false lastFrameType.value = event.type clearStaleProbe() + clearReconnectTracking() const detail = event instanceof CustomEvent ? (event.detail as { workflowId?: string } | null) @@ -610,6 +686,7 @@ export function useAgentCrdtFollower( } connected.value = false clearStaleProbe() + armReconnectTracking() recordDevEvent('reconnected', null) bridge.resubscribe() } @@ -687,6 +764,16 @@ export function useAgentCrdtFollower( // existing "reconcile on frame or on graph readiness" behaviour. const justActivated = active && previous?.[1] === false clearSubscribeRetry() + // A reconnect that was confirmed but still waiting for its catch-up + // frame DID succeed: report it before the binding changes. One that was + // never confirmed is dropped without a report. + flushPendingReconnectReport() + clearReconnectTracking() + // The new binding has never been confirmed and has its own seq space, and + // TEL-9's live counter must not carry an abandoned recovery's attempts + // into the next workflow's first report. + confirmedVersion = null + reconnectAttempt = 0 clearStaleProbe() connected.value = false knownDocNodeIds = new Set() @@ -742,6 +829,8 @@ export function useAgentCrdtFollower( // update twice after a remount. try { clearSubscribeRetry() + flushPendingReconnectReport() + clearReconnectTracking() clearStaleProbe() api.removeEventListener('reconnected', onReconnected) api.removeEventListener('status', onSocketActivity)