From 90556bac8e6818eef006c696ee4c5694caa7e7db Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 26 Oct 2026 21:18:49 +0000 Subject: [PATCH 1/3] fix(agent): budget the identity failure reports so a flapping socket cannot flood telemetry The retry budget (MAX_ATTEMPTS) bounds how many lookups one cycle makes, but retryNow() resets `failures` to 0 whenever the socket reopens. A socket that flaps against a down identity route therefore starts a fresh cycle each time and re-emits the same warning-level report forever. Give the reports their own budget, spent across the whole lifetime of an identity and refilled only by reset() (an account change), as docs/guidance/error-handling.md asks for when a retry can repeat the same failure. Retries themselves are unchanged: the socket still restarts the lookup, it just stops re-reporting a failure telemetry already has. --- .../services/agent/agentIdentity.test.ts | 28 +++++++++++++++++++ .../agent/services/agent/agentIdentity.ts | 13 ++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts b/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts index 99546372c35..f58ef7711f2 100644 --- a/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts @@ -267,4 +267,32 @@ describe('resolveAgentIdentity', () => { userId: 'local-user' }) }) + + it('stops reporting once the report budget is spent, so a flapping socket cannot flood telemetry', async () => { + const getIdentity = failing() + const identity = harness(getIdentity) + + await vi.advanceTimersByTimeAsync(10 * 60_000) + expect(identity.onFailure).toHaveBeenCalledTimes(8) + + for (let reopen = 0; reopen < 3; reopen += 1) { + identity.connect() + await vi.advanceTimersByTimeAsync(10 * 60_000) + } + + expect(getIdentity).toHaveBeenCalledTimes(32) + expect(identity.onFailure).toHaveBeenCalledTimes(8) + }) + + it('reports again for a new account, because reset refreshes the report budget', async () => { + const getIdentity = failing() + const identity = harness(getIdentity) + await vi.advanceTimersByTimeAsync(10 * 60_000) + expect(identity.onFailure).toHaveBeenCalledTimes(8) + + identity.reset() + await vi.advanceTimersByTimeAsync(10 * 60_000) + + expect(identity.onFailure).toHaveBeenCalledTimes(16) + }) }) diff --git a/src/workbench/extensions/agent/services/agent/agentIdentity.ts b/src/workbench/extensions/agent/services/agent/agentIdentity.ts index c9d8bb1bead..9b9b95971b8 100644 --- a/src/workbench/extensions/agent/services/agent/agentIdentity.ts +++ b/src/workbench/extensions/agent/services/agent/agentIdentity.ts @@ -5,6 +5,12 @@ const RETRY_BASE_MS = 1_000 const RETRY_MAX_MS = 30_000 /** Lookups tried before a transient failure is treated as a rejection. */ const MAX_ATTEMPTS = 8 +/** + * Failures reported per identity. The attempt budget restarts every time the + * socket reopens, so without a separate report budget a flapping socket would + * re-emit the same warning for as long as the panel stays mounted. + */ +const MAX_REPORTED_FAILURES = 8 interface AgentIdentityDeps { getIdentity(): Promise<{ userId: string }> @@ -35,6 +41,7 @@ export function resolveAgentIdentity( let inFlight = false let stopped = false let failures = 0 + let reported = 0 let retryTimer: ReturnType | null = null let generation = 0 let exhausted = false @@ -59,7 +66,10 @@ export function resolveAgentIdentity( function settleFailure(error: unknown): void { failures += 1 - deps.onFailure(error) + if (reported < MAX_REPORTED_FAILURES) { + reported += 1 + deps.onFailure(error) + } if (!deps.isRetryable(error)) { state.value = { status: 'rejected' } return @@ -106,6 +116,7 @@ export function resolveAgentIdentity( generation += 1 state.value = { status: 'pending' } failures = 0 + reported = 0 exhausted = false inFlight = false cancelRetry() From 3ec754d2bef14a82c2268cb7baa3318021ca7b4b Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 28 Oct 2026 03:35:29 +0000 Subject: [PATCH 2/3] fix(agent): always report a terminal identity failure, and mark suppression The report budget was charged before the retryability check, and MAX_REPORTED_FAILURES equalled MAX_ATTEMPTS, so one exhausted transient cycle spent the whole pool. A later non-retryable lookup (401/403, or a 404 for a missing route) then rejected permanently with no telemetry at all -- the most actionable failure was the one guaranteed to be silent. Terminal failures now always report and only retryable ones draw on the budget. When the budget runs out one marker is emitted, so a socket flapping for hours is no longer indistinguishable from a failure cycle that ended, and the budget is derived from MAX_ATTEMPTS so raising the attempts cannot silently drop the tail of the first window. --- .../services/agent/agentIdentity.test.ts | 30 +++++++++++++++- .../agent/services/agent/agentIdentity.ts | 36 ++++++++++++++----- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts b/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts index f58ef7711f2..c12e13bfcdf 100644 --- a/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts @@ -280,8 +280,36 @@ describe('resolveAgentIdentity', () => { await vi.advanceTimersByTimeAsync(10 * 60_000) } + // 8 reports for the first window, then exactly one marker saying the rest are + // suppressed -- without it this is indistinguishable from a cycle that ended. expect(getIdentity).toHaveBeenCalledTimes(32) - expect(identity.onFailure).toHaveBeenCalledTimes(8) + expect(identity.onFailure).toHaveBeenCalledTimes(9) + expect(identity.onFailure).toHaveBeenLastCalledWith( + expect.objectContaining({ + message: expect.stringContaining('further failure reports suppressed') + }) + ) + }) + + it('reports a terminal failure even after the retryable budget is spent', async () => { + const transient = new Error('agent still starting') + const terminal = new AgentApiError('forbidden', 403, undefined) + const getIdentity = vi + .fn<() => Promise<{ userId: string }>>() + .mockRejectedValue(transient) + const identity = harness(getIdentity, (error) => error !== terminal) + + await vi.advanceTimersByTimeAsync(10 * 60_000) + identity.onFailure.mockClear() + + // The budget is spent. A 401/403 or a missing route is the most actionable + // failure there is, so it must not be swallowed by the transient pool. + getIdentity.mockRejectedValue(terminal) + identity.connect() + await vi.advanceTimersByTimeAsync(10 * 60_000) + + expect(identity.onFailure).toHaveBeenCalledWith(terminal) + expect(identity.state.value).toEqual({ status: 'rejected' }) }) it('reports again for a new account, because reset refreshes the report budget', async () => { diff --git a/src/workbench/extensions/agent/services/agent/agentIdentity.ts b/src/workbench/extensions/agent/services/agent/agentIdentity.ts index 9b9b95971b8..0fe6e01293e 100644 --- a/src/workbench/extensions/agent/services/agent/agentIdentity.ts +++ b/src/workbench/extensions/agent/services/agent/agentIdentity.ts @@ -6,11 +6,14 @@ const RETRY_MAX_MS = 30_000 /** Lookups tried before a transient failure is treated as a rejection. */ const MAX_ATTEMPTS = 8 /** - * Failures reported per identity. The attempt budget restarts every time the - * socket reopens, so without a separate report budget a flapping socket would - * re-emit the same warning for as long as the panel stays mounted. + * Retryable failures reported per identity. The attempt budget restarts every time + * the socket reopens, so without a separate report budget a flapping socket would + * re-emit the same warning for as long as the panel stays mounted. Derived from + * MAX_ATTEMPTS so one full attempt window always reports: pinning it to a literal + * meant raising MAX_ATTEMPTS would silently drop the tail of the first window, + * including the attempt that exhausts retries. */ -const MAX_REPORTED_FAILURES = 8 +const MAX_REPORTED_FAILURES = MAX_ATTEMPTS interface AgentIdentityDeps { getIdentity(): Promise<{ userId: string }> @@ -42,6 +45,7 @@ export function resolveAgentIdentity( let stopped = false let failures = 0 let reported = 0 + let suppressed = false let retryTimer: ReturnType | null = null let generation = 0 let exhausted = false @@ -66,14 +70,29 @@ export function resolveAgentIdentity( function settleFailure(error: unknown): void { failures += 1 - if (reported < MAX_REPORTED_FAILURES) { - reported += 1 - deps.onFailure(error) - } + // A terminal failure -- a 401/403, or a 404 for a route that is not there -- is + // the most actionable one there is and ends the lookup for good, so it always + // reports. Charging it against the same budget as the transient failures meant an + // exhausted retry cycle could spend the budget first and leave it silent. if (!deps.isRetryable(error)) { + deps.onFailure(error) state.value = { status: 'rejected' } return } + if (reported < MAX_REPORTED_FAILURES) { + reported += 1 + deps.onFailure(error) + } else if (!suppressed) { + // Without this marker a socket flapping for hours looks exactly like a single + // failure cycle that ended, because both stop reporting at the same count. + suppressed = true + deps.onFailure( + new Error( + `Agent identity lookup still failing after ${failures} attempts; further failure reports suppressed`, + { cause: error } + ) + ) + } if (failures >= MAX_ATTEMPTS) { exhausted = true state.value = { status: 'rejected' } @@ -117,6 +136,7 @@ export function resolveAgentIdentity( state.value = { status: 'pending' } failures = 0 reported = 0 + suppressed = false exhausted = false inFlight = false cancelRetry() From d40121814000547addc10ebf3a700a00e26543eb Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 28 Oct 2026 06:49:46 +0000 Subject: [PATCH 3/3] fix(agent): count suppressed identity failures cumulatively A socket reopen resets `failures`, so the suppression marker described the ninth consecutive failure as "after 1 attempts". The marker now carries a total that only reset() clears, which is the number the telemetry reader needs. --- .../extensions/agent/services/agent/agentIdentity.test.ts | 4 +++- .../extensions/agent/services/agent/agentIdentity.ts | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts b/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts index c12e13bfcdf..3e5110e28e0 100644 --- a/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentIdentity.test.ts @@ -284,9 +284,11 @@ describe('resolveAgentIdentity', () => { // suppressed -- without it this is indistinguishable from a cycle that ended. expect(getIdentity).toHaveBeenCalledTimes(32) expect(identity.onFailure).toHaveBeenCalledTimes(9) + // The ninth failure, not the first of the second window: `failures` restarts on + // every reopen, so the marker has to count cumulatively or it reads "after 1". expect(identity.onFailure).toHaveBeenLastCalledWith( expect.objectContaining({ - message: expect.stringContaining('further failure reports suppressed') + message: expect.stringContaining('after 9 attempts') }) ) }) diff --git a/src/workbench/extensions/agent/services/agent/agentIdentity.ts b/src/workbench/extensions/agent/services/agent/agentIdentity.ts index 0fe6e01293e..0d17984d20f 100644 --- a/src/workbench/extensions/agent/services/agent/agentIdentity.ts +++ b/src/workbench/extensions/agent/services/agent/agentIdentity.ts @@ -46,6 +46,9 @@ export function resolveAgentIdentity( let failures = 0 let reported = 0 let suppressed = false + // `failures` restarts whenever the socket reopens, so it cannot describe how long + // the lookup has been failing. Only `reset()` clears this one. + let totalFailures = 0 let retryTimer: ReturnType | null = null let generation = 0 let exhausted = false @@ -70,6 +73,7 @@ export function resolveAgentIdentity( function settleFailure(error: unknown): void { failures += 1 + totalFailures += 1 // A terminal failure -- a 401/403, or a 404 for a route that is not there -- is // the most actionable one there is and ends the lookup for good, so it always // reports. Charging it against the same budget as the transient failures meant an @@ -88,7 +92,7 @@ export function resolveAgentIdentity( suppressed = true deps.onFailure( new Error( - `Agent identity lookup still failing after ${failures} attempts; further failure reports suppressed`, + `Agent identity lookup still failing after ${totalFailures} attempts; further failure reports suppressed`, { cause: error } ) ) @@ -135,6 +139,7 @@ export function resolveAgentIdentity( generation += 1 state.value = { status: 'pending' } failures = 0 + totalFailures = 0 reported = 0 suppressed = false exhausted = false