From ace0045a28d5e3da58a50939f1b4fcec58454dfb Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Mon, 26 Oct 2026 20:55:18 +0000 Subject: [PATCH 1/4] fix(agent): keep the event socket recoverable on every death path The agent event source had four ways to end up permanently dead or permanently hot: - A credential read that resolved to `null` bare-returned: no status, no retry. The panel only came back if `AgentPanelRoot`'s account watcher happened to fire, so a token that was briefly unavailable (a refresh in flight) killed the socket for the life of the panel. It now retries on the backoff, like the rejected-read branch next to it. - A single inbound frame reset the backoff, which defeated the minimum-duration reset in the `close` handler: a server that accepts the upgrade, writes one frame and hangs up produced a 1s reconnect loop that never widened. Only a connection that lasted clears the backoff now. - A `getAuthParams()` promise that never settled latched `connecting` true, and `connect()` early-returns on it, so even an explicit `reconnect()` was a no-op. `disconnect()` clears the latch. The generation-mismatch recursion existed only to work around that latch and goes away with it. - `new WebSocket(url)` was unguarded; a constructor throw (SecurityError on a blocked scheme or port) left the source dead with nothing scheduled, and surfaced as an unhandled rejection. It now reports the drop and schedules a reconnect. Each is covered by a test that fails without its fix. --- .../services/agent/agentEventSource.test.ts | 91 ++++++++++++++++++- .../agent/services/agent/agentEventSource.ts | 27 ++++-- 2 files changed, 105 insertions(+), 13 deletions(-) diff --git a/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts b/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts index 8a3d0bfbba3..17eb2726e2c 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts @@ -240,7 +240,7 @@ describe('createAgentEventSource', () => { }) describe('createAgentEventSource reconnect backoff', () => { - it('doubles the delay per refused connect and resets once a socket delivers a frame', async () => { + it('doubles the delay per refused connect', async () => { const source = createSource() source.subscribe(vi.fn()) @@ -252,12 +252,52 @@ describe('createAgentEventSource reconnect backoff', () => { expect(sockets()).toHaveLength(2) await vi.advanceTimersByTimeAsync(1) expect(sockets()).toHaveLength(3) + }) + + it('keeps backing off when the server accepts, sends one frame and closes', async () => { + const source = createSource() + + source.subscribe(vi.fn()) + await connected() + for (const wait of [1000, 2000, 4000]) { + lastSocket().open() + lastSocket().receive({ type: 'agent_message_delta', data: {} }) + lastSocket().close() + const opened = sockets().length + await vi.advanceTimersByTimeAsync(wait - 1) + expect(sockets()).toHaveLength(opened) + await vi.advanceTimersByTimeAsync(1) + expect(sockets()).toHaveLength(opened + 1) + } + }) + + it('schedules a reconnect when the socket constructor throws', async () => { + let throwOnConstruct = true + vi.stubGlobal( + 'WebSocket', + Object.assign( + function WebSocketStub(url: string) { + if (!throwOnConstruct) return new FakeSocket(url) + throwOnConstruct = false + throw new DOMException('insecure socket', 'SecurityError') + }, + { + CONNECTING: FakeSocket.CONNECTING, + OPEN: FakeSocket.OPEN, + CLOSED: FakeSocket.CLOSED + } + ) + ) + const source = createSource() + const status = vi.fn() + source.onStatus(status) + + source.subscribe(vi.fn()) + await connected() + expect(sockets()).toHaveLength(0) - sockets()[2].open() - sockets()[2].receive({ type: 'agent_message_delta', data: {} }) - sockets()[2].close() await vi.advanceTimersByTimeAsync(1000) - expect(sockets()).toHaveLength(4) + expect(sockets()).toHaveLength(1) }) it('keeps backing off when the server accepts the upgrade and then closes', async () => { @@ -435,6 +475,47 @@ describe('createAgentEventSource token', () => { expect(sockets()).toHaveLength(0) }) + it('opens the socket once a later read supplies the missing credential', async () => { + const reads: (() => Promise)[] = [ + async () => null, + async () => tokenParams('signed-in') + ] + const source = createSource({ + getAuthParams: () => reads.shift()?.() ?? Promise.resolve(null) + }) + + source.subscribe(vi.fn()) + await connected() + expect(sockets()).toHaveLength(0) + + await vi.advanceTimersByTimeAsync(1000) + + expect( + sockets().map((socket) => new URL(socket.url).searchParams.get('token')) + ).toEqual(['signed-in']) + }) + + it('reconnects after an auth read that never settles', async () => { + const reads: (() => Promise)[] = [ + () => new Promise(() => {}), + async () => tokenParams('recovered') + ] + const source = createSource({ + getAuthParams: () => reads.shift()?.() ?? Promise.resolve(null) + }) + + source.subscribe(vi.fn()) + await connected() + expect(sockets()).toHaveLength(0) + + source.reconnect() + await connected() + + expect( + sockets().map((socket) => new URL(socket.url).searchParams.get('token')) + ).toEqual(['recovered']) + }) + it('carries every auth param onto the socket URL', async () => { const source = createAgentEventSource({ endpoint: '/api/agent/events', diff --git a/src/workbench/extensions/agent/services/agent/agentEventSource.ts b/src/workbench/extensions/agent/services/agent/agentEventSource.ts index f4cc1546fdc..70cd930addb 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventSource.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventSource.ts @@ -74,22 +74,28 @@ export function createAgentEventSource({ } finally { connecting = false } - if (generation !== connectGeneration) { - void connect() - return - } + if (generation !== connectGeneration) return if (listeners.size === 0) return - if (!auth.ok) { + // A credential that cannot be read right now — a rejected refresh, or a + // token the provider has not minted yet — is retried on the backoff rather + // than leaving the source dead until something else calls `reconnect()`. + if (!auth.ok || auth.params === null) { notifyStatus(false) scheduleReconnect() return } - if (auth.params === null) return open(eventUrl(auth.params)) } function open(url: string): void { - const current = new WebSocket(url) + let current: WebSocket + try { + current = new WebSocket(url) + } catch { + notifyStatus(false) + scheduleReconnect() + return + } socket = current let openedAt: number | null = null current.addEventListener('open', () => { @@ -99,7 +105,6 @@ export function createAgentEventSource({ }) current.addEventListener('message', (event) => { if (socket !== current) return - failedConnects = 0 let frame: unknown = event.data if (typeof event.data === 'string') { try { @@ -115,6 +120,9 @@ export function createAgentEventSource({ }) current.addEventListener('close', () => { if (socket !== current) return + // Only a connection that lasted clears the backoff. A server that + // accepts the upgrade, writes a frame and hangs up must not reset it, or + // the panel reconnects in a tight loop that never widens. if (openedAt !== null && Date.now() - openedAt >= maxReconnectDelayMs) failedConnects = 0 socket = null @@ -125,6 +133,9 @@ export function createAgentEventSource({ function disconnect(): void { connectGeneration++ + // An auth read that never settles would otherwise latch `connecting` and + // make every later `connect()` — `reconnect()` included — a no-op. + connecting = false reconnectTimer.stop() const current = socket socket = null From 56aac6366709410e931e918a090dec014f0f1c1b Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 28 Oct 2026 03:43:20 +0000 Subject: [PATCH 2/4] fix(agent): scope the connect latch to its generation disconnect() clears `connecting` and bumps the generation, but the stale connect() still awaiting its auth read then cleared the latch again in its unconditional finally -- and by that point the latch belonged to the newer generation's in-flight read. A subscribe() arriving in that window passed the guard and started a second read in the same generation, so both survivors called open() and the second overwrote `socket` without closing the first. Every handler on that orphan bails on `socket !== current`, including close, so it stayed connected for the life of the page and never reconnected. Only the generation that still owns the latch may release it, which makes one auth read per generation reach open(). The test fails without the change. --- .../services/agent/agentEventSource.test.ts | 35 +++++++++++++++++++ .../agent/services/agent/agentEventSource.ts | 23 +++++++----- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts b/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts index 17eb2726e2c..ae066e7990f 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts @@ -569,4 +569,39 @@ describe('createAgentEventSource send', () => { expect(source.send('{"type":"doc_subscribe"}')).toBe(true) expect(lastSocket().send).toHaveBeenCalledWith('{"type":"doc_subscribe"}') }) + it('does not open a second socket when a stale auth read settles mid-reconnect', async () => { + // The stale read's `finally` used to clear `connecting` unconditionally, releasing + // the latch held by the newer generation's in-flight read. A subscribe arriving in + // that window started a third read, and both survivors called open(), so the second + // overwrote `socket` and the first stayed connected forever with every handler + // bailing on `socket !== current`. + const pending: Array<(params: URLSearchParams) => void> = [] + const source = createSource({ + getAuthParams: () => + new Promise((resolve) => pending.push(resolve)) + }) + + source.subscribe(() => {}) + await connected() + expect(pending).toHaveLength(1) + + source.reconnect() + await connected() + expect(pending).toHaveLength(2) + + pending[0](tokenParams('stale')) + await connected() + + source.subscribe(() => {}) + await connected() + expect(pending).toHaveLength(2) + + pending[1](tokenParams('current')) + await connected() + pending.at(-1)?.(tokenParams('extra')) + await connected() + + expect(sockets()).toHaveLength(1) + expect(lastSocket().url).toContain('token=current') + }) }) diff --git a/src/workbench/extensions/agent/services/agent/agentEventSource.ts b/src/workbench/extensions/agent/services/agent/agentEventSource.ts index 70cd930addb..5b9b8b64e6e 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventSource.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventSource.ts @@ -62,18 +62,25 @@ export function createAgentEventSource({ reconnectTimer.start() } - async function connect(): Promise { - if (listeners.size === 0 || socket !== null || connecting) return - connecting = true - const generation = connectGeneration - let auth: AuthRead + async function readAuth(generation: number): Promise { try { - auth = { ok: true, params: await getAuthParams() } + return { ok: true, params: await getAuthParams() } } catch { - auth = { ok: false } + return { ok: false } } finally { - connecting = false + // Only the generation that still owns the latch may release it. `disconnect()` + // clears `connecting` and bumps the generation, so a newer `connect()` can + // already be awaiting its own auth read by the time this stale one settles -- + // releasing the latch here would let a third read start in that generation. + if (generation === connectGeneration) connecting = false } + } + + async function connect(): Promise { + if (listeners.size === 0 || socket !== null || connecting) return + connecting = true + const generation = connectGeneration + const auth = await readAuth(generation) if (generation !== connectGeneration) return if (listeners.size === 0) return // A credential that cannot be read right now — a rejected refresh, or a From 0811274e176b573bc3dda6da615a016a63f79d8c Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 28 Oct 2026 06:44:56 +0000 Subject: [PATCH 3/4] fix(agent): release the connect latch after the generation check Clearing it inside readAuth's finally left a window between the release and connect()'s continuation. A subscribe() landing there started a second auth read in the same generation, both reached open(), and the second overwrote socket without closing the first -- the same orphan the generation scoping was meant to prevent, reached by a narrower path. Everything from the release to open() is now synchronous, so the window cannot exist. A stale read still leaves the latch alone, since disconnect() has already cleared it and it belongs to a newer connect() by then. --- .../agent/services/agent/agentEventSource.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/workbench/extensions/agent/services/agent/agentEventSource.ts b/src/workbench/extensions/agent/services/agent/agentEventSource.ts index 5b9b8b64e6e..84d1e38c886 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventSource.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventSource.ts @@ -62,17 +62,11 @@ export function createAgentEventSource({ reconnectTimer.start() } - async function readAuth(generation: number): Promise { + async function readAuth(): Promise { try { return { ok: true, params: await getAuthParams() } } catch { return { ok: false } - } finally { - // Only the generation that still owns the latch may release it. `disconnect()` - // clears `connecting` and bumps the generation, so a newer `connect()` can - // already be awaiting its own auth read by the time this stale one settles -- - // releasing the latch here would let a third read start in that generation. - if (generation === connectGeneration) connecting = false } } @@ -80,8 +74,16 @@ export function createAgentEventSource({ if (listeners.size === 0 || socket !== null || connecting) return connecting = true const generation = connectGeneration - const auth = await readAuth(generation) + const auth = await readAuth() + // A stale read must leave the latch alone: `disconnect()` already cleared it and + // bumped the generation, so by now it belongs to a newer `connect()`. if (generation !== connectGeneration) return + // Released here rather than in `readAuth`, because everything from this point to + // `open()` is synchronous. Clearing it before this continuation ran left a window + // in which a `subscribe()` could start a second read in the same generation; both + // would reach `open()`, and the second would overwrite `socket` without closing + // the first, leaving an orphan whose handlers all bail on `socket !== current`. + connecting = false if (listeners.size === 0) return // A credential that cannot be read right now — a rejected refresh, or a // token the provider has not minted yet — is retried on the backoff rather From 4a63d882bc6194d558dbc49c4bc5cf2405fac95b Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Wed, 28 Oct 2026 07:01:12 +0000 Subject: [PATCH 4/4] test(agent): cover the one-microtask latch race Pins the window closed by the previous commit: resolve the auth read, let a single microtask run so the old arrangement would have released the latch, then subscribe. Against that arrangement this opens two sockets; it opens one now. --- .../services/agent/agentEventSource.test.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts b/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts index ae066e7990f..24c6f1d73e7 100644 --- a/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts +++ b/src/workbench/extensions/agent/services/agent/agentEventSource.test.ts @@ -604,4 +604,28 @@ describe('createAgentEventSource send', () => { expect(sockets()).toHaveLength(1) expect(lastSocket().url).toContain('token=current') }) + it('does not open a second socket when a subscribe lands as the auth read settles', async () => { + // The window this closes is one microtask wide: with the latch released inside + // readAuth, it was already false by the time connect() resumed, so a subscribe() + // arriving in between started a second read in the SAME generation. Both reached + // open() and the second overwrote `socket`, orphaning the first. + const pending: Array<(params: URLSearchParams) => void> = [] + const source = createSource({ + getAuthParams: () => + new Promise((resolve) => pending.push(resolve)) + }) + + source.subscribe(() => {}) + await connected() + expect(pending).toHaveLength(1) + + pending[0](tokenParams('first')) + await Promise.resolve() + source.subscribe(() => {}) + await connected() + pending.at(-1)?.(tokenParams('second')) + await connected() + + expect(sockets()).toHaveLength(1) + }) })