From e4755dea6ca139455def75cd24a20889199a7905 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 19:53:48 -0700 Subject: [PATCH] [backport cloud/1.55] fix(agent): stop retrying a permanently refused CRDT doc subscribe (#18704) Manual backport of #18704, preserving the 1.55 composition-root and reset implementations while integrating the permanent-refusal lifecycle, notification, and retry reset. --- src/locales/en/main.json | 3 + .../extensions/agent/AgentPanelRoot.test.ts | 28 ++- .../extensions/agent/AgentPanelRoot.vue | 11 +- .../agent/crdt/agentCrdtDocLifecycle.test.ts | 220 ++++++++++++++++++ .../agent/crdt/agentCrdtDocLifecycle.ts | 130 ++++++++++- .../extensions/agent/crdt/devPanelLog.ts | 1 + .../agent/crdt/useAgentCrdtFollower.test.ts | 130 +++++++++++ .../agent/crdt/useAgentCrdtFollower.ts | 61 ++++- .../crdt/workflowSyncErrorDetail.test.ts | 51 ++++ .../agent/crdt/workflowSyncErrorDetail.ts | 29 +++ 10 files changed, 648 insertions(+), 16 deletions(-) create mode 100644 src/workbench/extensions/agent/crdt/workflowSyncErrorDetail.test.ts create mode 100644 src/workbench/extensions/agent/crdt/workflowSyncErrorDetail.ts diff --git a/src/locales/en/main.json b/src/locales/en/main.json index 8e5388c3428..1a6e3442503 100644 --- a/src/locales/en/main.json +++ b/src/locales/en/main.json @@ -5366,6 +5366,9 @@ "workflowReferenceUnavailableReason": "This referenced workflow is no longer available or you do not have access to it.", "removeWorkflowReference": "Remove {name} reference", "targetNavigationUnavailable": "This workflow target is no longer available.", + "workflowSyncFailedTitle": "This workflow can't sync", + "workflowSyncFailedDetail": "This workflow was created with an incompatible document version. Your canvas has been left unchanged.", + "workflowSyncFailedDetailCatalogMismatch": "This workflow uses a node catalog that is incompatible with this workspace. Your canvas has been left unchanged.", "runModeTriggerAutoLimit": "Auto (limited)", "runModeTriggerAutoLimitTooltip": "Ask when credit limit is reached", "showMore": "Show more", diff --git a/src/workbench/extensions/agent/AgentPanelRoot.test.ts b/src/workbench/extensions/agent/AgentPanelRoot.test.ts index 3088d55b0b1..8f3ff94079e 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.test.ts +++ b/src/workbench/extensions/agent/AgentPanelRoot.test.ts @@ -89,7 +89,9 @@ const ws = vi.hoisted(() => { listeners.get(type)?.delete(listener) } const emit = (type: string, data?: unknown): void => { - for (const listener of listeners.get(type) ?? []) listener({ detail: data }) + // The doc-frame pipeline ignores events that are not CustomEvent instances. + const event = new CustomEvent(type, { detail: data }) + for (const listener of listeners.get(type) ?? []) listener(event) } const clear = (): void => listeners.clear() return { add, remove, emit, clear } @@ -4342,6 +4344,30 @@ describe('AgentPanelRoot workflow binding', () => { ).toBeInTheDocument() }) + it('shows a sticky error toast when the doc-host permanently refuses the subscribe', async () => { + makeTab('wf-42') + mockMessagesEndpoint('wf-42') + + await renderAndSend('add a node') + + ws.emit('doc_subscribed', { + v: 1, + workflow_id: 'wf-42', + ok: false, + code: 'schema_version_mismatch', + message: 'Expected schema 2, found 1' + }) + + expect(useToastStore().messagesToAdd).toContainEqual( + expect.objectContaining({ + severity: 'error', + summary: i18n.global.t('agent.workflowSyncFailedTitle'), + detail: `${i18n.global.t('agent.workflowSyncFailedDetail')} (Expected schema 2, found 1)`, + life: 0 + }) + ) + }) + it('moves the spinner to the tab the agent creates mid-turn', async () => { makeTab('wf-42') mockMessagesEndpoint('wf-42') diff --git a/src/workbench/extensions/agent/AgentPanelRoot.vue b/src/workbench/extensions/agent/AgentPanelRoot.vue index ab2dc1e6457..02a7d98fc06 100644 --- a/src/workbench/extensions/agent/AgentPanelRoot.vue +++ b/src/workbench/extensions/agent/AgentPanelRoot.vue @@ -21,6 +21,7 @@ import { useTelemetry } from '@/platform/telemetry' import { useSettingStore } from '@/platform/settings/settingStore' import type { LiveAutogrowGroupAnswer } from '@/workbench/extensions/agent/crdt/graphMutations' import { createGraphMutations } from '@/workbench/extensions/agent/crdt/graphMutations' +import { formatWorkflowSyncErrorDetail } from '@/workbench/extensions/agent/crdt/workflowSyncErrorDetail' import { useWorkflowService } from '@/platform/workflow/core/services/workflowService' import type { ComfyWorkflow } from '@/platform/workflow/management/stores/comfyWorkflow' import { useWorkflowStore } from '@/platform/workflow/management/stores/workflowStore' @@ -684,7 +685,15 @@ const { if (status.value === 'idle') graphActivity.finishTurn() } }, - onReset: graphActivity.resetWorkflow + onReset: graphActivity.resetWorkflow, + onSyncError: (message, code) => + toast.add({ + severity: 'error', + summary: t('agent.workflowSyncFailedTitle'), + detail: formatWorkflowSyncErrorDetail(t, message, code), + // A permanent desync remains visible until the person dismisses it. + life: 0 + }) } ) // The bound document's serialized root graph id, independent of what is diff --git a/src/workbench/extensions/agent/crdt/agentCrdtDocLifecycle.test.ts b/src/workbench/extensions/agent/crdt/agentCrdtDocLifecycle.test.ts index b0a317aeae3..53469ddfcae 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtDocLifecycle.test.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtDocLifecycle.test.ts @@ -428,3 +428,223 @@ describe('AgentCrdtDocLifecycle refusal exhaustion', () => { ]) }) }) + +describe('AgentCrdtDocLifecycle schema_version_mismatch refusal', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + it('gives up immediately with no retry and reports it once', () => { + const { lifecycle, resubscribe, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + + lifecycle.onSubscribeRefused('schema_version_mismatch') + vi.advanceTimersByTime(10 * SUBSCRIBE_ACK_TIMEOUT_MS) + + expect(resubscribe).not.toHaveBeenCalled() + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(lifecycle.shouldDeferSubscribe()).toBe(true) + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + GAVE_UP_REPORT + ) + expect(devEvents()).toEqual([ + { + kind: 'subscribe_refused_permanent', + detail: { code: 'schema_version_mismatch', terminal: true } + } + ]) + }) + + it('a different refusal code still retries with backoff', () => { + const { lifecycle, resubscribe, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + + lifecycle.onSubscribeRefused('doc_not_found') + vi.advanceTimersByTime(500) + + expect(resubscribe).toHaveBeenCalledTimes(1) + expect(onGaveUp).not.toHaveBeenCalled() + expect(reportError).not.toHaveBeenCalled() + }) + + it('a reconnect releases the permanent give-up latch', () => { + const { lifecycle, resubscribe, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + expect(onGaveUp).toHaveBeenCalledTimes(1) + + lifecycle.onReconnected() + lifecycle.onSubscribeSent(WORKFLOW_ID) + vi.advanceTimersByTime(SUBSCRIBE_ACK_TIMEOUT_MS) + + expect(resubscribe).toHaveBeenCalledTimes(1) + }) + + it('a redelivered refusal after already giving up does not re-run the latch', () => { + const { lifecycle, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledTimes(1) + vi.mocked(recordDevEvent).mockClear() + + // A duplicate/redelivered doc_subscribed refusal frame for the same, + // already-latched attempt must not double the toast/telemetry. + lifecycle.onSubscribeRefused('schema_version_mismatch') + + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledTimes(1) + expect(devEvents()).toEqual([]) + }) + + it('giving up permanently cancels an already-armed retryable backoff', () => { + const { lifecycle, resubscribe } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + // Schedules a 500 ms retryable backoff. + lifecycle.onSubscribeRefused('doc_not_found') + // A later permanent refusal must cancel that still-armed timer, not let + // it fire a resubscribe after the channel has been given up on. + lifecycle.onSubscribeRefused('schema_version_mismatch') + + vi.advanceTimersByTime(10 * SUBSCRIBE_ACK_TIMEOUT_MS) + + expect(resubscribe).not.toHaveBeenCalled() + }) + + it.for(['schema_version_mismatch', 'catalog_mismatch'] as const)( + '%s gives up immediately with no retry and reports it once', + (code) => { + const { lifecycle, resubscribe, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + + lifecycle.onSubscribeRefused(code) + vi.advanceTimersByTime(10 * SUBSCRIBE_ACK_TIMEOUT_MS) + + expect(resubscribe).not.toHaveBeenCalled() + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(lifecycle.shouldDeferSubscribe()).toBe(true) + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + GAVE_UP_REPORT + ) + expect(devEvents()).toEqual([ + { + kind: 'subscribe_refused_permanent', + detail: { code, terminal: true } + } + ]) + } + ) + + it('unsupported gives up immediately with no retry but never notifies', () => { + const { lifecycle, resubscribe, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + + lifecycle.onSubscribeRefused('unsupported') + vi.advanceTimersByTime(10 * SUBSCRIBE_ACK_TIMEOUT_MS) + + expect(resubscribe).not.toHaveBeenCalled() + expect(lifecycle.shouldDeferSubscribe()).toBe(true) + expect(onGaveUp).not.toHaveBeenCalled() + expect(reportError).not.toHaveBeenCalled() + expect(devEvents()).toEqual([]) + }) + + it('a reconnect does not re-notify a permanent refusal for the same workflow', () => { + const { lifecycle, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledTimes(1) + + lifecycle.onReconnected() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledTimes(1) + + lifecycle.onReconnected() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledTimes(1) + }) + + it('a successful confirm after a permanent refusal re-arms notification for a later independent refusal', () => { + const { lifecycle, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledTimes(1) + + // The reconnect recovers this time: the doc-host confirms the subscribe. + lifecycle.onReconnected() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeConfirmed() + + // A later, independent permanent refusal for the SAME workflow is a new + // failure episode, not a duplicate of the one already reported - it + // must notify again rather than being suppressed by the stale marker. + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + + expect(onGaveUp).toHaveBeenCalledTimes(2) + expect(reportError).toHaveBeenCalledTimes(2) + }) + + it('a stale permanent refusal answering a superseded attempt does not undo a confirm that already won', () => { + const { lifecycle, resubscribe, onGaveUp } = wire() + // Attempt 1 leaves the transport and gets no answer within the ack + // budget, so the timeout fires a second attempt while attempt 1 is + // still outstanding at the doc-host. + lifecycle.onSubscribeSent(WORKFLOW_ID) + vi.advanceTimersByTime(SUBSCRIBE_ACK_TIMEOUT_MS) + expect(resubscribe).toHaveBeenCalledTimes(1) + + // Attempt 2 confirms first - the channel is healthy again. + lifecycle.onSubscribeConfirmed() + expect(lifecycle.shouldDeferSubscribe()).toBe(false) + + // Attempt 1's answer finally lands, refusing permanently. It cannot be + // answering attempt 2 (which already confirmed), so it must be + // attempt 1's stale answer - it must not undo the confirmed channel. + const notified = lifecycle.onSubscribeRefused('schema_version_mismatch') + + expect(notified).toBe(false) + expect(onGaveUp).not.toHaveBeenCalled() + expect(reportError).not.toHaveBeenCalled() + expect(lifecycle.shouldDeferSubscribe()).toBe(false) + expect(devEvents()).toEqual([ + { + kind: 'subscribe_ack_timeout', + detail: { attempt: 1, workflowId: WORKFLOW_ID } + } + ]) + + // A genuinely new failure after a fresh send still latches normally. + lifecycle.onSubscribeSent(WORKFLOW_ID) + const notifiedAgain = lifecycle.onSubscribeRefused( + 'schema_version_mismatch' + ) + expect(notifiedAgain).toBe(true) + expect(onGaveUp).toHaveBeenCalledTimes(1) + }) + + it('clearForRetarget re-arms notification for the same workflow id', () => { + const { lifecycle, onGaveUp } = wire() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + expect(onGaveUp).toHaveBeenCalledTimes(1) + expect(reportError).toHaveBeenCalledTimes(1) + + lifecycle.clearForRetarget() + lifecycle.onSubscribeSent(WORKFLOW_ID) + lifecycle.onSubscribeRefused('schema_version_mismatch') + + expect(onGaveUp).toHaveBeenCalledTimes(2) + expect(reportError).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/workbench/extensions/agent/crdt/agentCrdtDocLifecycle.ts b/src/workbench/extensions/agent/crdt/agentCrdtDocLifecycle.ts index cf97fd217c8..56af873228b 100644 --- a/src/workbench/extensions/agent/crdt/agentCrdtDocLifecycle.ts +++ b/src/workbench/extensions/agent/crdt/agentCrdtDocLifecycle.ts @@ -39,6 +39,31 @@ const DOC_ID_REFRESH_INTERVAL_MS = DOC_ID_TTL_MS / 2 const SUBSCRIBE_RETRY_BASE_MS = 500 const SUBSCRIBE_RETRY_MAX_ATTEMPTS = 6 +// PM-1604 / BE-11437: the doc-host's terminal classifications for a +// document this build can never read back - unlike every other refusal +// reason, no amount of backoff retry changes that outcome, so these codes +// skip the retry ladder entirely and latch the same give-up exit an +// unanswered ack times out into. `schema_version_mismatch` and +// `catalog_mismatch` are cloud's `socketDocCode()` sibling arms sharing the +// same "retrying will not help" contract; `unsupported` (`docSurfaceOffReason`) +// is permanent for the life of the connection. +const PERMANENT_SUBSCRIBE_REFUSAL_CODES: ReadonlySet = new Set([ + 'schema_version_mismatch', + 'catalog_mismatch', + 'unsupported' +]) + +// `unsupported` means the doc surface is off for this whole deployment, so +// every user hits it - latching it must not paint an "incompatible document +// version" toast that has nothing to do with any one person's workflow. +const SILENT_PERMANENT_REFUSAL_CODES: ReadonlySet = new Set([ + 'unsupported' +]) + +function isNotifyingPermanentRefusalCode(code: string): boolean { + return !SILENT_PERMANENT_REFUSAL_CODES.has(code) +} + /** * A `doc_subscribe` that left the transport and was never answered is retried * on the same lineage after this long. The ingest relay's own resync budget is @@ -162,6 +187,23 @@ export class AgentCrdtDocLifecycle { // a status-frame reconcile can turn the bounded retry into an unbounded one. // Released only by a lifecycle edge: confirm, reconnect, retarget. private gaveUp = false + // The workflow id last notified (reportError + onGaveUp) for a permanent + // refusal. A reconnect to the SAME workflow answered with the SAME + // permanent code is one unchanged fact, not a new one - re-arm only when + // the target actually changes (retarget/doc_reset) or a subscribe for it + // actually succeeds (`onSubscribeConfirmed`), never on a bare reconnect + // that never recovers. See `resetNotifiedGiveUp`. + private lastNotifiedGiveUpWorkflowId: string | null = null + // Stale-ack race: an ack-timeout resubscribe can leave the OLDER attempt + // still outstanding at the doc-host while a NEWER one (sent on the same + // lineage) confirms first. A confirm and a permanent refusal can never + // both be the answer to the SAME subscribe, so once this is true, any + // permanent refusal that arrives before another subscribe is sent cannot + // be answering the one that just confirmed - it must be the older + // attempt's late answer - and `onSubscribeRefused` discards it instead of + // tearing down the confirm's own probe/latch. Cleared by the next actual + // send (`onSubscribeSent`), which starts a fresh round. + private confirmedSinceLastSend = false // FEC-5: `Date.now()` of the last persisted-record write by this instance. // A confirmed subscribe always writes; doc-scoped frames re-stamp the expiry // no more often than DOC_ID_REFRESH_INTERVAL_MS, so a doc that keeps @@ -189,6 +231,11 @@ export class AgentCrdtDocLifecycle { onSubscribeConfirmed(): void { this.clearAckTimer() this.gaveUp = false + this.confirmedSinceLastSend = true + // A successful confirm ends the outage a prior permanent refusal + // notified about: a LATER independent refusal for this same workflow is + // a new failure episode, not a duplicate of the one already reported. + this.resetNotifiedGiveUp() this.clearSubscribeRetry() if (this.usedCatchUpGrace) { this.armStaleProbe() @@ -200,14 +247,26 @@ export class AgentCrdtDocLifecycle { if (workflowId !== null) this.persistConfirmedDocId(workflowId) } - onSubscribeRefused(): void { + /** Returns whether this refusal newly triggered a give-up notification. */ + onSubscribeRefused(code?: string): boolean { + if (code !== undefined && PERMANENT_SUBSCRIBE_REFUSAL_CODES.has(code)) { + // See `confirmedSinceLastSend`: bail out before touching any timer, + // so a stale straggler can't clear the probe/backoff state the + // confirm it raced behind already armed. + if (this.confirmedSinceLastSend) return false + this.clearAckTimer() + this.clearStaleProbe() + return this.giveUpPermanently(code, isNotifyingPermanentRefusalCode(code)) + } this.clearAckTimer() this.clearStaleProbe() this.scheduleSubscribeRetry() + return false } onSubscribeSent(workflowId: string): void { this.clearAckTimer() + this.confirmedSinceLastSend = false if (this.gaveUp) return this.ackTimer = setTimeout(() => { this.ackTimer = null @@ -229,9 +288,15 @@ export class AgentCrdtDocLifecycle { }, SUBSCRIBE_ACK_TIMEOUT_MS) } - /** A new socket is a new server-side session: every budget starts over. */ + /** + * A new socket is a new server-side session: every budget starts over, + * including the permanent-give-up latch (a redeployed host may now be + * readable). `lastNotifiedGiveUpWorkflowId` deliberately survives this - + * see `resetNotifiedGiveUp` - so a refusal that recurs for the same + * workflow after reconnecting doesn't re-fire the notification. + */ onReconnected(): void { - this.clearForRetarget() + this.resetForNewAttempt() } onDocumentUpdate(): void { @@ -259,11 +324,19 @@ export class AgentCrdtDocLifecycle { } clearForRetarget(): void { - this.clearAckTimer() - this.clearSubscribeRetry() - this.clearStaleProbe() - this.gaveUp = false - this.usedCatchUpGrace = false + this.resetForNewAttempt() + this.resetNotifiedGiveUp() + } + + /** + * Lets a permanent refusal for a different episode of the same workflow + * id notify again. Called when the subscribed target actually changes + * (retarget), its lineage restarts (`doc_reset`), or it actually confirms + * (`onSubscribeConfirmed`) - never on a bare reconnect that never + * recovers, which keeps the same episode and the same unrecoverable fact. + */ + resetNotifiedGiveUp(): void { + this.lastNotifiedGiveUpWorkflowId = null } destroy(): void { @@ -298,6 +371,14 @@ export class AgentCrdtDocLifecycle { }, delayMs) } + private resetForNewAttempt(): void { + this.clearAckTimer() + this.clearSubscribeRetry() + this.clearStaleProbe() + this.gaveUp = false + this.usedCatchUpGrace = false + } + private clearAckTimer(): void { if (this.ackTimer !== null) { clearTimeout(this.ackTimer) @@ -305,6 +386,39 @@ export class AgentCrdtDocLifecycle { } } + /** Returns whether this call actually notified (reportError + onGaveUp). */ + private giveUpPermanently(code: string, notify: boolean): boolean { + // A redelivered/duplicate refusal frame answers nothing new once the + // latch is already set — without this guard it would re-run the report + // and the caller's `onGaveUp` (toast/telemetry) a second time. + if (this.gaveUp) return false + this.gaveUp = true + // An already-armed retry from an earlier, retryable refusal must not + // survive the permanent latch: its callback checks only `workflowId()`, + // not `gaveUp`, so left alone it would still fire a resubscribe later. + this.clearSubscribeRetry() + if (!notify) return false + const workflowId = this.workflowId() + // A reconnect that lands the same permanent code for the same workflow + // is one unchanged fact - only a different workflow (retarget/doc_reset, + // see `resetNotifiedGiveUp`) is a new one worth telling the person about. + if (workflowId !== null && workflowId === this.lastNotifiedGiveUpWorkflowId) + return false + this.lastNotifiedGiveUpWorkflowId = workflowId + recordDevEvent( + 'subscribe_refused_permanent', + { code, terminal: true }, + { level: 'warn' } + ) + reportError(new Error(`agent doc subscribe permanently refused: ${code}`), { + errorType: 'failure_confirming_agent_doc_subscribe', + level: 'warning', + tags: { feature_area: 'agent', operation: 'sync', outcome: 'gave_up' } + }) + this.onGaveUp() + return true + } + private giveUp(workflowId: string): void { this.clearStaleProbe() this.gaveUp = true diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.ts b/src/workbench/extensions/agent/crdt/devPanelLog.ts index 63c7614897b..ddddc952ea9 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.ts @@ -35,6 +35,7 @@ export const DEV_EVENT_KINDS = [ 'reconnected', 'subscribe_retry', 'subscribe_ack_timeout', + 'subscribe_refused_permanent', 'stale_probe', 'catchup_probe', 'rebind', diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index ae12665a124..4dc91650ebc 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -742,6 +742,88 @@ describe('useAgentCrdtFollower', () => { unmount() }) + it('PM-1604 / BE-11437: retries a retryable refusal code with backoff', () => { + vi.useFakeTimers() + const { unmount } = mountFollower('wf-1') + dispatchFrame('doc_subscribed', { ok: false, code: 'not_found' }) + + vi.advanceTimersByTime(500) + expect(bridge().resubscribe).toHaveBeenCalledTimes(1) + unmount() + }) + + it('PM-1604 / BE-11437: does not retry schema_version_mismatch and surfaces it', () => { + vi.useFakeTimers() + const onSyncError = vi.fn() + const { unmount } = mountFollower('wf-1', true, () => null, { + onSyncError + }) + + dispatchFrame('doc_subscribed', { + ok: false, + code: 'schema_version_mismatch', + message: 'Expected schema 2, found 1' + }) + + vi.advanceTimersByTime(60_000) + apiState.target.dispatchEvent(new Event('status')) + + expect(bridge().resubscribe).not.toHaveBeenCalled() + expect(bridge().reconcile).not.toHaveBeenCalled() + expect(onSyncError).toHaveBeenCalledExactlyOnceWith( + 'Expected schema 2, found 1', + 'schema_version_mismatch' + ) + unmount() + }) + + it('PM-1604 / BE-11437: does not retry catalog_mismatch and surfaces it', () => { + vi.useFakeTimers() + const onSyncError = vi.fn() + const { unmount } = mountFollower('wf-1', true, () => null, { + onSyncError + }) + + dispatchFrame('doc_subscribed', { + ok: false, + code: 'catalog_mismatch', + message: 'catalog v3 required' + }) + + vi.advanceTimersByTime(60_000) + apiState.target.dispatchEvent(new Event('status')) + + expect(bridge().resubscribe).not.toHaveBeenCalled() + expect(bridge().reconcile).not.toHaveBeenCalled() + expect(onSyncError).toHaveBeenCalledExactlyOnceWith( + 'catalog v3 required', + 'catalog_mismatch' + ) + unmount() + }) + + it('PM-1604 / BE-11437: does not retry unsupported and never surfaces it', () => { + vi.useFakeTimers() + const onSyncError = vi.fn() + const { unmount } = mountFollower('wf-1', true, () => null, { + onSyncError + }) + + dispatchFrame('doc_subscribed', { + ok: false, + code: 'unsupported', + message: 'doc surface disabled' + }) + + vi.advanceTimersByTime(60_000) + apiState.target.dispatchEvent(new Event('status')) + + expect(bridge().resubscribe).not.toHaveBeenCalled() + expect(bridge().reconcile).not.toHaveBeenCalled() + expect(onSyncError).not.toHaveBeenCalled() + unmount() + }) + it('drops to disconnected on a schema error without touching the binding', () => { const { unmount, status } = mountFollower('wf-1') dispatchFrame('doc_subscribed', { ok: true }) @@ -1432,6 +1514,54 @@ describe('useAgentCrdtFollower', () => { unmount() }) + it('settles a stranded in-flight batch before notifying onSyncError of a permanent refusal', async () => { + vi.useFakeTimers() + const { recordDevEvent } = await import('./devPanelLog') + const workflowId = ref('wf-1') + const onSyncError = vi.fn() + let enqueue!: ReturnType< + typeof useAgentCrdtFollower + >['enqueueHumanOperations'] + const host = defineComponent({ + setup() { + const { enqueueHumanOperations } = useAgentCrdtFollower( + workflowId, + graphMutations, + () => null, + ref(true), + () => null, + { onSyncError } + ) + enqueue = enqueueHumanOperations + return () => null + } + }) + const { unmount } = render(host) + + enqueue([{ op: 'delete_node', node_id: '1', removed_links: [] }]) + await Promise.resolve() + expect(clientState.sendOps).toHaveBeenCalledTimes(1) + + bridge().subscribedWorkflowId = null + dispatchFrame('doc_subscribed', { + ok: false, + workflowId: 'wf-1', + code: 'schema_version_mismatch' + }) + + expect(onSyncError).toHaveBeenCalledTimes(1) + const settleCallIndex = vi + .mocked(recordDevEvent) + .mock.calls.findIndex(([event]) => event === 'human_ops_settled') + expect(settleCallIndex).toBeGreaterThanOrEqual(0) + const settleOrder = + vi.mocked(recordDevEvent).mock.invocationCallOrder[settleCallIndex] + // onDocReset's precedent: cleanup that could strand held/in-flight ops + // runs before notifying consumer code (the toast store) that could throw. + expect(settleOrder).toBeLessThan(onSyncError.mock.invocationCallOrder[0]) + unmount() + }) + it('a doc switch settles the transmitted in-flight batch for the old doc unconfirmed immediately, without waiting the resend', async () => { vi.useFakeTimers() const { recordDevEvent } = await import('./devPanelLog') diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index a4493b496ae..0eed59eb1f1 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -127,6 +127,36 @@ function emitPendingMaterializations( events.onMaterialized?.({ workflowId, actor, nodeIds }) } +interface SubscribeRefusalOutcome { + shouldNotify: boolean + message?: string + code?: string +} + +// PM-1604 / BE-11437: a subscribe refusal carries a `code` that is either +// retryable (the lifecycle keeps retrying on its own) or one of +// `PERMANENT_SUBSCRIBE_REFUSAL_CODES`, which the lifecycle won't recover +// from by itself — surface those to the person via `onSyncError`, except +// `unsupported`, which the lifecycle already declines to notify (a +// deployment with the doc surface off shouldn't toast every user). The +// caller notifies only after its own held-ops cleanup, matching +// `onDocReset`'s cleanup-before-notify order, so a throw from consumer code +// reaching into the toast store can't strand an in-flight op batch. `code` +// rides along so the presentation layer can pick accurate copy instead of +// collapsing every permanent code to the same message. +function handleSubscribeRefusal( + detail: { code?: unknown; message?: unknown } | null, + lifecycle: AgentCrdtDocLifecycle +): SubscribeRefusalOutcome { + const code = typeof detail?.code === 'string' ? detail.code : undefined + if (!lifecycle.onSubscribeRefused(code)) return { shouldNotify: false } + return { + shouldNotify: true, + message: typeof detail?.message === 'string' ? detail.message : undefined, + code + } +} + function notifyAgentMaterialization( update: ClassifiedDocUpdate, added: readonly string[], @@ -183,6 +213,20 @@ export interface AgentCrdtFollowerEvents { nodeIds: readonly NodeId[] }) => void onReset?: (workflowId: string) => void + /** + * PM-1604 / BE-11437: the doc-host classified a resync refusal as + * permanent (one of `PERMANENT_SUBSCRIBE_REFUSAL_CODES`) — the lifecycle + * has already stopped retrying it, so this is the one chance to tell the + * person their canvas is out of sync instead of leaving them to notice a + * channel that silently stopped updating. Not fired for `unsupported` + * (the doc surface is off for this deployment; every user hits it, so it + * latches silently) or for a refusal that repeats on reconnect for a + * workflow already notified. `code` is the doc-host's permanent refusal + * code (e.g. `schema_version_mismatch`, `catalog_mismatch`) so the + * presentation layer can choose accurate localized copy instead of a + * single message for every permanent reason. + */ + onSyncError?: (message?: string, code?: string) => void } // Nothing is re-thrown: an error escaping onBeforeUnmount reaches Vue's @@ -423,7 +467,12 @@ function startAgentCrdtFollower( const onSubscribed: EventListener = (event) => { if (!(event instanceof CustomEvent)) return if (!isTargetActive.value) return - const ok = event.detail?.ok === true + const detail = event.detail as { + ok?: unknown + code?: unknown + message?: unknown + } | null + const ok = detail?.ok === true connected.value = ok lastFrameType.value = event.type recordDevEvent('doc_subscribed', event.detail ?? null) @@ -431,12 +480,14 @@ function startAgentCrdtFollower( lifecycle.onSubscribeConfirmed() resumeHeldOpsIfSubscribed() } else { - lifecycle.onSubscribeRefused() + const refusal = handleSubscribeRefusal(detail, lifecycle) // FE #16637 residual: a refusal is the earliest signal the sender can // get that its in-flight batch's doc is gone — don't make it wait out // the 10 s result-silence window to notice on its own. releaseHeldOps() sender.abortIfUnbound() + if (refusal.shouldNotify) + events.onSyncError?.(refusal.message, refusal.code) } } const onUpdate: EventListener = (event) => { @@ -508,13 +559,11 @@ function startAgentCrdtFollower( updatesApplied.value = 0 lastFrameType.value = event.type lifecycle.clearStaleProbe() + lifecycle.resetNotifiedGiveUp() knownDocNodeIds = new Set() pendingLiveNodeIds.clear() confirmedDeletes.clear() - recordDevEvent( - 'doc_reset', - event instanceof CustomEvent ? (event.detail ?? null) : null - ) + recordDevEvent('doc_reset', detail) } const onFollowerReplaced: EventListener = (event) => { // Gate on this composable's own INTENT, not the bridge's send REALITY diff --git a/src/workbench/extensions/agent/crdt/workflowSyncErrorDetail.test.ts b/src/workbench/extensions/agent/crdt/workflowSyncErrorDetail.test.ts new file mode 100644 index 00000000000..d9e8ecc0be2 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/workflowSyncErrorDetail.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' + +import { formatWorkflowSyncErrorDetail } from './workflowSyncErrorDetail' + +const translate = (key: string): string => { + if (key === 'agent.workflowSyncFailedDetail') + return 'Your canvas is unchanged.' + if (key === 'agent.workflowSyncFailedDetailCatalogMismatch') + return 'Your node catalog is incompatible.' + return key +} + +describe('formatWorkflowSyncErrorDetail', () => { + it('falls back to the localized copy alone when there is no raw message', () => { + expect(formatWorkflowSyncErrorDetail(translate, undefined)).toBe( + 'Your canvas is unchanged.' + ) + }) + + it('appends a short raw message as extra context after the localized copy', () => { + expect( + formatWorkflowSyncErrorDetail(translate, 'Expected schema 2, found 1') + ).toBe('Your canvas is unchanged. (Expected schema 2, found 1)') + }) + + it('uses the catalog-mismatch copy when the code names that reason', () => { + expect( + formatWorkflowSyncErrorDetail(translate, undefined, 'catalog_mismatch') + ).toBe('Your node catalog is incompatible.') + }) + + it('falls back to the schema-mismatch copy for an undifferentiated code', () => { + expect( + formatWorkflowSyncErrorDetail( + translate, + undefined, + 'schema_version_mismatch' + ) + ).toBe('Your canvas is unchanged.') + }) + + it('clamps a multi-KB raw message instead of reproducing it in full', () => { + const hugeMessage = 'x'.repeat(5000) + + const result = formatWorkflowSyncErrorDetail(translate, hugeMessage) + + expect(result.startsWith('Your canvas is unchanged. (')).toBe(true) + // 200-char clamp plus the truncation ellipsis, well under the raw 5000. + expect(result.length).toBeLessThan(250) + }) +}) diff --git a/src/workbench/extensions/agent/crdt/workflowSyncErrorDetail.ts b/src/workbench/extensions/agent/crdt/workflowSyncErrorDetail.ts new file mode 100644 index 00000000000..061ef7c1178 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/workflowSyncErrorDetail.ts @@ -0,0 +1,29 @@ +import { truncate } from 'es-toolkit/compat' + +/** Cap on the doc-host's raw refusal message appended to the localized detail + * below — the host's `message` is developer text of unbounded length, not + * meant to replace the reassurance the localized copy gives the person. */ +const SYNC_ERROR_RAW_MESSAGE_MAX_LENGTH = 200 + +/** + * PM-1604 / BE-11437: the permanent-desync toast's `detail` always leads with + * the localized copy (so the person keeps the "canvas unchanged" reassurance + * even when the doc-host's raw `message` is untranslated developer text or + * absent), appending that raw message, clamped, as extra context only when + * present. `code` picks which localized copy leads: `catalog_mismatch` + * names an incompatible node catalog, not an incompatible document version, + * so it gets its own key rather than reusing `schema_version_mismatch`'s. + */ +export function formatWorkflowSyncErrorDetail( + translate: (key: string) => string, + message: string | undefined, + code?: string +): string { + const detail = translate( + code === 'catalog_mismatch' + ? 'agent.workflowSyncFailedDetailCatalogMismatch' + : 'agent.workflowSyncFailedDetail' + ) + if (!message) return detail + return `${detail} (${truncate(message, { length: SYNC_ERROR_RAW_MESSAGE_MAX_LENGTH })})` +}