From e82040d830088988ad59dfe6dfc976bbdc1df360 Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Mon, 12 Oct 2026 22:41:57 +0000 Subject: [PATCH 1/8] fix: report lost agent edits through platform telemetry Amp-Thread-ID: https://ampcode.com/threads/T-01a1323c-7d66-709b-9454-cfc5463f9c65 --- src/platform/telemetry/reportError.test.ts | 4 +- src/platform/telemetry/reportError.ts | 10 ++- .../extensions/agent/crdt/docOpMinter.test.ts | 23 +++++ .../extensions/agent/crdt/docOpMinter.ts | 15 ++-- .../extensions/agent/crdt/opSender.test.ts | 86 ++++++++++++++++--- .../extensions/agent/crdt/opSender.ts | 3 + .../extensions/agent/crdt/rejectedOpNotice.ts | 74 ++++++---------- .../extensions/agent/crdt/reportLostEdit.ts | 39 +++++++++ ...ntCrdtFollower.rejectedWidgetWrite.test.ts | 26 +++--- .../agent/crdt/useAgentCrdtFollower.test.ts | 5 +- .../agent/crdt/useAgentCrdtFollower.ts | 45 +++------- 11 files changed, 207 insertions(+), 123 deletions(-) create mode 100644 src/workbench/extensions/agent/crdt/reportLostEdit.ts diff --git a/src/platform/telemetry/reportError.test.ts b/src/platform/telemetry/reportError.test.ts index 938bd6615ba..696f4bba2d8 100644 --- a/src/platform/telemetry/reportError.test.ts +++ b/src/platform/telemetry/reportError.test.ts @@ -72,12 +72,14 @@ describe('reportError', () => { reportError(error, { surface: 'platform', - errorType: 'workspace_auth_gate_initialization_failure' + errorType: 'workspace_auth_gate_initialization_failure', + fingerprint: ['workspace_auth_gate_initialization_failure'] }) expect(captureException).toHaveBeenCalledWith( error, expect.objectContaining({ + fingerprint: ['workspace_auth_gate_initialization_failure'], tags: expect.objectContaining({ bucket: 'canary' }) }) ) diff --git a/src/platform/telemetry/reportError.ts b/src/platform/telemetry/reportError.ts index 2f35259b92a..29fb1e5c127 100644 --- a/src/platform/telemetry/reportError.ts +++ b/src/platform/telemetry/reportError.ts @@ -31,6 +31,7 @@ export interface ReportErrorOptions { errorType: string /** Product surface responsible for acting on this failure. */ surface: Surface + fingerprint?: string[] tags?: Record context?: Record level?: 'warning' | 'error' @@ -171,13 +172,15 @@ function dispatchToSentry( surface: Surface, tags: Record, context: Record, - level: ReportErrorOptions['level'] + level: ReportErrorOptions['level'], + fingerprint: ReportErrorOptions['fingerprint'] ): boolean { try { captureException(error, { tags: { ...tags, error_type: errorType, surface }, extra: context, - level + level, + ...(fingerprint ? { fingerprint } : {}) }) return true } catch (reporterFailure) { @@ -249,7 +252,8 @@ function dispatch( surface, tags, context, - level + level, + options.fingerprint ) } if (datadogLive) { diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts index 4f1added2be..6b6d41f15cb 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts @@ -204,6 +204,29 @@ describe('attachDocOpMinter', () => { afterEach(() => minter.detach()) + it('reports mint divergence once when a node cannot be serialized', async () => { + const node = new TestSource() + graph.add(node) + vi.spyOn(node, 'serialize').mockImplementation(() => { + throw new Error('private widget content') + }) + + await afterFlush() + await afterFlush() + + expect(minted).toEqual([]) + expect(reportError).toHaveBeenCalledExactlyOnceWith( + new Error('failure_syncing_agent_edit_mint_divergence'), + expect.objectContaining({ + fingerprint: ['failure_syncing_agent_edit_mint_divergence'], + context: expect.objectContaining({ + code: 'missing_node_snapshot', + operations: ['add_node'] + }) + }) + ) + }) + describe('mints one op per graph command', () => { it.for([ { diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.ts b/src/workbench/extensions/agent/crdt/docOpMinter.ts index 8ea2d63f0ac..0ea96679f38 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.ts @@ -44,6 +44,7 @@ import { import type { GraphOperation } from './graphOperations' import type { DocPromotedWidgets } from './agentSubgraphDefinitions' +import { reportLostEdit } from './reportLostEdit' export interface DocOpMinterDeps { /** Slice 00's product gate. */ @@ -290,10 +291,9 @@ function routedWidgetOperation( } const subgraphNodePath = findSubgraphNodePathById(graph, owningGraphId) if (subgraphNodePath === null || subgraphNodePath.length === 0) { - console.error( - '[agent-crdt] set_widget with an unresolvable owner not minted; the bound doc diverges from the local graph', - nodeKey(owningGraphId, event.nodeId) + `:${event.name}` - ) + reportLostEdit('mint_divergence', [operation], { + code: 'unresolvable_widget_owner' + }) return null } const [head, ...rest] = subgraphNodePath @@ -379,10 +379,9 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { if (node.graph !== graph) return null const snapshot = wireNodeSnapshot(node) if (!snapshot) { - console.error( - '[agent-crdt] add_node mint dropped: no snapshot for node', - node.id - ) + reportLostEdit('mint_divergence', [{ op: 'add_node' }], { + code: 'missing_node_snapshot' + }) return null } return { diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index 57f4268a21f..4dc7826d8a0 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -1,8 +1,17 @@ import type { Op } from '@comfyorg/comfy-multi-player' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + afterEach, + beforeEach, + describe, + expect, + it, + onTestFinished, + vi +} from 'vitest' import type { Mock } from 'vitest' import { reportError } from '@/platform/telemetry/reportError' +import { api } from '@/scripts/api' import type { DocOp } from './docFrameClient' import type { GraphOperation } from './graphOperations' @@ -30,6 +39,12 @@ function summarizeSettlement(outcome: BatchOutcome): SettlementSummary { } } +function reportsOfType(errorType: string) { + return vi + .mocked(reportError) + .mock.calls.filter(([, options]) => options.errorType === errorType) +} + const detachListenerFailureCases = [ [ 'in-flight', @@ -188,6 +203,46 @@ describe('createOpSender', () => { sender.detach() }) + it.for([ + ['undeliverable', false, WORKFLOW, 3_000], + ['undeliverable', true, null, 0], + ['unconfirmed', true, WORKFLOW, 0], + ['unacknowledged', true, WORKFLOW, 20_000] + ] as const)( + 'reports %s exactly once with transport=%s, workflow=%s, elapsed=%s', + ([state, available, workflow, elapsed]) => { + transportUp = available + boundWorkflow = workflow + const previousClientId = api.clientId + api.clientId = 'api-tab' + onTestFinished(() => { + api.clientId = previousClientId + }) + + sender.enqueue([addNode(7), addNode(12)]) + vi.advanceTimersByTime(elapsed) + sender.detach() + sender.abortAll() + vi.advanceTimersByTime(60_000) + + expect(settled).toHaveLength(1) + expect(settled[0].state).toBe(state) + expect(reportError).toHaveBeenCalledExactlyOnceWith(expect.any(Error), { + surface: 'agent', + errorType: `failure_syncing_agent_edit_${state}`, + fingerprint: [`failure_syncing_agent_edit_${state}`], + tags: { tab: 'api-tab', outcome: state }, + context: { + code: state, + workflowId: undefined, + opId: settled[0].ops[0].op_id, + opIds: settled[0].ops.map((op) => op.op_id), + operations: ['add_node', 'add_node'] + } + }) + } + ) + it('mints once and sends a doc_ops batch with the wire envelope', () => { sender.enqueue([addNode(1), addNode(2)]) @@ -999,7 +1054,9 @@ describe('createOpSender', () => { localSender.detach() expect(localSettled.map(summarizeSettlement)).toEqual(expectedSurvivors) - expect(reportError).toHaveBeenCalledTimes(1) + expect( + reportsOfType('failure_settling_agent_op_sender_detach') + ).toHaveLength(1) expect(reportError).toHaveBeenCalledWith( expect.any(Error), expect.objectContaining({ @@ -1029,7 +1086,9 @@ describe('createOpSender', () => { localSender.detach() - expect(reportError).toHaveBeenCalledTimes(1) + expect( + reportsOfType('failure_settling_agent_op_sender_detach') + ).toHaveLength(1) expect(localUnsubscribe).toHaveBeenCalledOnce() }) @@ -1050,7 +1109,9 @@ describe('createOpSender', () => { localSender.enqueue([addNode(1)]) localSender.enqueue([addNode(2)]) - expect(reportError).toHaveBeenCalledTimes(1) + expect( + reportsOfType('failure_settling_agent_op_sender_detach') + ).toHaveLength(1) expect(reportError).toHaveBeenCalledWith( expect.any(Error), expect.objectContaining({ @@ -1066,12 +1127,9 @@ describe('createOpSender', () => { { state: 'undeliverable', nodeIds: [1] } ]) expect(sender.pending()).toBe(0) - expect(reportError).toHaveBeenCalledExactlyOnceWith( - expect.any(TypeError), - expect.objectContaining({ - errorType: 'failure_serializing_agent_op_sender' - }) - ) + expect(reportsOfType('failure_serializing_agent_op_sender')).toEqual([ + [expect.any(TypeError), expect.objectContaining({ surface: 'agent' })] + ]) sender.detach() @@ -1143,7 +1201,7 @@ describe('createOpSender', () => { enqueueCircular(4) enqueueCircular(5) - expect(reportError).toHaveBeenCalledTimes(3) + expect(reportsOfType('failure_serializing_agent_op_sender')).toHaveLength(3) }) it('keeps serialization telemetry bounded across successful sends and re-arms by time', () => { @@ -1153,17 +1211,17 @@ describe('createOpSender', () => { enqueueCircular(1) enqueueCircular(2) enqueueCircular(3) - expect(reportError).toHaveBeenCalledTimes(3) + expect(reportsOfType('failure_serializing_agent_op_sender')).toHaveLength(3) sender.enqueue([addNode(4)]) ackInFlight() enqueueCircular(5) - expect(reportError).toHaveBeenCalledTimes(3) + expect(reportsOfType('failure_serializing_agent_op_sender')).toHaveLength(3) vi.advanceTimersByTime(60_000) enqueueCircular(6) - expect(reportError).toHaveBeenCalledTimes(4) + expect(reportsOfType('failure_serializing_agent_op_sender')).toHaveLength(4) }) it('rejects an unserializable non-batchable op before transport', () => { diff --git a/src/workbench/extensions/agent/crdt/opSender.ts b/src/workbench/extensions/agent/crdt/opSender.ts index ed2a4fe7621..328d7a871bf 100644 --- a/src/workbench/extensions/agent/crdt/opSender.ts +++ b/src/workbench/extensions/agent/crdt/opSender.ts @@ -28,6 +28,7 @@ import { WIRE_MAX_OPS_PER_BATCH } from './opEnvelope' import type { SizedOp } from './opEnvelope' +import { reportLostEdit } from './reportLostEdit' const SEND_RETRY_LIMIT = 5 const SEND_RETRY_INTERVAL_MS = 500 @@ -271,6 +272,8 @@ export function createOpSender(deps: OpSenderDeps): OpSender { ): (outcome: BatchOutcome) => void { let reportedFailure = false return (outcome) => { + if (outcome.state !== 'acknowledged') + reportLostEdit(outcome.state, outcome.ops, { code: outcome.state }) try { deps.onBatchSettled(outcome) } catch (cause) { diff --git a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts index 4d8082ac9c1..3539183ac19 100644 --- a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts +++ b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts @@ -1,15 +1,15 @@ -import { throttle } from 'es-toolkit' -import type { ThrottledFunction } from 'es-toolkit' import type { Op } from '@comfyorg/comfy-multi-player' import { useToast } from '@/components/ui/toast/toastStore' import { i18n } from '@/i18n' -import { reportError } from '@/platform/telemetry/reportError' +import { createToastId } from '@/types/toastId' import type { DocOpFailure } from './docFrameClient' import type { OpsResultView } from './opSender' +import { reportLostEdit } from './reportLostEdit' const NOTICE_LIFE_MS = 10_000 +const NOTICE_ID = createToastId() function rejectedOp(ops: readonly Op[], failed: DocOpFailure): Op | undefined { return ops.find((op) => op.op_id === failed.op_id) ?? ops[failed.index] @@ -56,45 +56,14 @@ function noticeKey(widgetWrite: boolean, partial: boolean): string { } export function createRejectedOpNotifier() { - const throttledToasts = new Map void>>() const reportedCodes = new Set() - function reportOnce( - code: string, - widgetWrite: boolean, - result: OpsResultView, - failed: DocOpFailure | undefined - ): void { - if (reportedCodes.has(code)) return - reportedCodes.add(code) - reportError( - new Error(`the agent doc host rejected a human edit (${code})`), - { - surface: 'agent', - errorType: widgetWrite - ? 'error_applying_agent_widget_edit' - : 'error_applying_agent_graph_edit', - context: { workflowId: result.workflowId, opId: failed?.op_id, code }, - level: 'warning' - } - ) - } - - function throttledToast(detail: string): ThrottledFunction<() => void> { - const existing = throttledToasts.get(detail) - if (existing) return existing - const created = throttle( - () => { - useToast().error(i18n.global.t('g.error'), { - description: detail, - duration: NOTICE_LIFE_MS - }) - }, - NOTICE_LIFE_MS, - { edges: ['leading'] } - ) - throttledToasts.set(detail, created) - return created + function showToast(key: string): void { + useToast().error(i18n.global.t('g.error'), { + id: NOTICE_ID, + description: i18n.global.t(key), + duration: NOTICE_LIFE_MS + }) } return { @@ -103,16 +72,27 @@ export function createRejectedOpNotifier() { const { failed } = result const widgetWrite = isRejectedWidgetWrite(ops, failed) const code = failed?.code ?? result.code ?? 'unspecified' - reportOnce(code, widgetWrite, result, failed) + if (!reportedCodes.has(code)) { + reportedCodes.add(code) + reportLostEdit( + 'mint_divergence', + failed + ? [rejectedOp(ops, failed)].filter((op) => op !== undefined) + : ops, + { + code, + workflowId: result.workflowId, + errorType: widgetWrite + ? 'error_applying_agent_widget_edit' + : 'error_applying_agent_graph_edit' + } + ) + } const partial = result.applied.length > 0 && failed !== undefined - throttledToast(i18n.global.t(noticeKey(widgetWrite, partial)))() + showToast(noticeKey(widgetWrite, partial)) }, notifyWidgetWriteRefused() { - throttledToast(i18n.global.t('agent.editRejected.widgetWrite'))() - }, - cancel() { - for (const throttled of throttledToasts.values()) throttled.cancel() - throttledToasts.clear() + showToast('agent.editRejected.widgetWrite') } } } diff --git a/src/workbench/extensions/agent/crdt/reportLostEdit.ts b/src/workbench/extensions/agent/crdt/reportLostEdit.ts new file mode 100644 index 00000000000..84e887b75b9 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/reportLostEdit.ts @@ -0,0 +1,39 @@ +import type { Op } from '@comfyorg/comfy-multi-player' + +import { reportError } from '@/platform/telemetry/reportError' +import { api } from '@/scripts/api' + +import { recordDevEvent } from './devPanelLog' + +type LostEditKind = + | 'undeliverable' + | 'unconfirmed' + | 'unacknowledged' + | 'mint_divergence' + +export function reportLostEdit( + kind: LostEditKind, + ops: readonly (Pick & Partial>)[], + cause: { code: string; errorType?: string; workflowId?: string } +): void { + const errorType = cause.errorType ?? `failure_syncing_agent_edit_${kind}` + const context = { + code: cause.code, + workflowId: cause.workflowId, + opId: ops[0]?.op_id, + opIds: ops.map((op) => op.op_id), + operations: ops.map((op) => op.op) + } + reportError(new Error(errorType), { + surface: 'agent', + errorType, + fingerprint: [errorType], + tags: { tab: api.clientId, outcome: kind }, + context + }) + recordDevEvent( + 'human_ops_settled', + { state: kind, ...context }, + { level: 'warn' } + ) +} diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts index 1fb3473b71f..40fa8b1a0c9 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts @@ -1,8 +1,9 @@ import { fromPartial } from '@total-typescript/shoehorn' -import { render } from '@testing-library/vue' +import { render, screen } from '@testing-library/vue' import { describe, expect, it, onTestFinished, vi } from 'vitest' -import { defineComponent, ref } from 'vue' +import { defineComponent, nextTick, ref } from 'vue' +import Toaster from '@/components/ui/toast/Toaster.vue' import { useToast } from '@/components/ui/toast/toastStore' import { i18n } from '@/i18n' import { reportError } from '@/platform/telemetry/reportError' @@ -326,27 +327,22 @@ describe('a human edit the doc host rejects', () => { ]) }) - // A permanent seen-message set would pass the suppression test above and - // then silence every later rejection for the follower's lifetime. - it('speaks again once the throttle window has passed', async () => { - vi.useFakeTimers({ shouldAdvanceTime: true }) - onTestFinished(() => { - vi.useRealTimers() - }) + it('shows a later rejection after the user dismisses the notice', async () => { const { submit } = mountFollower() answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) - await vi.advanceTimersByTimeAsync(10_001) + useToast().dismiss(useToast().toasts[0].id) + expect(toastDescriptions()).toEqual([]) answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) expect(toastDescriptions()).toEqual([ - expect.stringContaining(WIDGET_REJECTION_TEXT), expect.stringContaining(WIDGET_REJECTION_TEXT) ]) }) - it('still speaks for a rejection of another kind inside that window', async () => { + it('replaces the notice with the latest rejection instead of stacking it', async () => { const { submit } = mountFollower() + render(Toaster, { global: { plugins: [i18n] } }) answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) answerWithOpsResult( @@ -354,9 +350,13 @@ describe('a human edit the doc host rejects', () => { ) expect(toastDescriptions()).toEqual([ - expect.stringContaining(WIDGET_REJECTION_TEXT), expect.stringContaining(GENERIC_REJECTION_TEXT) ]) + await nextTick() + expect(screen.getAllByTestId('toast')).toHaveLength(1) + const announcement = screen.getByRole('alert', { name: 'Notification' }) + expect(announcement).toHaveTextContent(GENERIC_REJECTION_TEXT) + expect(announcement).not.toHaveTextContent(WIDGET_REJECTION_TEXT) }) it('stays silent when the host applies the edit', async () => { diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index d7c0a8aa34b..be39983b934 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -1930,10 +1930,11 @@ describe('useAgentCrdtFollower', () => { removed_count: 0, apply_failure_count: 0 }) - expect(telemetryState.reportError).toHaveBeenCalledWith( + expect(telemetryState.reportError).toHaveBeenCalledExactlyOnceWith( expect.any(Error), expect.objectContaining({ - errorType: 'agent_crdt_human_ops_rejected', + errorType: 'error_applying_agent_graph_edit', + fingerprint: ['error_applying_agent_graph_edit'], context: expect.objectContaining({ opId: ops[1].op_id, code: 'unknown_node' diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index de3ab35ce04..5f443c0b810 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -18,7 +18,6 @@ import type { AgentGraphProjectionStage } from '@/platform/telemetry/types' import { api } from '@/scripts/api' import { parseNodeId } from '@/types/nodeId' import type { NodeId } from '@/types/nodeId' -import { createUuidv4 } from '@/utils/uuid' import { useAgentPanelStore } from '@/workbench/extensions/agent/stores/agent/agentPanelStore' import { @@ -289,31 +288,6 @@ export type AgentCrdtApplierDeps = Omit< 'getGraph' | 'holdsLocalWrite' > -function reportRejectedHumanOps( - workflowId: string | null, - ops: readonly Op[], - result: OpsResultView -): void { - const settled = new Set([...result.applied, ...result.skipped]) - const rejected = ops.filter((op) => !settled.has(op.op_id)) - const { failed } = result - reportError( - new Error( - `The doc host rejected ${rejected.length} local edit(s): ${failed?.message ?? 'no diagnostics'}` - ), - { - surface: 'agent', - errorType: 'agent_crdt_human_ops_rejected', - context: { - workflowId, - opId: failed?.op_id ?? rejected[0]?.op_id, - code: failed?.code, - rejectedOps: rejected.map((op) => op.op) - } - } - ) -} - export function useAgentCrdtFollower( workflowId: Ref, userId: () => string | null = () => null, @@ -428,8 +402,8 @@ function startAgentCrdtFollower( connected.value = false } ) - const tabId = createUuidv4() - const ownActor = (): string => `human:${userId() ?? 'anonymous'}:${tabId}` + const ownActor = (): string => + `human:${userId() ?? 'anonymous'}:${api.clientId ?? ''}` // Doc node ids whose human delete the host has applied but whose effect // frame has not yet removed them from the doc. Kept pending for the // reconcile so the result-to-effect window cannot resurrect them. @@ -487,8 +461,6 @@ function startAgentCrdtFollower( outcome: Extract ) => { const workflowId = outcome.result.workflowId ?? bridge.subscribedWorkflowId - if (outcome.result.failed) - reportRejectedHumanOps(workflowId, outcome.ops, outcome.result) if (workflowId === null) return const applied = new Set(outcome.result.applied) @@ -508,8 +480,10 @@ function startAgentCrdtFollower( } const settleHumanOps = (outcome: BatchOutcome) => { - if (outcome.state === 'acknowledged') trackAcknowledgedDeletes(outcome) - recordDevEvent('human_ops_settled', outcome) + if (outcome.state === 'acknowledged') { + trackAcknowledgedDeletes(outcome) + if (outcome.result.ok) recordDevEvent('human_ops_settled', outcome) + } projection.settleLocalWrites(outcome.ops) if (outcome.state === 'acknowledged' && !outcome.result.ok) revertRejectedOps(outcome) @@ -530,7 +504,9 @@ function startAgentCrdtFollower( // Send REALITY, not this composable's intent: the sender re-reads it before // every send and resend, so ops never reach a doc we are not subscribed to. workflowId: () => bridge.subscribedWorkflowId, - tab: tabId, + get tab() { + return api.clientId ?? '' + }, actor: ownActor, baseVersion: () => bridge.lastSequence, onBatchSettled: settleHumanOps @@ -1024,7 +1000,6 @@ function startAgentCrdtFollower( () => bridge.removeEventListener('doc_subscribe_sent', onSubscribeSent), () => bridge.removeEventListener('doc_reseed_result', onReseedResult), () => sender.detach(), - () => rejectedOpNotifier.cancel(), () => pendingRejected.clear(), () => coalescer.detach(), () => { @@ -1049,7 +1024,7 @@ function startAgentCrdtFollower( const debugSnapshot = (): CrdtDebugSnapshot => readCrdtSnapshot(bridge.follower.doc, { status: status.value, - tabId, + tabId: api.clientId ?? null, lastSeq: bridge.lastSequence, schemaError: bridge.lastSchemaError?.message ?? null }) From 8b6d4841a7cfd3ddf6d0761bbdaeb865f43d4672 Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Tue, 13 Oct 2026 02:27:29 +0000 Subject: [PATCH 2/8] fix: preserve CRDT settlement diagnostics Amp-Thread-ID: https://ampcode.com/threads/T-01a1323c-7d66-709b-9454-cfc5463f9c65 --- src/workbench/extensions/agent/crdt/devPanelLog.ts | 1 + src/workbench/extensions/agent/crdt/reportLostEdit.ts | 10 +++++----- .../extensions/agent/crdt/useAgentCrdtFollower.ts | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.ts b/src/workbench/extensions/agent/crdt/devPanelLog.ts index 115541be084..b2e171c86d6 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.ts @@ -30,6 +30,7 @@ export const DEV_EVENT_KINDS = [ 'doc_ops_result', 'doc_ops_result_dropped', 'human_ops_settled', + 'mint_divergence', 'doc_reset', 'doc_nodes_changed', 'schema_error', diff --git a/src/workbench/extensions/agent/crdt/reportLostEdit.ts b/src/workbench/extensions/agent/crdt/reportLostEdit.ts index 84e887b75b9..55a6354b043 100644 --- a/src/workbench/extensions/agent/crdt/reportLostEdit.ts +++ b/src/workbench/extensions/agent/crdt/reportLostEdit.ts @@ -31,9 +31,9 @@ export function reportLostEdit( tags: { tab: api.clientId, outcome: kind }, context }) - recordDevEvent( - 'human_ops_settled', - { state: kind, ...context }, - { level: 'warn' } - ) + if (kind === 'mint_divergence') { + recordDevEvent('mint_divergence', context, { level: 'warn' }) + } else { + recordDevEvent('human_ops_settled', { state: kind, ops }) + } } diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 5f443c0b810..eb7e1ead2ad 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -482,7 +482,7 @@ function startAgentCrdtFollower( const settleHumanOps = (outcome: BatchOutcome) => { if (outcome.state === 'acknowledged') { trackAcknowledgedDeletes(outcome) - if (outcome.result.ok) recordDevEvent('human_ops_settled', outcome) + recordDevEvent('human_ops_settled', outcome) } projection.settleLocalWrites(outcome.ops) if (outcome.state === 'acknowledged' && !outcome.result.ok) From 2bfbf60da016a3c6a80443ed3b0c40ae175aff2c Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Wed, 14 Oct 2026 01:32:01 +0000 Subject: [PATCH 3/8] fix: report every host-rejected edit batch --- .../extensions/agent/crdt/devPanelLog.ts | 1 + .../extensions/agent/crdt/rejectedOpNotice.ts | 31 ++++++++--------- .../extensions/agent/crdt/reportLostEdit.ts | 5 +-- ...ntCrdtFollower.rejectedWidgetWrite.test.ts | 33 ++++++++++++------- 4 files changed, 39 insertions(+), 31 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.ts b/src/workbench/extensions/agent/crdt/devPanelLog.ts index b2e171c86d6..500e4b7475b 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.ts @@ -31,6 +31,7 @@ export const DEV_EVENT_KINDS = [ 'doc_ops_result_dropped', 'human_ops_settled', 'mint_divergence', + 'host_rejected', 'doc_reset', 'doc_nodes_changed', 'schema_error', diff --git a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts index 3539183ac19..a7545daef02 100644 --- a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts +++ b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts @@ -56,8 +56,6 @@ function noticeKey(widgetWrite: boolean, partial: boolean): string { } export function createRejectedOpNotifier() { - const reportedCodes = new Set() - function showToast(key: string): void { useToast().error(i18n.global.t('g.error'), { id: NOTICE_ID, @@ -72,22 +70,19 @@ export function createRejectedOpNotifier() { const { failed } = result const widgetWrite = isRejectedWidgetWrite(ops, failed) const code = failed?.code ?? result.code ?? 'unspecified' - if (!reportedCodes.has(code)) { - reportedCodes.add(code) - reportLostEdit( - 'mint_divergence', - failed - ? [rejectedOp(ops, failed)].filter((op) => op !== undefined) - : ops, - { - code, - workflowId: result.workflowId, - errorType: widgetWrite - ? 'error_applying_agent_widget_edit' - : 'error_applying_agent_graph_edit' - } - ) - } + reportLostEdit( + 'host_rejected', + failed + ? [rejectedOp(ops, failed)].filter((op) => op !== undefined) + : ops, + { + code, + workflowId: result.workflowId, + errorType: widgetWrite + ? 'error_applying_agent_widget_edit' + : 'error_applying_agent_graph_edit' + } + ) const partial = result.applied.length > 0 && failed !== undefined showToast(noticeKey(widgetWrite, partial)) }, diff --git a/src/workbench/extensions/agent/crdt/reportLostEdit.ts b/src/workbench/extensions/agent/crdt/reportLostEdit.ts index 55a6354b043..31106daa94a 100644 --- a/src/workbench/extensions/agent/crdt/reportLostEdit.ts +++ b/src/workbench/extensions/agent/crdt/reportLostEdit.ts @@ -10,6 +10,7 @@ type LostEditKind = | 'unconfirmed' | 'unacknowledged' | 'mint_divergence' + | 'host_rejected' export function reportLostEdit( kind: LostEditKind, @@ -31,8 +32,8 @@ export function reportLostEdit( tags: { tab: api.clientId, outcome: kind }, context }) - if (kind === 'mint_divergence') { - recordDevEvent('mint_divergence', context, { level: 'warn' }) + if (kind === 'mint_divergence' || kind === 'host_rejected') { + recordDevEvent(kind, context, { level: 'warn' }) } else { recordDevEvent('human_ops_settled', { state: kind, ops }) } diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts index 40fa8b1a0c9..d6cc96ec497 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts @@ -199,6 +199,13 @@ describe('a human edit the doc host rejects', () => { expect(toastDescriptions()).toEqual([ expect.stringContaining(WIDGET_REJECTION_TEXT) ]) + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'error_applying_agent_widget_edit', + tags: expect.objectContaining({ outcome: 'host_rejected' }) + }) + ) } ) @@ -374,10 +381,6 @@ describe('a human edit the doc host rejects', () => { expect(useToast().toasts).toEqual([]) }) - // The host refuses a whole batch with a top-level `code` and no `failed` - // entry. Read only from `failed`, every one of these collapses to - // 'unspecified' — and because the telemetry dedupe keys on that, the first - // would suppress the report for every later kind. it('classifies a batch-level refusal that carries no failed entry', async () => { const { submit } = mountFollower() await submit(WIDGET_EDIT) @@ -388,24 +391,32 @@ describe('a human edit the doc host rejects', () => { expect.stringContaining(GENERIC_REJECTION_TEXT) ]) expect(reportedCodes()).toEqual(['catalog_mismatch']) + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'error_applying_agent_graph_edit', + tags: expect.objectContaining({ outcome: 'host_rejected' }) + }) + ) }) - // Asserted on telemetry rather than toasts: every generic notice renders the - // same string, so they share one throttle key and the second is swallowed on - // screen even though it is a different failure. - it('reports each distinct batch-level code once', async () => { + it('reports each rejected batch once even when rejection codes repeat', async () => { const { submit } = mountFollower() - // One pending batch per answer: the notifier runs when a batch SETTLES, so - // a result with nothing outstanding settles nothing and reports nothing. await submit(WIDGET_EDIT) answerWithOpsResult(batchRefusal('overloaded')) + answerWithOpsResult(batchRefusal('overloaded')) await submit(WIDGET_EDIT) answerWithOpsResult(batchRefusal('overloaded')) + answerWithOpsResult(batchRefusal('overloaded')) await submit(WIDGET_EDIT) answerWithOpsResult(batchRefusal('catalog_mismatch')) - expect(reportedCodes()).toEqual(['overloaded', 'catalog_mismatch']) + expect(reportedCodes()).toEqual([ + 'overloaded', + 'overloaded', + 'catalog_mismatch' + ]) }) // Schema §4 aborts the remainder of a batch, so the prefix is still applied. From 10f2a9517d3ad1f89d6d1f7c971a53a5ef067f68 Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Thu, 15 Oct 2026 12:50:39 +0000 Subject: [PATCH 4/8] fix: preserve lost-edit diagnostic causes and scope Amp-Thread-ID: https://ampcode.com/threads/T-01a1323c-7d66-709b-9454-cfc5463f9c65 --- ...nvas-authority-and-draft-reconciliation.md | 9 +- .../extensions/agent/crdt/devPanelLog.test.ts | 2 + .../extensions/agent/crdt/devPanelLog.ts | 1 + .../extensions/agent/crdt/docOpMinter.test.ts | 43 ++++++- .../extensions/agent/crdt/docOpMinter.ts | 32 +++-- .../extensions/agent/crdt/opSender.test.ts | 42 +++++-- .../extensions/agent/crdt/opSender.ts | 119 ++++++++++-------- .../extensions/agent/crdt/rejectedOpNotice.ts | 5 +- .../extensions/agent/crdt/reportLostEdit.ts | 13 +- ...ntCrdtFollower.rejectedWidgetWrite.test.ts | 33 ++--- 10 files changed, 204 insertions(+), 95 deletions(-) diff --git a/docs/adr/CRDT-AUTHORITY-0035-human-canvas-authority-and-draft-reconciliation.md b/docs/adr/CRDT-AUTHORITY-0035-human-canvas-authority-and-draft-reconciliation.md index 30245c12ed6..8fca31f58c4 100644 --- a/docs/adr/CRDT-AUTHORITY-0035-human-canvas-authority-and-draft-reconciliation.md +++ b/docs/adr/CRDT-AUTHORITY-0035-human-canvas-authority-and-draft-reconciliation.md @@ -150,9 +150,12 @@ Alternatives considered and rejected: operations of that batch unapplied, where separate batches were independent. Chunking at the wire cap bounds a batch, and the ledger already models `unprocessed`. -- Human operations settled `undeliverable` or `unconfirmed` (at unbind, at - `doc_reset`, on a dead transport) are still reported only to the dev panel; - surfacing them to the user is a follow-up. +- Human operations settled `undeliverable`, `unconfirmed`, or + `unacknowledged` produce one `reportError` call and a dev-panel entry per + batch. Reports include the mint-time workflow id and termination cause. + `doc_reset` identifies batches superseded by the draft under decision 4, + so lost-edit queries can exclude them. User-facing notices for sender + termination remain a follow-up. - Two browser tabs on the same document, both agent-bound, are out of scope: which tab's draft wins a divergence is decided by the server's sequence check, not by anything the frontend promises. diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.test.ts b/src/workbench/extensions/agent/crdt/devPanelLog.test.ts index 0229f057014..9a8a8e049ac 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.test.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.test.ts @@ -203,6 +203,7 @@ describe('devPanelLog', () => { applied: [], skipped: ['op-1'], code: 'op_failed', + operations: ['set_widget', 'delete_node', 'private user prose'], message: 'unknown node 7', failed: { op_id: 'op-1', code: 'unknown_node', message: 'unknown node 7' } }) @@ -213,6 +214,7 @@ describe('devPanelLog', () => { applied: [], skipped: ['op-1'], code: 'op_failed', + operations: ['set_widget', 'delete_node', 'String(18)'], message: 'String(14)', failed: { op_id: 'op-1', code: 'unknown_node', message: 'String(14)' } }) diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.ts b/src/workbench/extensions/agent/crdt/devPanelLog.ts index 500e4b7475b..419c4e82cfd 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.ts @@ -106,6 +106,7 @@ const STRUCTURAL_STRING_KEYS = new Set([ 'op', 'op_id', 'op_ids', + 'operations', 'outcome', 'phase', 'prompt_id', diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts index 6b6d41f15cb..4e832151617 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts @@ -216,11 +216,13 @@ describe('attachDocOpMinter', () => { expect(minted).toEqual([]) expect(reportError).toHaveBeenCalledExactlyOnceWith( - new Error('failure_syncing_agent_edit_mint_divergence'), + new Error('agent_crdt_node_snapshot_missing'), expect.objectContaining({ - fingerprint: ['failure_syncing_agent_edit_mint_divergence'], + fingerprint: ['agent_crdt_node_snapshot_missing'], context: expect.objectContaining({ code: 'missing_node_snapshot', + graphId: graph.id, + nodeId: node.id, operations: ['add_node'] }) }) @@ -1617,6 +1619,43 @@ describe('attachDocOpMinter', () => { expect(minted).toEqual([]) }) + it('reports an unresolved widget owner once across flushes without suppressing another owner', async () => { + const staleGraph = new LGraph() + const otherGraph = new LGraph() + const { source: stale } = seedGraph(staleGraph) + const { source: other } = seedGraph(otherGraph) + const { source: active } = seedGraph(graph) + + stale.widgets![0].value = 21 + active.widgets![0].value = 31 + await afterFlush() + stale.widgets![0].value = 22 + other.widgets![0].value = 23 + await afterFlush() + + expect(minted).toEqual([ + expect.objectContaining({ node_id: active.id, value: 31 }) + ]) + expect( + vi.mocked(reportError).mock.calls.map(([, options]) => options) + ).toEqual([ + expect.objectContaining({ + errorType: 'agent_crdt_widget_owner_unresolved', + context: expect.objectContaining({ + graphId: staleGraph.id, + nodeId: stale.id + }) + }), + expect.objectContaining({ + errorType: 'agent_crdt_widget_owner_unresolved', + context: expect.objectContaining({ + graphId: otherGraph.id, + nodeId: other.id + }) + }) + ]) + }) + it('mints a set_widget that names a subgraph owner with the subgraph-node path', async () => { const subgraph = createTestSubgraph({ rootGraph: graph }) const host = createTestSubgraphNode(subgraph) diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.ts b/src/workbench/extensions/agent/crdt/docOpMinter.ts index 0ea96679f38..2537396bf27 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.ts @@ -275,7 +275,7 @@ function routedWidgetOperation( event: IntentOf<'set_widget'>, node: LGraphNode | null, deps: Pick -): GraphOperation | PromotedWriteRefusal | null { +): GraphOperation | PromotedWriteRefusal | { unresolvedGraphId: string } { const operation = { op: 'set_widget', node_id: event.nodeId, @@ -290,12 +290,8 @@ function routedWidgetOperation( return 'reason' in promoted ? promoted : { ...operation, promoted } } const subgraphNodePath = findSubgraphNodePathById(graph, owningGraphId) - if (subgraphNodePath === null || subgraphNodePath.length === 0) { - reportLostEdit('mint_divergence', [operation], { - code: 'unresolvable_widget_owner' - }) - return null - } + if (subgraphNodePath === null || subgraphNodePath.length === 0) + return { unresolvedGraphId: owningGraphId } const [head, ...rest] = subgraphNodePath return { ...operation, @@ -379,8 +375,14 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { if (node.graph !== graph) return null const snapshot = wireNodeSnapshot(node) if (!snapshot) { + const key = `snapshot:${nodeKey(graph.id, node.id)}` + if (reported.has(key)) return null + reported.add(key) reportLostEdit('mint_divergence', [{ op: 'add_node' }], { - code: 'missing_node_snapshot' + code: 'missing_node_snapshot', + errorType: 'agent_crdt_node_snapshot_missing', + graphId: graph.id, + nodeId: node.id }) return null } @@ -462,7 +464,19 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { if (!isValueWidgetWrite(owner, event)) return const rootGraphId = deps.boundRootGraphId() ?? graph.id const routed = routedWidgetOperation(graph, rootGraphId, event, owner, deps) - if (routed === null) return + if ('unresolvedGraphId' in routed) { + const graphId = routed.unresolvedGraphId + const key = `owner:${nodeKey(graphId, event.nodeId)}` + if (reportedRefusals.has(key)) return + reportedRefusals.add(key) + reportLostEdit('mint_divergence', [{ op: 'set_widget' }], { + code: 'unresolvable_widget_owner', + errorType: 'agent_crdt_widget_owner_unresolved', + graphId, + nodeId: event.nodeId + }) + return + } if ('reason' in routed) { reportRefusal(rootGraphId, event, routed) deps.onWidgetWriteRefused() diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index 4dc7826d8a0..cdfbd0a205f 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -204,13 +204,13 @@ describe('createOpSender', () => { }) it.for([ - ['undeliverable', false, WORKFLOW, 3_000], - ['undeliverable', true, null, 0], - ['unconfirmed', true, WORKFLOW, 0], - ['unacknowledged', true, WORKFLOW, 20_000] + ['undeliverable', false, WORKFLOW, 3_000, 'transport_exhausted'], + ['undeliverable', true, null, 0, 'unbound'], + ['unconfirmed', true, WORKFLOW, 0, 'detached'], + ['unacknowledged', true, WORKFLOW, 20_000, 'result_timeout'] ] as const)( 'reports %s exactly once with transport=%s, workflow=%s, elapsed=%s', - ([state, available, workflow, elapsed]) => { + ([state, available, workflow, elapsed, code]) => { transportUp = available boundWorkflow = workflow const previousClientId = api.clientId @@ -231,10 +231,11 @@ describe('createOpSender', () => { surface: 'agent', errorType: `failure_syncing_agent_edit_${state}`, fingerprint: [`failure_syncing_agent_edit_${state}`], + level: 'error', tags: { tab: 'api-tab', outcome: state }, context: { - code: state, - workflowId: undefined, + code, + workflowId: workflow ?? undefined, opId: settled[0].ops[0].op_id, opIds: settled[0].ops.map((op) => op.op_id), operations: ['add_node', 'add_node'] @@ -476,6 +477,15 @@ describe('createOpSender', () => { expect(sent).toHaveLength(2) expect(sent[1].workflowId).toBe('wf-2') expect(settled[0].state).toBe('unconfirmed') + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + expect.objectContaining({ + context: expect.objectContaining({ + code: 'unbound', + workflowId: WORKFLOW + }) + }) + ) }) it('drains 20,000 queued batches after unbinding without overflowing the stack', () => { @@ -1130,6 +1140,17 @@ describe('createOpSender', () => { expect(reportsOfType('failure_serializing_agent_op_sender')).toEqual([ [expect.any(TypeError), expect.objectContaining({ surface: 'agent' })] ]) + expect(reportsOfType('failure_syncing_agent_edit_undeliverable')).toEqual([ + [ + expect.any(Error), + expect.objectContaining({ + context: expect.objectContaining({ + code: 'serialization_failed', + workflowId: WORKFLOW + }) + }) + ] + ]) sender.detach() @@ -1492,6 +1513,13 @@ describe('createOpSender', () => { ) ).toEqual([[1], [2], [3]]) expect(sender.pending()).toBe(0) + expect( + vi.mocked(reportError).mock.calls.map(([, options]) => options.context) + ).toEqual([ + expect.objectContaining({ code: 'doc_reset', workflowId: WORKFLOW }), + expect.objectContaining({ code: 'doc_reset', workflowId: WORKFLOW }), + expect.objectContaining({ code: 'doc_reset', workflowId: WORKFLOW }) + ]) sender.enqueue([addNode(4)]) expect(sent).toHaveLength(2) diff --git a/src/workbench/extensions/agent/crdt/opSender.ts b/src/workbench/extensions/agent/crdt/opSender.ts index 328d7a871bf..8c9347ec0de 100644 --- a/src/workbench/extensions/agent/crdt/opSender.ts +++ b/src/workbench/extensions/agent/crdt/opSender.ts @@ -268,12 +268,16 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } function guardedSettlementNotifier( - errorType: string - ): (outcome: BatchOutcome) => void { + errorType: string, + code: string + ): (outcome: BatchOutcome, workflowId: string | null) => void { let reportedFailure = false - return (outcome) => { + return (outcome, workflowId) => { if (outcome.state !== 'acknowledged') - reportLostEdit(outcome.state, outcome.ops, { code: outcome.state }) + reportLostEdit(outcome.state, outcome.ops, { + code, + workflowId: workflowId ?? undefined + }) try { deps.onBatchSettled(outcome) } catch (cause) { @@ -285,12 +289,17 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } const notifyDetachSettlement = guardedSettlementNotifier( - 'failure_settling_agent_op_sender_detach' + 'failure_settling_agent_op_sender_detach', + 'detached' ) - function settle(outcome: BatchOutcome): void { + function settle(outcome: BatchOutcome, code: string): void { + const workflowId = inFlight?.workflowId ?? null if (inFlight?.timer) clearTimeout(inFlight.timer) inFlight = null - guardedSettlementNotifier('failure_settling_agent_op_sender')(outcome) + guardedSettlementNotifier('failure_settling_agent_op_sender', code)( + outcome, + workflowId + ) pump() } @@ -300,23 +309,18 @@ export function createOpSender(deps: OpSenderDeps): OpSender { batch.parked = true return } - // A lost subscription is not a transport that recovers in 500 ms: settle - // now rather than spend the retry budget while later batches wait behind. if (deps.workflowId() !== batch.workflowId) { - settleUnbound(batch) + settleUnbound(batch, 'unbound') return } if (!trySend(batch)) { if (attempt < SEND_RETRY_LIMIT) { - // Tracked in the same slot as the result timer (they never overlap: - // the result timer is armed only after a successful send) so - // settle()/detach() clear a pending retry too. batch.timer = setTimeout( () => transmit(batch, attempt + 1), SEND_RETRY_INTERVAL_MS ) } else { - settleUnbound(batch) + settleUnbound(batch, 'transport_exhausted') } return } @@ -346,13 +350,16 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } } - function settleUnbound(batch: InFlight): void { + function settleUnbound(batch: InFlight, code: string): void { if (inFlight !== batch) return retire(batch, 0) - settle({ - state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable', - ops: batch.ops - }) + settle( + { + state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable', + ops: batch.ops + }, + code + ) } function armResultTimeout(batch: InFlight): void { @@ -361,18 +368,18 @@ export function createOpSender(deps: OpSenderDeps): OpSender { if (inFlight !== batch) return if (batch.resent) { retire(batch, 0) - settle({ state: 'unacknowledged', ops: batch.ops }) + settle({ state: 'unacknowledged', ops: batch.ops }, 'result_timeout') return } - // One silent-result resend of the SAME minted ops: idempotent at the - // applier through the op_id gate. batch.resent = true batch.reportedThrow = false transmit(batch, 0) }, RESULT_TIMEOUT_MS) } - function drainOutstanding(notify: (outcome: BatchOutcome) => void): void { + function drainOutstanding( + notify: (outcome: BatchOutcome, workflowId: string | null) => void + ): void { const unsealed = open open = null const queued = queue.slice(queueHead) @@ -384,17 +391,23 @@ export function createOpSender(deps: OpSenderDeps): OpSender { const batch = active if (batch.timer) clearTimeout(batch.timer) retire(batch, 0) - notify({ - state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable', - ops: batch.ops - }) + notify( + { + state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable', + ops: batch.ops + }, + batch.workflowId + ) } for (const batch of queued) { - notify({ state: 'undeliverable', ops: batch.ops }) + notify({ state: 'undeliverable', ops: batch.ops }, batch.workflowId) } if (unsealed) { for (const chunk of chunkWireOps(unsealed.ops)) { - notify({ state: 'undeliverable', ops: chunk.map((sized) => sized.op) }) + notify( + { state: 'undeliverable', ops: chunk.map((sized) => sized.op) }, + unsealed.workflowId + ) } } } @@ -435,13 +448,17 @@ export function createOpSender(deps: OpSenderDeps): OpSender { function settleUndeliverableInBoundedGroups( ops: Op[], - notify: (outcome: BatchOutcome) => void + notify: (outcome: BatchOutcome, workflowId: string | null) => void, + workflowId: string | null ): void { for (let index = 0; index < ops.length; index += WIRE_MAX_OPS_PER_BATCH) { - notify({ - state: 'undeliverable', - ops: ops.slice(index, index + WIRE_MAX_OPS_PER_BATCH) - }) + notify( + { + state: 'undeliverable', + ops: ops.slice(index, index + WIRE_MAX_OPS_PER_BATCH) + }, + workflowId + ) } } @@ -527,14 +544,19 @@ export function createOpSender(deps: OpSenderDeps): OpSender { cause }: Admission): void { if (detached) { - settleUndeliverableInBoundedGroups(minted, notifyDetachSettlement) + settleUndeliverableInBoundedGroups( + minted, + notifyDetachSettlement, + workflowId + ) return } const notifyFailure = guardedSettlementNotifier( - 'failure_settling_agent_op_sender' + 'failure_settling_agent_op_sender', + workflowId === null ? 'unbound' : 'serialization_failed' ) if (workflowId === null) { - settleUndeliverableInBoundedGroups(minted, notifyFailure) + settleUndeliverableInBoundedGroups(minted, notifyFailure, workflowId) return } appendToOpenGroup(workflowId, admitted) @@ -542,7 +564,7 @@ export function createOpSender(deps: OpSenderDeps): OpSender { if (mayReportSerializationFailure()) { reportDegraded(cause, 'failure_serializing_agent_op_sender') } - settleUndeliverableInBoundedGroups(rejected, notifyFailure) + settleUndeliverableInBoundedGroups(rejected, notifyFailure, workflowId) } function seal(): void { @@ -577,27 +599,19 @@ export function createOpSender(deps: OpSenderDeps): OpSender { if (identified.length > 0) { if (batch && identified.some((opId) => batch.opIds.has(opId))) { retire(batch, 1) - settle({ state: 'acknowledged', ops: batch.ops, result }) + settle({ state: 'acknowledged', ops: batch.ops, result }, 'host_result') } else if (identified.some((opId) => retiredOpIds.has(opId))) { - // A retired batch's own answer consumes the credit reserved for it; - // ops this sender never minted are nobody's answer here. drainStaleCredit() } return } - // Anonymous failure (empty lists, no failed op_id): a late result with - // no batch waiting, one addressed to another workflow, or one a stale - // credit could explain drains that credit so it cannot swallow a future - // batch's own result. Only then is it the in-flight batch's. if (batch === null || staleAnonymousBudget > 0) { drainStaleCredit() return } - // A parked batch or one waiting for a transport retry has not put a send - // on the wire, so an anonymous result cannot belong to it. if (batch.sends === 0) return retire(batch, 1) - settle({ state: 'acknowledged', ops: batch.ops, result }) + settle({ state: 'acknowledged', ops: batch.ops, result }, 'host_result') }) return { @@ -635,16 +649,17 @@ export function createOpSender(deps: OpSenderDeps): OpSender { }, abortIfUnbound() { if (inFlight && deps.workflowId() !== inFlight.workflowId) { - settleUnbound(inFlight) + settleUnbound(inFlight, 'unbound') } }, abortAll() { - // A doc_reset replaces only the bound document (the follower guards on - // isCurrentWorkflow), so only its cursor restarts. const resetWorkflowId = deps.workflowId() if (resetWorkflowId !== null) lastMintedVersions.delete(resetWorkflowId) drainOutstanding( - guardedSettlementNotifier('failure_settling_agent_op_sender_abort') + guardedSettlementNotifier( + 'failure_settling_agent_op_sender_abort', + 'doc_reset' + ) ) }, detach() { diff --git a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts index a7545daef02..145c8b72988 100644 --- a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts +++ b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts @@ -70,11 +70,10 @@ export function createRejectedOpNotifier() { const { failed } = result const widgetWrite = isRejectedWidgetWrite(ops, failed) const code = failed?.code ?? result.code ?? 'unspecified' + const accepted = new Set([...result.applied, ...result.skipped]) reportLostEdit( 'host_rejected', - failed - ? [rejectedOp(ops, failed)].filter((op) => op !== undefined) - : ops, + ops.filter((op) => !accepted.has(op.op_id)), { code, workflowId: result.workflowId, diff --git a/src/workbench/extensions/agent/crdt/reportLostEdit.ts b/src/workbench/extensions/agent/crdt/reportLostEdit.ts index 31106daa94a..97746fccbbf 100644 --- a/src/workbench/extensions/agent/crdt/reportLostEdit.ts +++ b/src/workbench/extensions/agent/crdt/reportLostEdit.ts @@ -15,11 +15,17 @@ type LostEditKind = export function reportLostEdit( kind: LostEditKind, ops: readonly (Pick & Partial>)[], - cause: { code: string; errorType?: string; workflowId?: string } + cause: { + code: string + errorType?: string + workflowId?: string + graphId?: string + nodeId?: string | number + } ): void { - const errorType = cause.errorType ?? `failure_syncing_agent_edit_${kind}` + const { errorType = `failure_syncing_agent_edit_${kind}`, ...detail } = cause const context = { - code: cause.code, + ...detail, workflowId: cause.workflowId, opId: ops[0]?.op_id, opIds: ops.map((op) => op.op_id), @@ -29,6 +35,7 @@ export function reportLostEdit( surface: 'agent', errorType, fingerprint: [errorType], + level: kind === 'host_rejected' ? 'warning' : 'error', tags: { tab: api.clientId, outcome: kind }, context }) diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts index d6cc96ec497..1e4c6f3bdb5 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts @@ -419,33 +419,28 @@ describe('a human edit the doc host rejects', () => { ]) }) - // Schema §4 aborts the remainder of a batch, so the prefix is still applied. - // Both partial branches are user-facing copy, so both are exercised: a wrong - // key on either would tell the user nothing was saved after a batch that - // saved a prefix. The rejected op's KIND picks the branch, so the two rows - // differ only in which op the host rejects. it.for([ { branch: 'widget', ops: [NODE_ADD, WIDGET_EDIT], code: 'opaque_widgets', - expected: WIDGET_REJECTION_TEXT + expected: WIDGET_REJECTION_TEXT, + errorType: 'error_applying_agent_widget_edit' }, { branch: 'generic', ops: [WIDGET_EDIT, NODE_ADD], code: 'uncatalogued_widget_write', - expected: GENERIC_REJECTION_TEXT + expected: GENERIC_REJECTION_TEXT, + errorType: 'error_applying_agent_graph_edit' } ])( 'does not claim nothing was saved when a batch applied a prefix ($branch)', - async ({ ops, code, expected }) => { + async ({ ops, code, expected, errorType }) => { const { submitBatch } = mountFollower() - const frames = await submitBatch(ops) - // EXACTLY one frame carrying BOTH ops: across two frames the host could - // never answer with one applied id and one failure index. + const frames = await submitBatch([NODE_DELETE, ...ops, NODE_DELETE]) expect(frames).toHaveLength(1) - const [[appliedId, rejectedId]] = frames + const [[skippedId, appliedId, rejectedId, tailId]] = frames expect(rejectedId).toBeDefined() answerWithOpsResult({ @@ -453,17 +448,23 @@ describe('a human edit the doc host rejects', () => { workflow_id: WORKFLOW_ID, ok: false, applied: [appliedId], - skipped: [], + skipped: [skippedId], failed: { - index: 1, + index: 2, op_id: rejectedId, code, message: 'node is absent from the pinned catalog' } }) - // Still carries the canonical substring the e2e spec filters on, so the - // partial variant cannot break that contract. + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + expect.objectContaining({ + errorType, + level: 'warning', + context: expect.objectContaining({ opIds: [rejectedId, tailId] }) + }) + ) expect(toastDescriptions()).toEqual([expect.stringContaining(expected)]) expect(toastDescriptions()[0]).toContain('some earlier edits were') } From e1a7b74ed0fb7c2b070a90caa3ff0a7f64a060cf Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Thu, 15 Oct 2026 14:07:36 +0000 Subject: [PATCH 5/8] fix: retain queryable lost-edit diagnostic metadata Amp-Thread-ID: https://ampcode.com/threads/T-01a1323c-7d66-709b-9454-cfc5463f9c65 --- src/workbench/extensions/agent/crdt/devPanelLog.test.ts | 2 ++ src/workbench/extensions/agent/crdt/devPanelLog.ts | 1 + src/workbench/extensions/agent/crdt/opSender.test.ts | 5 ++++- src/workbench/extensions/agent/crdt/reportLostEdit.ts | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.test.ts b/src/workbench/extensions/agent/crdt/devPanelLog.test.ts index 9a8a8e049ac..ccdb6ce3284 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.test.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.test.ts @@ -199,6 +199,7 @@ describe('devPanelLog', () => { it('keeps the structural diagnosis on failed frames', () => { recordDevEvent('doc_ops_result', { workflowId: 'wf-1', + graphId: '1ff39644-78b1-48bb-9509-53b671c672f1', ok: false, applied: [], skipped: ['op-1'], @@ -210,6 +211,7 @@ describe('devPanelLog', () => { expect(devEvents.value[0]?.detail).toEqual({ workflowId: 'wf-1', + graphId: '1ff39644-78b1-48bb-9509-53b671c672f1', ok: false, applied: [], skipped: ['op-1'], diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.ts b/src/workbench/extensions/agent/crdt/devPanelLog.ts index 419c4e82cfd..206bbf09c5d 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.ts @@ -98,6 +98,7 @@ const STRUCTURAL_STRING_KEYS = new Set([ 'doc_id', 'failed', 'frame_type', + 'graph_id', 'id', 'kind', 'level', diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index cdfbd0a205f..4afd379d16e 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -232,7 +232,7 @@ describe('createOpSender', () => { errorType: `failure_syncing_agent_edit_${state}`, fingerprint: [`failure_syncing_agent_edit_${state}`], level: 'error', - tags: { tab: 'api-tab', outcome: state }, + tags: { tab: 'api-tab', outcome: state, code }, context: { code, workflowId: workflow ?? undefined, @@ -1520,6 +1520,9 @@ describe('createOpSender', () => { expect.objectContaining({ code: 'doc_reset', workflowId: WORKFLOW }), expect.objectContaining({ code: 'doc_reset', workflowId: WORKFLOW }) ]) + expect( + vi.mocked(reportError).mock.calls.map(([, options]) => options.tags?.code) + ).toEqual(['doc_reset', 'doc_reset', 'doc_reset']) sender.enqueue([addNode(4)]) expect(sent).toHaveLength(2) diff --git a/src/workbench/extensions/agent/crdt/reportLostEdit.ts b/src/workbench/extensions/agent/crdt/reportLostEdit.ts index 97746fccbbf..b0ca24e2f1f 100644 --- a/src/workbench/extensions/agent/crdt/reportLostEdit.ts +++ b/src/workbench/extensions/agent/crdt/reportLostEdit.ts @@ -36,7 +36,7 @@ export function reportLostEdit( errorType, fingerprint: [errorType], level: kind === 'host_rejected' ? 'warning' : 'error', - tags: { tab: api.clientId, outcome: kind }, + tags: { tab: api.clientId, outcome: kind, code: cause.code }, context }) if (kind === 'mint_divergence' || kind === 'host_rejected') { From 1ce3f0e64268a44b10653670a177aba8da3b94b1 Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Wed, 21 Oct 2026 21:56:26 +0000 Subject: [PATCH 6/8] fix: preserve agent rejection diagnostics Amp-Thread-ID: https://ampcode.com/threads/T-32b78461-8ff4-4233-84f2-2eac5cb948d7 --- .../extensions/agent/crdt/rejectedOpNotice.ts | 75 ++++++++++++------- .../extensions/agent/crdt/reportLostEdit.ts | 2 + ...ntCrdtFollower.rejectedWidgetWrite.test.ts | 9 ++- .../agent/crdt/useAgentCrdtFollower.ts | 7 +- 4 files changed, 61 insertions(+), 32 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts index 63a314079fc..cac0e670426 100644 --- a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts +++ b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts @@ -1,3 +1,5 @@ +import { throttle } from 'es-toolkit' +import type { ThrottledFunction } from 'es-toolkit' import type { Op } from '@comfyorg/comfy-multi-player' import { useToast } from '@/components/ui/toast/toastStore' @@ -122,41 +124,62 @@ function noticeKey(widgetWrite: boolean, partial: boolean): string { } export function createRejectedOpNotifier() { - function showToast(description: string): void { - useToast().error(i18n.global.t('g.error'), { - id: NOTICE_ID, - description, - duration: NOTICE_LIFE_MS - }) + const throttledToasts = new Map< + string, + ThrottledFunction<(description: string) => void> + >() + + function showToast(kind: string, description: string): void { + const toast = throttledToasts.get(kind) + if (toast) return toast(description) + const throttled = throttle( + (detail: string) => + useToast().error(i18n.global.t('g.error'), { + id: NOTICE_ID, + description: detail, + duration: NOTICE_LIFE_MS + }), + NOTICE_LIFE_MS, + { edges: ['leading'] } + ) + throttledToasts.set(kind, throttled) + throttled(description) + } + + function notifyRejected(ops: readonly Op[], result: OpsResultView): void { + const { failed } = result + const widgetWrite = isRejectedWidgetWrite(ops, failed) + const code = failed?.code ?? result.code ?? 'unspecified' + const accepted = new Set([...result.applied, ...result.skipped]) + reportLostEdit( + 'host_rejected', + ops.filter((op) => !accepted.has(op.op_id)), + { + code, + workflowId: result.workflowId, + message: failed?.message, + failedIndex: failed?.index, + errorType: widgetWrite + ? 'error_applying_agent_widget_edit' + : 'error_applying_agent_graph_edit' + } + ) + const partial = result.applied.length > 0 && failed !== undefined + const kind = noticeKey(widgetWrite, partial) + const notice = i18n.global.t(kind) + const target = failed && rejectedTarget(rejectedOp(ops, failed)) + showToast(kind, withReason(notice, target, code)) } return { notify(ops: readonly Op[], result: OpsResultView) { if (result.ok) return - const { failed } = result - const widgetWrite = isRejectedWidgetWrite(ops, failed) - const code = failed?.code ?? result.code ?? 'unspecified' - const accepted = new Set([...result.applied, ...result.skipped]) - reportLostEdit( - 'host_rejected', - ops.filter((op) => !accepted.has(op.op_id)), - { - code, - workflowId: result.workflowId, - errorType: widgetWrite - ? 'error_applying_agent_widget_edit' - : 'error_applying_agent_graph_edit' - } - ) - const partial = result.applied.length > 0 && failed !== undefined - const kind = noticeKey(widgetWrite, partial) - const notice = i18n.global.t(kind) - const target = failed && rejectedTarget(rejectedOp(ops, failed)) - showToast(withReason(notice, target, code)) + notifyRejected(ops, result) }, notifyWidgetWriteRefused(nodeId: string | number, widgetName: string) { const notice = i18n.global.t('agent.editRejected.widgetWrite') showToast( + 'agent.editRejected.widgetWrite', withReason(notice, nodeFieldTarget(nodeId, widgetName), 'unspecified') ) } diff --git a/src/workbench/extensions/agent/crdt/reportLostEdit.ts b/src/workbench/extensions/agent/crdt/reportLostEdit.ts index b0ca24e2f1f..9525f20db35 100644 --- a/src/workbench/extensions/agent/crdt/reportLostEdit.ts +++ b/src/workbench/extensions/agent/crdt/reportLostEdit.ts @@ -21,6 +21,8 @@ export function reportLostEdit( workflowId?: string graphId?: string nodeId?: string | number + message?: string + failedIndex?: number } ): void { const { errorType = `failure_syncing_agent_edit_${kind}`, ...detail } = cause diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts index 99ba4256875..c8320792dab 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts @@ -424,12 +424,15 @@ describe('a human edit the doc host rejects', () => { ]) }) - it('shows a later rejection after the user dismisses the notice', async () => { + it('shows a later rejection after the throttle window', async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }) + onTestFinished(() => { + vi.useRealTimers() + }) const { submit } = mountFollower() answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) - useToast().dismiss(useToast().toasts[0].id) - expect(toastDescriptions()).toEqual([]) + await vi.advanceTimersByTimeAsync(10_001) answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) expect(toastDescriptions()).toEqual([ diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index b3899c8b187..6e12023a95f 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -19,6 +19,7 @@ import type { AgentGraphProjectionStage } from '@/platform/telemetry/types' import { api } from '@/scripts/api' import { parseNodeId } from '@/types/nodeId' import type { NodeId } from '@/types/nodeId' +import { createUuidv4 } from '@/utils/uuid' import { useAgentPanelStore } from '@/workbench/extensions/agent/stores/agent/agentPanelStore' import type { @@ -399,8 +400,8 @@ function startAgentCrdtFollower( terminal.value = reason } ) - const ownActor = (): string => - `human:${userId() ?? 'anonymous'}:${api.clientId ?? ''}` + const tabId = createUuidv4() + const ownActor = (): string => `human:${userId() ?? 'anonymous'}:${tabId}` // Doc node ids whose human delete the host has applied but whose effect // frame has not yet removed them from the doc. Kept pending for the // reconcile so the result-to-effect window cannot resurrect them. @@ -527,7 +528,7 @@ function startAgentCrdtFollower( // every send and resend, so ops never reach a doc we are not subscribed to. workflowId: () => bridge.subscribedWorkflowId, get tab() { - return api.clientId ?? '' + return tabId }, actor: ownActor, baseVersion: () => bridge.lastSequence, From 9cdea09ed504f18687e76530a6ad7b8146960bca Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Wed, 21 Oct 2026 22:04:14 +0000 Subject: [PATCH 7/8] fix: budget lost edit reports Amp-Thread-ID: https://ampcode.com/threads/T-32b78461-8ff4-4233-84f2-2eac5cb948d7 --- .../extensions/agent/crdt/opSender.test.ts | 24 +++++++++++++++++++ .../extensions/agent/crdt/opSender.ts | 14 +++++++---- .../extensions/agent/crdt/reportLostEdit.ts | 20 +++++++++------- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index 4afd379d16e..d77f6e525e6 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -18,7 +18,14 @@ import type { GraphOperation } from './graphOperations' import { createOpSender } from './opSender' import type { BatchOutcome, OpsResultView } from './opSender' +const devLogState = vi.hoisted(() => ({ + recordDevEvent: vi.fn() +})) + vi.mock(import('@/platform/telemetry/reportError')) +vi.mock(import('./devPanelLog'), () => ({ + recordDevEvent: devLogState.recordDevEvent +})) const WORKFLOW = 'wf-1' const TAB = 'tab-1' @@ -1528,6 +1535,23 @@ describe('createOpSender', () => { expect(sent).toHaveLength(2) }) + it('budgets platform lost-edit reports without skipping settlement events', () => { + sender.enqueue([addNode(1)]) + sender.enqueue([addNode(2)]) + sender.enqueue([addNode(3)]) + sender.enqueue([addNode(4)]) + sender.enqueue([addNode(5)]) + + sender.abortAll() + + expect(reportError).toHaveBeenCalledTimes(3) + expect( + devLogState.recordDevEvent.mock.calls.filter( + ([event]) => event === 'human_ops_settled' + ) + ).toHaveLength(5) + }) + it('abortAll settles every other batch when one settlement listener throws', () => { const localSettled: BatchOutcome[] = [] const localSender = createOpSender({ diff --git a/src/workbench/extensions/agent/crdt/opSender.ts b/src/workbench/extensions/agent/crdt/opSender.ts index 9a48230e617..60ad8ad6fd7 100644 --- a/src/workbench/extensions/agent/crdt/opSender.ts +++ b/src/workbench/extensions/agent/crdt/opSender.ts @@ -234,6 +234,7 @@ export function createOpSender(deps: OpSenderDeps): OpSender { // correlate a result to the send it answers. let staleAnonymousBudget = 0 const retiredOpIds = new Set() + const mayReportLostEdit = createReportBudget() const mayReportSettlementFailure = createReportBudget() const mayReportSerializationFailure = createReportBudget() @@ -274,10 +275,15 @@ export function createOpSender(deps: OpSenderDeps): OpSender { let reportedFailure = false return (outcome, workflowId) => { if (outcome.state !== 'acknowledged') - reportLostEdit(outcome.state, outcome.ops, { - code, - workflowId: workflowId ?? undefined - }) + reportLostEdit( + outcome.state, + outcome.ops, + { + code, + workflowId: workflowId ?? undefined + }, + { shouldReport: mayReportLostEdit() } + ) try { deps.onBatchSettled(outcome) } catch (cause) { diff --git a/src/workbench/extensions/agent/crdt/reportLostEdit.ts b/src/workbench/extensions/agent/crdt/reportLostEdit.ts index 9525f20db35..b725e62b327 100644 --- a/src/workbench/extensions/agent/crdt/reportLostEdit.ts +++ b/src/workbench/extensions/agent/crdt/reportLostEdit.ts @@ -23,7 +23,8 @@ export function reportLostEdit( nodeId?: string | number message?: string failedIndex?: number - } + }, + { shouldReport = true }: { shouldReport?: boolean } = {} ): void { const { errorType = `failure_syncing_agent_edit_${kind}`, ...detail } = cause const context = { @@ -33,14 +34,15 @@ export function reportLostEdit( opIds: ops.map((op) => op.op_id), operations: ops.map((op) => op.op) } - reportError(new Error(errorType), { - surface: 'agent', - errorType, - fingerprint: [errorType], - level: kind === 'host_rejected' ? 'warning' : 'error', - tags: { tab: api.clientId, outcome: kind, code: cause.code }, - context - }) + if (shouldReport) + reportError(new Error(errorType), { + surface: 'agent', + errorType, + fingerprint: [errorType], + level: kind === 'host_rejected' ? 'warning' : 'error', + tags: { tab: api.clientId, outcome: kind, code: cause.code }, + context + }) if (kind === 'mint_divergence' || kind === 'host_rejected') { recordDevEvent(kind, context, { level: 'warn' }) } else { From 2baa01c321ee3b43aa655e7dbf02db84652d5685 Mon Sep 17 00:00:00 2001 From: Alexander Brown Date: Wed, 21 Oct 2026 22:12:30 +0000 Subject: [PATCH 8/8] fix: report crdt actor tab Amp-Thread-ID: https://ampcode.com/threads/T-32b78461-8ff4-4233-84f2-2eac5cb948d7 --- src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index 6e12023a95f..dc6d233d55a 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -16,7 +16,6 @@ import type { LGraph } from '@/lib/litegraph/src/litegraph' import { useTelemetry } from '@/platform/telemetry' import { reportError } from '@/platform/telemetry/reportError' import type { AgentGraphProjectionStage } from '@/platform/telemetry/types' -import { api } from '@/scripts/api' import { parseNodeId } from '@/types/nodeId' import type { NodeId } from '@/types/nodeId' import { createUuidv4 } from '@/utils/uuid' @@ -1147,7 +1146,7 @@ function startAgentCrdtFollower( const debugSnapshot = (): CrdtDebugSnapshot => readCrdtSnapshot(bridge.follower.doc, { status: status.value, - tabId: api.clientId ?? null, + tabId, lastSeq: bridge.lastSequence, schemaError: bridge.lastSchemaError?.message ?? null })