diff --git a/docs/adr/CRDT-AUTHORITY-0035-human-canvas-authority-and-draft-reconciliation.md b/docs/adr/CRDT-AUTHORITY-0035-human-canvas-authority-and-draft-reconciliation.md index 30245c12ed6..8fca31f58c4 100644 --- a/docs/adr/CRDT-AUTHORITY-0035-human-canvas-authority-and-draft-reconciliation.md +++ b/docs/adr/CRDT-AUTHORITY-0035-human-canvas-authority-and-draft-reconciliation.md @@ -150,9 +150,12 @@ Alternatives considered and rejected: operations of that batch unapplied, where separate batches were independent. Chunking at the wire cap bounds a batch, and the ledger already models `unprocessed`. -- Human operations settled `undeliverable` or `unconfirmed` (at unbind, at - `doc_reset`, on a dead transport) are still reported only to the dev panel; - surfacing them to the user is a follow-up. +- Human operations settled `undeliverable`, `unconfirmed`, or + `unacknowledged` produce one `reportError` call and a dev-panel entry per + batch. Reports include the mint-time workflow id and termination cause. + `doc_reset` identifies batches superseded by the draft under decision 4, + so lost-edit queries can exclude them. User-facing notices for sender + termination remain a follow-up. - Two browser tabs on the same document, both agent-bound, are out of scope: which tab's draft wins a divergence is decided by the server's sequence check, not by anything the frontend promises. diff --git a/src/platform/telemetry/reportError.test.ts b/src/platform/telemetry/reportError.test.ts index 938bd6615ba..696f4bba2d8 100644 --- a/src/platform/telemetry/reportError.test.ts +++ b/src/platform/telemetry/reportError.test.ts @@ -72,12 +72,14 @@ describe('reportError', () => { reportError(error, { surface: 'platform', - errorType: 'workspace_auth_gate_initialization_failure' + errorType: 'workspace_auth_gate_initialization_failure', + fingerprint: ['workspace_auth_gate_initialization_failure'] }) expect(captureException).toHaveBeenCalledWith( error, expect.objectContaining({ + fingerprint: ['workspace_auth_gate_initialization_failure'], tags: expect.objectContaining({ bucket: 'canary' }) }) ) diff --git a/src/platform/telemetry/reportError.ts b/src/platform/telemetry/reportError.ts index 2f35259b92a..29fb1e5c127 100644 --- a/src/platform/telemetry/reportError.ts +++ b/src/platform/telemetry/reportError.ts @@ -31,6 +31,7 @@ export interface ReportErrorOptions { errorType: string /** Product surface responsible for acting on this failure. */ surface: Surface + fingerprint?: string[] tags?: Record context?: Record level?: 'warning' | 'error' @@ -171,13 +172,15 @@ function dispatchToSentry( surface: Surface, tags: Record, context: Record, - level: ReportErrorOptions['level'] + level: ReportErrorOptions['level'], + fingerprint: ReportErrorOptions['fingerprint'] ): boolean { try { captureException(error, { tags: { ...tags, error_type: errorType, surface }, extra: context, - level + level, + ...(fingerprint ? { fingerprint } : {}) }) return true } catch (reporterFailure) { @@ -249,7 +252,8 @@ function dispatch( surface, tags, context, - level + level, + options.fingerprint ) } if (datadogLive) { diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.test.ts b/src/workbench/extensions/agent/crdt/devPanelLog.test.ts index 0229f057014..ccdb6ce3284 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.test.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.test.ts @@ -199,20 +199,24 @@ describe('devPanelLog', () => { it('keeps the structural diagnosis on failed frames', () => { recordDevEvent('doc_ops_result', { workflowId: 'wf-1', + graphId: '1ff39644-78b1-48bb-9509-53b671c672f1', ok: false, applied: [], skipped: ['op-1'], code: 'op_failed', + operations: ['set_widget', 'delete_node', 'private user prose'], message: 'unknown node 7', failed: { op_id: 'op-1', code: 'unknown_node', message: 'unknown node 7' } }) expect(devEvents.value[0]?.detail).toEqual({ workflowId: 'wf-1', + graphId: '1ff39644-78b1-48bb-9509-53b671c672f1', ok: false, applied: [], skipped: ['op-1'], code: 'op_failed', + operations: ['set_widget', 'delete_node', 'String(18)'], message: 'String(14)', failed: { op_id: 'op-1', code: 'unknown_node', message: 'String(14)' } }) diff --git a/src/workbench/extensions/agent/crdt/devPanelLog.ts b/src/workbench/extensions/agent/crdt/devPanelLog.ts index 115541be084..206bbf09c5d 100644 --- a/src/workbench/extensions/agent/crdt/devPanelLog.ts +++ b/src/workbench/extensions/agent/crdt/devPanelLog.ts @@ -30,6 +30,8 @@ export const DEV_EVENT_KINDS = [ 'doc_ops_result', 'doc_ops_result_dropped', 'human_ops_settled', + 'mint_divergence', + 'host_rejected', 'doc_reset', 'doc_nodes_changed', 'schema_error', @@ -96,6 +98,7 @@ const STRUCTURAL_STRING_KEYS = new Set([ 'doc_id', 'failed', 'frame_type', + 'graph_id', 'id', 'kind', 'level', @@ -104,6 +107,7 @@ const STRUCTURAL_STRING_KEYS = new Set([ 'op', 'op_id', 'op_ids', + 'operations', 'outcome', 'phase', 'prompt_id', diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts index 6a1b909ea54..755f6e5f28d 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.test.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.test.ts @@ -258,6 +258,31 @@ describe('attachDocOpMinter', () => { } }) + it('reports mint divergence once when a node cannot be serialized', async () => { + const node = new TestSource() + graph.add(node) + vi.spyOn(node, 'serialize').mockImplementation(() => { + throw new Error('private widget content') + }) + + await afterFlush() + await afterFlush() + + expect(minted).toEqual([]) + expect(reportError).toHaveBeenCalledExactlyOnceWith( + new Error('agent_crdt_node_snapshot_missing'), + expect.objectContaining({ + fingerprint: ['agent_crdt_node_snapshot_missing'], + context: expect.objectContaining({ + code: 'missing_node_snapshot', + graphId: graph.id, + nodeId: node.id, + operations: ['add_node'] + }) + }) + ) + }) + describe('mints one op per graph command', () => { it.for([ { @@ -2047,21 +2072,27 @@ describe('attachDocOpMinter', () => { expect( vi.mocked(reportError).mock.calls.map(([, options]) => options) ).toEqual([ - { - surface: 'agent', + expect.objectContaining({ errorType: 'agent_crdt_widget_owner_unresolved', - context: { graphId: staleGraph.id, nodeId: stale.id } - }, - { - surface: 'agent', + context: expect.objectContaining({ + graphId: staleGraph.id, + nodeId: stale.id + }) + }), + expect.objectContaining({ errorType: 'agent_crdt_widget_owner_unresolved', - context: { graphId: otherGraph.id, nodeId: other.id } - }, - { - surface: 'agent', + context: expect.objectContaining({ + graphId: otherGraph.id, + nodeId: other.id + }) + }), + expect.objectContaining({ errorType: 'agent_crdt_widget_owner_unresolved', - context: { graphId: staleGraph.id, nodeId: sibling.id } - } + context: expect.objectContaining({ + graphId: staleGraph.id, + nodeId: sibling.id + }) + }) ]) }) diff --git a/src/workbench/extensions/agent/crdt/docOpMinter.ts b/src/workbench/extensions/agent/crdt/docOpMinter.ts index fa88f494b3e..89641f318ed 100644 --- a/src/workbench/extensions/agent/crdt/docOpMinter.ts +++ b/src/workbench/extensions/agent/crdt/docOpMinter.ts @@ -51,6 +51,7 @@ import { } from './agentCrdtDocLifecycle' import type { GraphOperation } from './graphOperations' import type { DocPromotedWidgets } from './agentSubgraphDefinitions' +import { reportLostEdit } from './reportLostEdit' export interface DocOpMinterDeps { /** Slice 00's product gate. */ @@ -446,12 +447,15 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { if (node.graph !== graph) return null const snapshot = wireNodeSnapshot(node) if (!snapshot) { - reportOnce( - `snapshot:${nodeKey(graph.id, node.id)}`, - 'Agent node add has no snapshot', - 'agent_crdt_node_snapshot_missing', - { graphId: graph.id, nodeId: node.id } - ) + const key = `snapshot:${nodeKey(graph.id, node.id)}` + if (reported.has(key)) return null + reported.add(key) + reportLostEdit('mint_divergence', [{ op: 'add_node' }], { + code: 'missing_node_snapshot', + errorType: 'agent_crdt_node_snapshot_missing', + graphId: graph.id, + nodeId: node.id + }) return null } return { @@ -537,7 +541,7 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { const rootGraphId = deps.boundRootGraphId() ?? graph.id const routed = routedWidgetOperation(graph, rootGraphId, event, owner, deps) if ('unresolvedGraphId' in routed) { - reportUnresolvedOwner(routed.unresolvedGraphId, event) + reportUnresolvedWidgetOwner(routed.unresolvedGraphId, event) return } if ('reason' in routed) { @@ -547,6 +551,21 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { schedule({ kind: 'op', operation: routed }) } + function reportUnresolvedWidgetOwner( + graphId: string, + event: IntentOf<'set_widget'> + ): void { + const key = `owner:${nodeKey(graphId, event.nodeId)}` + if (reportedRefusals.has(key)) return + reportedRefusals.add(key) + reportLostEdit('mint_divergence', [{ op: 'set_widget' }], { + code: 'unresolvable_widget_owner', + errorType: 'agent_crdt_widget_owner_unresolved', + graphId, + nodeId: event.nodeId + }) + } + function takeHeldWrite( event: IntentOf<'set_widget'> ): HeldPromotedWrite | null { @@ -556,19 +575,6 @@ export function attachDocOpMinter(deps: DocOpMinterDeps): DocOpMinter { return held } - function reportUnresolvedOwner( - graphId: string, - event: IntentOf<'set_widget'> - ): void { - reportOnce( - `owner:${nodeKey(graphId, event.nodeId)}`, - 'Agent widget write has an unresolvable owner', - 'agent_crdt_widget_owner_unresolved', - { graphId, nodeId: event.nodeId }, - reportedRefusals - ) - } - function holdOrRefuseWidgetWrite( rootGraphId: string, event: IntentOf<'set_widget'>, diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index 57f4268a21f..d77f6e525e6 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -1,15 +1,31 @@ import type { Op } from '@comfyorg/comfy-multi-player' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + afterEach, + beforeEach, + describe, + expect, + it, + onTestFinished, + vi +} from 'vitest' import type { Mock } from 'vitest' import { reportError } from '@/platform/telemetry/reportError' +import { api } from '@/scripts/api' import type { DocOp } from './docFrameClient' import type { GraphOperation } from './graphOperations' import { createOpSender } from './opSender' import type { BatchOutcome, OpsResultView } from './opSender' +const devLogState = vi.hoisted(() => ({ + recordDevEvent: vi.fn() +})) + vi.mock(import('@/platform/telemetry/reportError')) +vi.mock(import('./devPanelLog'), () => ({ + recordDevEvent: devLogState.recordDevEvent +})) const WORKFLOW = 'wf-1' const TAB = 'tab-1' @@ -30,6 +46,12 @@ function summarizeSettlement(outcome: BatchOutcome): SettlementSummary { } } +function reportsOfType(errorType: string) { + return vi + .mocked(reportError) + .mock.calls.filter(([, options]) => options.errorType === errorType) +} + const detachListenerFailureCases = [ [ 'in-flight', @@ -188,6 +210,47 @@ describe('createOpSender', () => { sender.detach() }) + it.for([ + ['undeliverable', false, WORKFLOW, 3_000, 'transport_exhausted'], + ['undeliverable', true, null, 0, 'unbound'], + ['unconfirmed', true, WORKFLOW, 0, 'detached'], + ['unacknowledged', true, WORKFLOW, 20_000, 'result_timeout'] + ] as const)( + 'reports %s exactly once with transport=%s, workflow=%s, elapsed=%s', + ([state, available, workflow, elapsed, code]) => { + transportUp = available + boundWorkflow = workflow + const previousClientId = api.clientId + api.clientId = 'api-tab' + onTestFinished(() => { + api.clientId = previousClientId + }) + + sender.enqueue([addNode(7), addNode(12)]) + vi.advanceTimersByTime(elapsed) + sender.detach() + sender.abortAll() + vi.advanceTimersByTime(60_000) + + expect(settled).toHaveLength(1) + expect(settled[0].state).toBe(state) + expect(reportError).toHaveBeenCalledExactlyOnceWith(expect.any(Error), { + surface: 'agent', + errorType: `failure_syncing_agent_edit_${state}`, + fingerprint: [`failure_syncing_agent_edit_${state}`], + level: 'error', + tags: { tab: 'api-tab', outcome: state, code }, + context: { + code, + workflowId: workflow ?? undefined, + opId: settled[0].ops[0].op_id, + opIds: settled[0].ops.map((op) => op.op_id), + operations: ['add_node', 'add_node'] + } + }) + } + ) + it('mints once and sends a doc_ops batch with the wire envelope', () => { sender.enqueue([addNode(1), addNode(2)]) @@ -421,6 +484,15 @@ describe('createOpSender', () => { expect(sent).toHaveLength(2) expect(sent[1].workflowId).toBe('wf-2') expect(settled[0].state).toBe('unconfirmed') + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + expect.objectContaining({ + context: expect.objectContaining({ + code: 'unbound', + workflowId: WORKFLOW + }) + }) + ) }) it('drains 20,000 queued batches after unbinding without overflowing the stack', () => { @@ -999,7 +1071,9 @@ describe('createOpSender', () => { localSender.detach() expect(localSettled.map(summarizeSettlement)).toEqual(expectedSurvivors) - expect(reportError).toHaveBeenCalledTimes(1) + expect( + reportsOfType('failure_settling_agent_op_sender_detach') + ).toHaveLength(1) expect(reportError).toHaveBeenCalledWith( expect.any(Error), expect.objectContaining({ @@ -1029,7 +1103,9 @@ describe('createOpSender', () => { localSender.detach() - expect(reportError).toHaveBeenCalledTimes(1) + expect( + reportsOfType('failure_settling_agent_op_sender_detach') + ).toHaveLength(1) expect(localUnsubscribe).toHaveBeenCalledOnce() }) @@ -1050,7 +1126,9 @@ describe('createOpSender', () => { localSender.enqueue([addNode(1)]) localSender.enqueue([addNode(2)]) - expect(reportError).toHaveBeenCalledTimes(1) + expect( + reportsOfType('failure_settling_agent_op_sender_detach') + ).toHaveLength(1) expect(reportError).toHaveBeenCalledWith( expect.any(Error), expect.objectContaining({ @@ -1066,12 +1144,20 @@ describe('createOpSender', () => { { state: 'undeliverable', nodeIds: [1] } ]) expect(sender.pending()).toBe(0) - expect(reportError).toHaveBeenCalledExactlyOnceWith( - expect.any(TypeError), - expect.objectContaining({ - errorType: 'failure_serializing_agent_op_sender' - }) - ) + expect(reportsOfType('failure_serializing_agent_op_sender')).toEqual([ + [expect.any(TypeError), expect.objectContaining({ surface: 'agent' })] + ]) + expect(reportsOfType('failure_syncing_agent_edit_undeliverable')).toEqual([ + [ + expect.any(Error), + expect.objectContaining({ + context: expect.objectContaining({ + code: 'serialization_failed', + workflowId: WORKFLOW + }) + }) + ] + ]) sender.detach() @@ -1143,7 +1229,7 @@ describe('createOpSender', () => { enqueueCircular(4) enqueueCircular(5) - expect(reportError).toHaveBeenCalledTimes(3) + expect(reportsOfType('failure_serializing_agent_op_sender')).toHaveLength(3) }) it('keeps serialization telemetry bounded across successful sends and re-arms by time', () => { @@ -1153,17 +1239,17 @@ describe('createOpSender', () => { enqueueCircular(1) enqueueCircular(2) enqueueCircular(3) - expect(reportError).toHaveBeenCalledTimes(3) + expect(reportsOfType('failure_serializing_agent_op_sender')).toHaveLength(3) sender.enqueue([addNode(4)]) ackInFlight() enqueueCircular(5) - expect(reportError).toHaveBeenCalledTimes(3) + expect(reportsOfType('failure_serializing_agent_op_sender')).toHaveLength(3) vi.advanceTimersByTime(60_000) enqueueCircular(6) - expect(reportError).toHaveBeenCalledTimes(4) + expect(reportsOfType('failure_serializing_agent_op_sender')).toHaveLength(4) }) it('rejects an unserializable non-batchable op before transport', () => { @@ -1434,11 +1520,38 @@ describe('createOpSender', () => { ) ).toEqual([[1], [2], [3]]) expect(sender.pending()).toBe(0) + expect( + vi.mocked(reportError).mock.calls.map(([, options]) => options.context) + ).toEqual([ + expect.objectContaining({ code: 'doc_reset', workflowId: WORKFLOW }), + expect.objectContaining({ code: 'doc_reset', workflowId: WORKFLOW }), + expect.objectContaining({ code: 'doc_reset', workflowId: WORKFLOW }) + ]) + expect( + vi.mocked(reportError).mock.calls.map(([, options]) => options.tags?.code) + ).toEqual(['doc_reset', 'doc_reset', 'doc_reset']) sender.enqueue([addNode(4)]) expect(sent).toHaveLength(2) }) + it('budgets platform lost-edit reports without skipping settlement events', () => { + sender.enqueue([addNode(1)]) + sender.enqueue([addNode(2)]) + sender.enqueue([addNode(3)]) + sender.enqueue([addNode(4)]) + sender.enqueue([addNode(5)]) + + sender.abortAll() + + expect(reportError).toHaveBeenCalledTimes(3) + expect( + devLogState.recordDevEvent.mock.calls.filter( + ([event]) => event === 'human_ops_settled' + ) + ).toHaveLength(5) + }) + it('abortAll settles every other batch when one settlement listener throws', () => { const localSettled: BatchOutcome[] = [] const localSender = createOpSender({ diff --git a/src/workbench/extensions/agent/crdt/opSender.ts b/src/workbench/extensions/agent/crdt/opSender.ts index ed2a4fe7621..60ad8ad6fd7 100644 --- a/src/workbench/extensions/agent/crdt/opSender.ts +++ b/src/workbench/extensions/agent/crdt/opSender.ts @@ -28,6 +28,7 @@ import { WIRE_MAX_OPS_PER_BATCH } from './opEnvelope' import type { SizedOp } from './opEnvelope' +import { reportLostEdit } from './reportLostEdit' const SEND_RETRY_LIMIT = 5 const SEND_RETRY_INTERVAL_MS = 500 @@ -233,6 +234,7 @@ export function createOpSender(deps: OpSenderDeps): OpSender { // correlate a result to the send it answers. let staleAnonymousBudget = 0 const retiredOpIds = new Set() + const mayReportLostEdit = createReportBudget() const mayReportSettlementFailure = createReportBudget() const mayReportSerializationFailure = createReportBudget() @@ -267,10 +269,21 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } function guardedSettlementNotifier( - errorType: string - ): (outcome: BatchOutcome) => void { + errorType: string, + code: string + ): (outcome: BatchOutcome, workflowId: string | null) => void { let reportedFailure = false - return (outcome) => { + return (outcome, workflowId) => { + if (outcome.state !== 'acknowledged') + reportLostEdit( + outcome.state, + outcome.ops, + { + code, + workflowId: workflowId ?? undefined + }, + { shouldReport: mayReportLostEdit() } + ) try { deps.onBatchSettled(outcome) } catch (cause) { @@ -282,12 +295,17 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } const notifyDetachSettlement = guardedSettlementNotifier( - 'failure_settling_agent_op_sender_detach' + 'failure_settling_agent_op_sender_detach', + 'detached' ) - function settle(outcome: BatchOutcome): void { + function settle(outcome: BatchOutcome, code: string): void { + const workflowId = inFlight?.workflowId ?? null if (inFlight?.timer) clearTimeout(inFlight.timer) inFlight = null - guardedSettlementNotifier('failure_settling_agent_op_sender')(outcome) + guardedSettlementNotifier('failure_settling_agent_op_sender', code)( + outcome, + workflowId + ) pump() } @@ -297,23 +315,18 @@ export function createOpSender(deps: OpSenderDeps): OpSender { batch.parked = true return } - // A lost subscription is not a transport that recovers in 500 ms: settle - // now rather than spend the retry budget while later batches wait behind. if (deps.workflowId() !== batch.workflowId) { - settleUnbound(batch) + settleUnbound(batch, 'unbound') return } if (!trySend(batch)) { if (attempt < SEND_RETRY_LIMIT) { - // Tracked in the same slot as the result timer (they never overlap: - // the result timer is armed only after a successful send) so - // settle()/detach() clear a pending retry too. batch.timer = setTimeout( () => transmit(batch, attempt + 1), SEND_RETRY_INTERVAL_MS ) } else { - settleUnbound(batch) + settleUnbound(batch, 'transport_exhausted') } return } @@ -343,13 +356,16 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } } - function settleUnbound(batch: InFlight): void { + function settleUnbound(batch: InFlight, code: string): void { if (inFlight !== batch) return retire(batch, 0) - settle({ - state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable', - ops: batch.ops - }) + settle( + { + state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable', + ops: batch.ops + }, + code + ) } function armResultTimeout(batch: InFlight): void { @@ -358,18 +374,18 @@ export function createOpSender(deps: OpSenderDeps): OpSender { if (inFlight !== batch) return if (batch.resent) { retire(batch, 0) - settle({ state: 'unacknowledged', ops: batch.ops }) + settle({ state: 'unacknowledged', ops: batch.ops }, 'result_timeout') return } - // One silent-result resend of the SAME minted ops: idempotent at the - // applier through the op_id gate. batch.resent = true batch.reportedThrow = false transmit(batch, 0) }, RESULT_TIMEOUT_MS) } - function drainOutstanding(notify: (outcome: BatchOutcome) => void): void { + function drainOutstanding( + notify: (outcome: BatchOutcome, workflowId: string | null) => void + ): void { const unsealed = open open = null const queued = queue.slice(queueHead) @@ -381,17 +397,23 @@ export function createOpSender(deps: OpSenderDeps): OpSender { const batch = active if (batch.timer) clearTimeout(batch.timer) retire(batch, 0) - notify({ - state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable', - ops: batch.ops - }) + notify( + { + state: batch.sends > 0 ? 'unconfirmed' : 'undeliverable', + ops: batch.ops + }, + batch.workflowId + ) } for (const batch of queued) { - notify({ state: 'undeliverable', ops: batch.ops }) + notify({ state: 'undeliverable', ops: batch.ops }, batch.workflowId) } if (unsealed) { for (const chunk of chunkWireOps(unsealed.ops)) { - notify({ state: 'undeliverable', ops: chunk.map((sized) => sized.op) }) + notify( + { state: 'undeliverable', ops: chunk.map((sized) => sized.op) }, + unsealed.workflowId + ) } } } @@ -432,13 +454,17 @@ export function createOpSender(deps: OpSenderDeps): OpSender { function settleUndeliverableInBoundedGroups( ops: Op[], - notify: (outcome: BatchOutcome) => void + notify: (outcome: BatchOutcome, workflowId: string | null) => void, + workflowId: string | null ): void { for (let index = 0; index < ops.length; index += WIRE_MAX_OPS_PER_BATCH) { - notify({ - state: 'undeliverable', - ops: ops.slice(index, index + WIRE_MAX_OPS_PER_BATCH) - }) + notify( + { + state: 'undeliverable', + ops: ops.slice(index, index + WIRE_MAX_OPS_PER_BATCH) + }, + workflowId + ) } } @@ -524,14 +550,19 @@ export function createOpSender(deps: OpSenderDeps): OpSender { cause }: Admission): void { if (detached) { - settleUndeliverableInBoundedGroups(minted, notifyDetachSettlement) + settleUndeliverableInBoundedGroups( + minted, + notifyDetachSettlement, + workflowId + ) return } const notifyFailure = guardedSettlementNotifier( - 'failure_settling_agent_op_sender' + 'failure_settling_agent_op_sender', + workflowId === null ? 'unbound' : 'serialization_failed' ) if (workflowId === null) { - settleUndeliverableInBoundedGroups(minted, notifyFailure) + settleUndeliverableInBoundedGroups(minted, notifyFailure, workflowId) return } appendToOpenGroup(workflowId, admitted) @@ -539,7 +570,7 @@ export function createOpSender(deps: OpSenderDeps): OpSender { if (mayReportSerializationFailure()) { reportDegraded(cause, 'failure_serializing_agent_op_sender') } - settleUndeliverableInBoundedGroups(rejected, notifyFailure) + settleUndeliverableInBoundedGroups(rejected, notifyFailure, workflowId) } function seal(): void { @@ -560,42 +591,53 @@ export function createOpSender(deps: OpSenderDeps): OpSender { pump() } - const unsubscribe = deps.onOpsResult((result) => { - if (detached) return - if (inFlight === null && staleAnonymousBudget === 0) return - const identified = [...result.applied, ...result.skipped] - if (result.failed?.op_id) identified.push(result.failed.op_id) - const batch = - inFlight !== null && - (result.workflowId === undefined || - result.workflowId === inFlight.workflowId) - ? inFlight - : null - if (identified.length > 0) { - if (batch && identified.some((opId) => batch.opIds.has(opId))) { - retire(batch, 1) - settle({ state: 'acknowledged', ops: batch.ops, result }) - } else if (identified.some((opId) => retiredOpIds.has(opId))) { - // A retired batch's own answer consumes the credit reserved for it; - // ops this sender never minted are nobody's answer here. - drainStaleCredit() - } + function resultBatch(result: OpsResultView): InFlight | null { + if (inFlight === null) return null + if (result.workflowId && result.workflowId !== inFlight.workflowId) + return null + return inFlight + } + + function resultOpIds(result: OpsResultView): string[] { + return [ + ...result.applied, + ...result.skipped, + ...(result.failed?.op_id ? [result.failed.op_id] : []) + ] + } + + function settleResult(batch: InFlight, result: OpsResultView): void { + retire(batch, 1) + settle({ state: 'acknowledged', ops: batch.ops, result }, 'host_result') + } + + function settleIdentifiedResult( + batch: InFlight | null, + result: OpsResultView, + opIds: readonly string[] + ): void { + if (batch && opIds.some((opId) => batch.opIds.has(opId))) { + settleResult(batch, result) return } - // Anonymous failure (empty lists, no failed op_id): a late result with - // no batch waiting, one addressed to another workflow, or one a stale - // credit could explain drains that credit so it cannot swallow a future - // batch's own result. Only then is it the in-flight batch's. + if (opIds.some((opId) => retiredOpIds.has(opId))) drainStaleCredit() + } + + function handleOpsResult(result: OpsResultView): void { + if (detached) return + if (inFlight === null && staleAnonymousBudget === 0) return + const opIds = resultOpIds(result) + const batch = resultBatch(result) + if (opIds.length > 0) return settleIdentifiedResult(batch, result, opIds) if (batch === null || staleAnonymousBudget > 0) { drainStaleCredit() return } - // A parked batch or one waiting for a transport retry has not put a send - // on the wire, so an anonymous result cannot belong to it. if (batch.sends === 0) return - retire(batch, 1) - settle({ state: 'acknowledged', ops: batch.ops, result }) - }) + settleResult(batch, result) + } + + const unsubscribe = deps.onOpsResult(handleOpsResult) return { enqueue(operations) { @@ -632,16 +674,17 @@ export function createOpSender(deps: OpSenderDeps): OpSender { }, abortIfUnbound() { if (inFlight && deps.workflowId() !== inFlight.workflowId) { - settleUnbound(inFlight) + settleUnbound(inFlight, 'unbound') } }, abortAll() { - // A doc_reset replaces only the bound document (the follower guards on - // isCurrentWorkflow), so only its cursor restarts. const resetWorkflowId = deps.workflowId() if (resetWorkflowId !== null) lastMintedVersions.delete(resetWorkflowId) drainOutstanding( - guardedSettlementNotifier('failure_settling_agent_op_sender_abort') + guardedSettlementNotifier( + 'failure_settling_agent_op_sender_abort', + 'doc_reset' + ) ) }, detach() { diff --git a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts index ece7ea3dc81..cac0e670426 100644 --- a/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts +++ b/src/workbench/extensions/agent/crdt/rejectedOpNotice.ts @@ -4,12 +4,14 @@ import type { Op } from '@comfyorg/comfy-multi-player' import { useToast } from '@/components/ui/toast/toastStore' import { i18n } from '@/i18n' -import { reportError } from '@/platform/telemetry/reportError' +import { createToastId } from '@/types/toastId' import type { DocOpFailure } from './docFrameClient' import type { OpsResultView } from './opSender' +import { reportLostEdit } from './reportLostEdit' const NOTICE_LIFE_MS = 10_000 +const NOTICE_ID = createToastId() function rejectedOp(ops: readonly Op[], failed: DocOpFailure): Op | undefined { return ops.find((op) => op.op_id === failed.op_id) ?? ops[failed.index] @@ -124,71 +126,62 @@ function noticeKey(widgetWrite: boolean, partial: boolean): string { export function createRejectedOpNotifier() { const throttledToasts = new Map< string, - ThrottledFunction<(detail: string) => void> + ThrottledFunction<(description: string) => void> >() - const reportedCodes = new Set() - function reportOnce( - code: string, - widgetWrite: boolean, - result: OpsResultView, - failed: DocOpFailure | undefined - ): void { - if (reportedCodes.has(code)) return - reportedCodes.add(code) - reportError( - new Error(`the agent doc host rejected a human edit (${code})`), - { - surface: 'agent', - errorType: widgetWrite - ? 'error_applying_agent_widget_edit' - : 'error_applying_agent_graph_edit', - context: { workflowId: result.workflowId, opId: failed?.op_id, code }, - level: 'warning' - } - ) - } - - function showThrottled(kind: string, description: string): void { - const existing = throttledToasts.get(kind) - if (existing) return existing(description) - const created = throttle( - (detail: string) => { + function showToast(kind: string, description: string): void { + const toast = throttledToasts.get(kind) + if (toast) return toast(description) + const throttled = throttle( + (detail: string) => useToast().error(i18n.global.t('g.error'), { + id: NOTICE_ID, description: detail, duration: NOTICE_LIFE_MS - }) - }, + }), NOTICE_LIFE_MS, { edges: ['leading'] } ) - throttledToasts.set(kind, created) - created(description) + throttledToasts.set(kind, throttled) + throttled(description) + } + + function notifyRejected(ops: readonly Op[], result: OpsResultView): void { + const { failed } = result + const widgetWrite = isRejectedWidgetWrite(ops, failed) + const code = failed?.code ?? result.code ?? 'unspecified' + const accepted = new Set([...result.applied, ...result.skipped]) + reportLostEdit( + 'host_rejected', + ops.filter((op) => !accepted.has(op.op_id)), + { + code, + workflowId: result.workflowId, + message: failed?.message, + failedIndex: failed?.index, + errorType: widgetWrite + ? 'error_applying_agent_widget_edit' + : 'error_applying_agent_graph_edit' + } + ) + const partial = result.applied.length > 0 && failed !== undefined + const kind = noticeKey(widgetWrite, partial) + const notice = i18n.global.t(kind) + const target = failed && rejectedTarget(rejectedOp(ops, failed)) + showToast(kind, withReason(notice, target, code)) } return { notify(ops: readonly Op[], result: OpsResultView) { if (result.ok) return - const { failed } = result - const widgetWrite = isRejectedWidgetWrite(ops, failed) - const code = failed?.code ?? result.code ?? 'unspecified' - reportOnce(code, widgetWrite, result, failed) - const partial = result.applied.length > 0 && failed !== undefined - const kind = noticeKey(widgetWrite, partial) - const notice = i18n.global.t(kind) - const target = failed && rejectedTarget(rejectedOp(ops, failed)) - showThrottled(kind, withReason(notice, target, code)) + notifyRejected(ops, result) }, notifyWidgetWriteRefused(nodeId: string | number, widgetName: string) { const notice = i18n.global.t('agent.editRejected.widgetWrite') - showThrottled( + showToast( 'agent.editRejected.widgetWrite', withReason(notice, nodeFieldTarget(nodeId, widgetName), 'unspecified') ) - }, - cancel() { - for (const throttled of throttledToasts.values()) throttled.cancel() - throttledToasts.clear() } } } diff --git a/src/workbench/extensions/agent/crdt/reportLostEdit.ts b/src/workbench/extensions/agent/crdt/reportLostEdit.ts new file mode 100644 index 00000000000..b725e62b327 --- /dev/null +++ b/src/workbench/extensions/agent/crdt/reportLostEdit.ts @@ -0,0 +1,51 @@ +import type { Op } from '@comfyorg/comfy-multi-player' + +import { reportError } from '@/platform/telemetry/reportError' +import { api } from '@/scripts/api' + +import { recordDevEvent } from './devPanelLog' + +type LostEditKind = + | 'undeliverable' + | 'unconfirmed' + | 'unacknowledged' + | 'mint_divergence' + | 'host_rejected' + +export function reportLostEdit( + kind: LostEditKind, + ops: readonly (Pick & Partial>)[], + cause: { + code: string + errorType?: string + workflowId?: string + graphId?: string + nodeId?: string | number + message?: string + failedIndex?: number + }, + { shouldReport = true }: { shouldReport?: boolean } = {} +): void { + const { errorType = `failure_syncing_agent_edit_${kind}`, ...detail } = cause + const context = { + ...detail, + workflowId: cause.workflowId, + opId: ops[0]?.op_id, + opIds: ops.map((op) => op.op_id), + operations: ops.map((op) => op.op) + } + if (shouldReport) + reportError(new Error(errorType), { + surface: 'agent', + errorType, + fingerprint: [errorType], + level: kind === 'host_rejected' ? 'warning' : 'error', + tags: { tab: api.clientId, outcome: kind, code: cause.code }, + context + }) + if (kind === 'mint_divergence' || kind === 'host_rejected') { + recordDevEvent(kind, context, { level: 'warn' }) + } else { + recordDevEvent('human_ops_settled', { state: kind, ops }) + } +} diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts index d05408245f7..c8320792dab 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.rejectedWidgetWrite.test.ts @@ -1,7 +1,8 @@ -import { render } from '@testing-library/vue' +import { render, screen } from '@testing-library/vue' import { describe, expect, it, onTestFinished, vi } from 'vitest' -import { defineComponent, ref } from 'vue' +import { defineComponent, nextTick, ref } from 'vue' +import Toaster from '@/components/ui/toast/Toaster.vue' import { useToast } from '@/components/ui/toast/toastStore' import { i18n } from '@/i18n' import { reportError } from '@/platform/telemetry/reportError' @@ -215,6 +216,13 @@ describe('a human edit the doc host rejects', () => { expect(toastDescriptions()).toEqual([ expect.stringContaining(WIDGET_REJECTION_TEXT) ]) + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'error_applying_agent_widget_edit', + tags: expect.objectContaining({ outcome: 'host_rejected' }) + }) + ) } ) @@ -389,9 +397,7 @@ describe('a human edit the doc host rejects', () => { answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) expect(toastDescriptions()).toEqual([ - expect.stringMatching( - new RegExp(`${WIDGET_REJECTION_TEXT}.*Rejected: node 3 seed$`) - ) + expect.stringContaining(WIDGET_REJECTION_TEXT) ]) }) @@ -418,7 +424,7 @@ describe('a human edit the doc host rejects', () => { ]) }) - it('speaks again once the throttle window has passed', async () => { + it('shows a later rejection after the throttle window', async () => { vi.useFakeTimers({ shouldAdvanceTime: true }) onTestFinished(() => { vi.useRealTimers() @@ -427,20 +433,16 @@ describe('a human edit the doc host rejects', () => { answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) await vi.advanceTimersByTimeAsync(10_001) - answerWithOpsResult( - rejection(await submit(PUNCTUATED_WIDGET_EDIT), 'unknown_widget') - ) + answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) expect(toastDescriptions()).toEqual([ - expect.stringContaining('Rejected: node 3 seed (opaque_widgets)'), - expect.stringContaining( - "Rejected: node 3 Width/Height & 'x' (unknown_widget)" - ) + expect.stringContaining(WIDGET_REJECTION_TEXT) ]) }) - it('still speaks for a rejection of another kind inside that window', async () => { + it('replaces the notice with the latest rejection instead of stacking it', async () => { const { submit } = mountFollower() + render(Toaster, { global: { plugins: [i18n] } }) answerWithOpsResult(rejection(await submit(WIDGET_EDIT), 'opaque_widgets')) answerWithOpsResult( @@ -448,9 +450,13 @@ describe('a human edit the doc host rejects', () => { ) expect(toastDescriptions()).toEqual([ - expect.stringContaining(WIDGET_REJECTION_TEXT), expect.stringContaining(GENERIC_REJECTION_TEXT) ]) + await nextTick() + expect(screen.getAllByTestId('toast')).toHaveLength(1) + const announcement = screen.getByRole('alert') + expect(announcement).toHaveTextContent(GENERIC_REJECTION_TEXT) + expect(announcement).not.toHaveTextContent(WIDGET_REJECTION_TEXT) }) it('stays silent when the host applies the edit', async () => { @@ -468,10 +474,6 @@ describe('a human edit the doc host rejects', () => { expect(useToast().toasts).toEqual([]) }) - // The host refuses a whole batch with a top-level `code` and no `failed` - // entry. Read only from `failed`, every one of these collapses to - // 'unspecified' — and because the telemetry dedupe keys on that, the first - // would suppress the report for every later kind. it('classifies a batch-level refusal that carries no failed entry', async () => { const { submit } = mountFollower() await submit(WIDGET_EDIT) @@ -484,53 +486,56 @@ describe('a human edit the doc host rejects', () => { ) ]) expect(reportedCodes()).toEqual(['catalog_mismatch']) + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'error_applying_agent_graph_edit', + tags: expect.objectContaining({ outcome: 'host_rejected' }) + }) + ) }) - // Asserted on telemetry rather than toasts: every generic notice renders the - // same string, so they share one throttle key and the second is swallowed on - // screen even though it is a different failure. - it('reports each distinct batch-level code once', async () => { + it('reports each rejected batch once even when rejection codes repeat', async () => { const { submit } = mountFollower() - // One pending batch per answer: the notifier runs when a batch SETTLES, so - // a result with nothing outstanding settles nothing and reports nothing. await submit(WIDGET_EDIT) answerWithOpsResult(batchRefusal('overloaded')) + answerWithOpsResult(batchRefusal('overloaded')) await submit(WIDGET_EDIT) answerWithOpsResult(batchRefusal('overloaded')) + answerWithOpsResult(batchRefusal('overloaded')) await submit(WIDGET_EDIT) answerWithOpsResult(batchRefusal('catalog_mismatch')) - expect(reportedCodes()).toEqual(['overloaded', 'catalog_mismatch']) + expect(reportedCodes()).toEqual([ + 'overloaded', + 'overloaded', + 'catalog_mismatch' + ]) }) - // Schema §4 aborts the remainder of a batch, so the prefix is still applied. - // Both partial branches are user-facing copy, so both are exercised: a wrong - // key on either would tell the user nothing was saved after a batch that - // saved a prefix. The rejected op's KIND picks the branch, so the two rows - // differ only in which op the host rejects. it.for([ { branch: 'widget', ops: [NODE_ADD, WIDGET_EDIT], code: 'opaque_widgets', - expected: WIDGET_REJECTION_TEXT + expected: WIDGET_REJECTION_TEXT, + errorType: 'error_applying_agent_widget_edit' }, { branch: 'generic', ops: [WIDGET_EDIT, NODE_ADD], code: 'uncatalogued_widget_write', - expected: GENERIC_REJECTION_TEXT + expected: GENERIC_REJECTION_TEXT, + errorType: 'error_applying_agent_graph_edit' } ])( 'does not claim nothing was saved when a batch applied a prefix ($branch)', - async ({ ops, code, expected }) => { + async ({ ops, code, expected, errorType }) => { const { submitBatch } = mountFollower() - const frames = await submitBatch(ops) - // EXACTLY one frame carrying BOTH ops: across two frames the host could - // never answer with one applied id and one failure index. + const frames = await submitBatch([NODE_DELETE, ...ops, NODE_DELETE]) expect(frames).toHaveLength(1) - const [[appliedId, rejectedId]] = frames + const [[skippedId, appliedId, rejectedId, tailId]] = frames expect(rejectedId).toBeDefined() answerWithOpsResult({ @@ -538,17 +543,23 @@ describe('a human edit the doc host rejects', () => { workflow_id: WORKFLOW_ID, ok: false, applied: [appliedId], - skipped: [], + skipped: [skippedId], failed: { - index: 1, + index: 2, op_id: rejectedId, code, message: 'node is absent from the pinned catalog' } }) - // Still carries the canonical substring the e2e spec filters on, so the - // partial variant cannot break that contract. + expect(reportError).toHaveBeenCalledExactlyOnceWith( + expect.any(Error), + expect.objectContaining({ + errorType, + level: 'warning', + context: expect.objectContaining({ opIds: [rejectedId, tailId] }) + }) + ) expect(toastDescriptions()).toEqual([expect.stringContaining(expected)]) expect(toastDescriptions()[0]).toContain('some earlier edits were') } diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts index f5fcfb0ebe2..ee72da60204 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.test.ts @@ -2660,10 +2660,11 @@ describe('useAgentCrdtFollower', () => { removed_count: 0, apply_failure_count: 0 }) - expect(telemetryState.reportError).toHaveBeenCalledWith( + expect(telemetryState.reportError).toHaveBeenCalledExactlyOnceWith( expect.any(Error), expect.objectContaining({ - errorType: 'agent_crdt_human_ops_rejected', + errorType: 'error_applying_agent_graph_edit', + fingerprint: ['error_applying_agent_graph_edit'], context: expect.objectContaining({ opId: ops[1].op_id, code: 'unknown_node' diff --git a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts index d7af4e8a6f6..dc6d233d55a 100644 --- a/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts +++ b/src/workbench/extensions/agent/crdt/useAgentCrdtFollower.ts @@ -257,31 +257,6 @@ type AgentCrdtApplierDeps = Omit< 'getGraph' | 'holdsLocalWrite' > -function reportRejectedHumanOps( - workflowId: string | null, - ops: readonly Op[], - result: OpsResultView -): void { - const settled = new Set([...result.applied, ...result.skipped]) - const rejected = ops.filter((op) => !settled.has(op.op_id)) - const { failed } = result - reportError( - new Error( - `The doc host rejected ${rejected.length} local edit(s): ${failed?.message ?? 'no diagnostics'}` - ), - { - surface: 'agent', - errorType: 'agent_crdt_human_ops_rejected', - context: { - workflowId, - opId: failed?.op_id ?? rejected[0]?.op_id, - code: failed?.code, - rejectedOps: rejected.map((op) => op.op) - } - } - ) -} - export interface AgentCrdtFollowerOptions { transport: AgentDocFrameTransport userId?: () => string | null @@ -508,8 +483,6 @@ function startAgentCrdtFollower( outcome: Extract ) => { const workflowId = outcome.result.workflowId ?? bridge.subscribedWorkflowId - if (outcome.result.failed) - reportRejectedHumanOps(workflowId, outcome.ops, outcome.result) if (workflowId === null) return const applied = new Set(outcome.result.applied) @@ -529,8 +502,10 @@ function startAgentCrdtFollower( } const settleHumanOps = (outcome: BatchOutcome) => { - if (outcome.state === 'acknowledged') trackAcknowledgedDeletes(outcome) - recordDevEvent('human_ops_settled', outcome) + if (outcome.state === 'acknowledged') { + trackAcknowledgedDeletes(outcome) + recordDevEvent('human_ops_settled', outcome) + } projection.settleLocalWrites(outcome.ops) if (outcome.state === 'acknowledged' && !outcome.result.ok) revertRejectedOps(outcome) @@ -551,7 +526,9 @@ function startAgentCrdtFollower( // Send REALITY, not this composable's intent: the sender re-reads it before // every send and resend, so ops never reach a doc we are not subscribed to. workflowId: () => bridge.subscribedWorkflowId, - tab: tabId, + get tab() { + return tabId + }, actor: ownActor, baseVersion: () => bridge.lastSequence, onBatchSettled: settleHumanOps @@ -1143,7 +1120,6 @@ function startAgentCrdtFollower( () => bridge.removeEventListener('doc_subscribe_sent', onSubscribeSent), () => bridge.removeEventListener('doc_reseed_result', onReseedResult), () => sender.detach(), - () => rejectedOpNotifier.cancel(), () => pendingRejected.clear(), () => coalescer.detach(), () => {