diff --git a/src/workbench/extensions/agent/crdt/opSender.test.ts b/src/workbench/extensions/agent/crdt/opSender.test.ts index 471e5b07afb..f9fe53515de 100644 --- a/src/workbench/extensions/agent/crdt/opSender.test.ts +++ b/src/workbench/extensions/agent/crdt/opSender.test.ts @@ -122,6 +122,7 @@ describe('createOpSender', () => { let transportUp: boolean let transportThrows: boolean let boundWorkflow: string | null + let observedSequence: number let sender: ReturnType const unsubscribe = vi.fn(() => { resultListener = null @@ -144,6 +145,7 @@ describe('createOpSender', () => { transportUp = true transportThrows = false boundWorkflow = WORKFLOW + observedSequence = 41 sender = createOpSender({ sendOps: (workflowId, tab, ops) => { if (transportThrows) throw new Error('frame serialization failed') @@ -158,7 +160,7 @@ describe('createOpSender', () => { workflowId: () => boundWorkflow, tab: TAB, actor: () => ACTOR, - baseVersion: () => 41, + baseVersion: () => observedSequence, onBatchSettled: (outcome) => settled.push(outcome) }) }) @@ -198,6 +200,110 @@ describe('createOpSender', () => { expect(sent[1].ops[0].base_version).toBe(41) }) + it('a document reset restarts only the reset document, not every workflow visited', () => { + sender.admit([addNode(1)]) + sender.admit([addNode(2)]) + sender.flush() + boundWorkflow = 'wf-2' + sender.admit([addNode(3)]) + sender.abortAll() + sender.admit([addNode(4)]) + // Back on wf-1 before its re-subscribe is acknowledged. + boundWorkflow = WORKFLOW + observedSequence = 0 + sender.admit([addNode(5)]) + + expect( + sender.pendingOps().map(({ workflowId, ops }) => ({ + workflowId, + versions: ops.map((op) => op.base_version) + })) + ).toEqual([ + { workflowId: 'wf-2', versions: [41] }, + { workflowId: WORKFLOW, versions: [43] } + ]) + }) + + it('an admission taken while unbound leaves the bound workflow its own counter', () => { + sender.admit([addNode(1)]) + boundWorkflow = null + + sender.admit([addNode(2)]) + + boundWorkflow = WORKFLOW + sender.admit([addNode(3)]) + sender.flush() + + expect( + sender.pendingOps().flatMap(({ ops }) => ops.map((op) => op.base_version)) + ).toEqual([41, 42]) + expect(settled.map(summarizeSettlement)).toEqual([ + { state: 'undeliverable', nodeIds: [2] } + ]) + }) + + it('keeps one Lamport counter per workflow across a switch away and back', () => { + sender.admit([addNode(1)]) + boundWorkflow = 'wf-2' + sender.admit([addNode(2)]) + boundWorkflow = WORKFLOW + sender.admit([addNode(3)]) + sender.flush() + + const minted = sender.pendingOps().map(({ workflowId, ops }) => ({ + workflowId, + versions: ops.map((op) => op.base_version) + })) + expect(minted).toEqual([ + { workflowId: WORKFLOW, versions: [41] }, + { workflowId: 'wf-2', versions: [41] }, + { workflowId: WORKFLOW, versions: [42] } + ]) + }) + + it('keeps a workflow its counter while a re-subscribe has not been acknowledged', () => { + sender.admit([addNode(1)]) + sender.admit([addNode(2)]) + boundWorkflow = 'wf-2' + observedSequence = 0 + sender.admit([addNode(3)]) + boundWorkflow = WORKFLOW + sender.admit([addNode(4)]) + sender.flush() + + expect( + sender.pendingOps().map(({ workflowId, ops }) => ({ + workflowId, + versions: ops.map((op) => op.base_version) + })) + ).toEqual([ + { workflowId: WORKFLOW, versions: [41, 42] }, + { workflowId: 'wf-2', versions: [0] }, + { workflowId: WORKFLOW, versions: [43] } + ]) + }) + + it('mints past every stamp in a multi-operation admission', () => { + sender.admit([addNode(1), addNode(2), addNode(3)]) + sender.admit([addNode(4)]) + sender.flush() + + expect( + sender.pendingOps().flatMap(({ ops }) => ops.map((op) => op.base_version)) + ).toEqual([41, 42, 43, 44]) + }) + + it('mints from the observed sequence once the document advances past this actor', () => { + sender.admit([addNode(1)]) + observedSequence = 90 + sender.admit([addNode(2)]) + sender.flush() + + expect( + sender.pendingOps().flatMap(({ ops }) => ops.map((op) => op.base_version)) + ).toEqual([41, 90]) + }) + it('serializes batches: the next sends only after the result settles the first', () => { sender.enqueue([addNode(1)]) sender.enqueue([addNode(2)]) @@ -1375,6 +1481,18 @@ describe('createOpSender', () => { expect(localSettled.map(summarizeSettlement)).toEqual([ { state: 'undeliverable', nodeIds: [1, 2] } ]) + expect(reportError).toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'failure_chunking_agent_op_sender_abort' + }) + ) + expect(reportError).not.toHaveBeenCalledWith( + expect.any(Error), + expect.objectContaining({ + errorType: 'failure_chunking_agent_op_sender' + }) + ) expect(localSender.pending()).toBe(0) localSender.detach() }) @@ -1429,13 +1547,17 @@ describe('createOpSender', () => { it('resumes a pump requested during workflow-change sealing', () => { const localSent: Array<{ workflowId: string; ops: Op[] }> = [] const localSettled: BatchOutcome[] = [] + let localResultListener!: (result: OpsResultView) => void let workflow = 'wf-old' const localSender = createOpSender({ sendOps: (workflowId, _tab, ops) => { localSent.push({ workflowId, ops }) return true }, - onOpsResult: () => vi.fn(), + onOpsResult: (listener) => { + localResultListener = listener + return vi.fn() + }, workflowId: () => workflow, tab: TAB, actor: () => ACTOR, @@ -1459,13 +1581,45 @@ describe('createOpSender', () => { localSender.admit([addNode(2)]) - expect(localSent).toHaveLength(1) - expect(localSent[0].workflowId).toBe('wf-new') - expect(nodeIdsOf(localSent[0].ops)).toEqual([3]) - expect(localSettled.map(summarizeSettlement)).toContainEqual({ - state: 'undeliverable', - nodeIds: [1] - }) + expect( + localSent.map(({ workflowId, ops }) => ({ + workflowId, + nodeIds: nodeIdsOf(ops) + })) + ).toEqual([{ workflowId: 'wf-new', nodeIds: [2] }]) + expect( + localSender.pendingOps().map(({ workflowId, ops }) => ({ + workflowId, + nodeIds: nodeIdsOf(ops), + versions: ops.map((op) => op.base_version) + })) + ).toEqual([ + { workflowId: 'wf-new', nodeIds: [2], versions: [41] }, + { workflowId: 'wf-old', nodeIds: [1], versions: [41] }, + { workflowId: 'wf-new', nodeIds: [3], versions: [42] } + ]) + const deliveredIds = localSent[0].ops.map((op) => op.op_id) + + localResultListener({ ok: true, applied: deliveredIds, skipped: [] }) + + expect( + localSent.map(({ workflowId, ops }) => ({ + workflowId, + nodeIds: nodeIdsOf(ops), + opIds: ops.map((op) => op.op_id) + })) + ).toEqual([ + { workflowId: 'wf-new', nodeIds: [2], opIds: deliveredIds }, + { + workflowId: 'wf-new', + nodeIds: [3], + opIds: [expect.stringMatching(/^[0-9a-f]{32}$/)] + } + ]) + expect(localSettled.map(summarizeSettlement)).toEqual([ + { state: 'acknowledged', nodeIds: [2] }, + { state: 'undeliverable', nodeIds: [1] } + ]) localSender.detach() }) diff --git a/src/workbench/extensions/agent/crdt/opSender.ts b/src/workbench/extensions/agent/crdt/opSender.ts index c945edbf166..0b13256fe26 100644 --- a/src/workbench/extensions/agent/crdt/opSender.ts +++ b/src/workbench/extensions/agent/crdt/opSender.ts @@ -75,12 +75,22 @@ export type BatchOutcome = | { state: 'unconfirmed'; ops: Op[] } | { state: 'undeliverable'; ops: Op[] } +interface AdmissionGroup { + workflowId: string + ops: Op[] +} + export interface OpSender { enqueue(operations: GraphOperation[]): void /** * Mint and target-pin operations into the open admission group without * starting transport delivery. Consecutive admissions for one workflow * share the group until `flush()` seals it. + * + * If custom serialization re-enters and closes the open group, these ops are + * sealed at the queue position they were admitted at, keeping their + * `op_id`s, and delivery waits until they are placed. Mint order is + * preserved per document, not across documents. */ admit(operations: GraphOperation[]): void /** Seal the open admission group into wire batches and start delivery. */ @@ -88,7 +98,7 @@ export interface OpSender { /** Unsettled batch count for observability; 0 = drained. */ pending(): number /** Every unsettled batch, in-flight first, each addressed to its mint-time workflow. */ - pendingOps(): ReadonlyArray<{ workflowId: string; ops: Op[] }> + pendingOps(): ReadonlyArray /** * The bound workflow's tab went inactive: the subscription is paused, not * lost. Until `resume()`, a batch reaching `transmit()` is parked instead @@ -144,16 +154,21 @@ interface InFlight { } export function createOpSender(deps: OpSenderDeps): OpSender { - const queue: Array<{ workflowId: string; ops: Op[] }> = [] + const queue: AdmissionGroup[] = [] let queueHead = 0 - let open: { workflowId: string; ops: Op[] } | null = null + let open: AdmissionGroup | null = null let inFlight: InFlight | null = null - let lastMintedVersion = -1 - let lastMintedWorkflowId: string | null = null + // One Lamport cursor per workflow, cleared only by that doc's reset: the + // observed sequence reads 0 between a subscribe and its ack, so only the + // cursor keeps a re-bound workflow's stamps past ones this actor used there. + const lastMintedVersions = new Map() let detached = false let suspended = false let pumping = false - let sealing = 0 + // Delivery deferral depth: a seal or an admission in progress parks the pump + // until it unwinds, so work a reentrant serialization callback enqueues + // cannot reach the wire ahead of the earlier-minted ops still being placed. + let pumpDeferrals = 0 let pumpRequested = false let stateEpoch = 0 let abortGeneration = 0 @@ -179,6 +194,23 @@ export function createOpSender(deps: OpSenderDeps): OpSender { >() const MAX_CHUNK_FAILURE_REPORTS = 3 + function mintBaseVersion(workflowId: string | null): number { + const observed = deps.baseVersion() + if (workflowId === null) return observed + const lastMinted = lastMintedVersions.get(workflowId) + return lastMinted === undefined + ? observed + : Math.max(observed, lastMinted + 1) + } + + function releasePumpDeferral(): void { + pumpDeferrals-- + if (pumpDeferrals === 0 && pumpRequested) { + pumpRequested = false + pump() + } + } + function retire(batch: InFlight, answered: number): void { const outstanding = batch.sends - answered if (outstanding <= 0) return @@ -362,7 +394,7 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } } - function dequeue(): { workflowId: string; ops: Op[] } | undefined { + function dequeue(): AdmissionGroup | undefined { if (queueHead >= queue.length) return const queued = queue[queueHead] queueHead++ @@ -374,7 +406,7 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } function pump(): void { - if (sealing > 0) { + if (pumpDeferrals > 0) { pumpRequested = true return } @@ -423,7 +455,8 @@ export function createOpSender(deps: OpSenderDeps): OpSender { minted: Op[], admissionTarget: string, admissionEpoch: number, - admissionAbortGeneration: number + admissionAbortGeneration: number, + admissionInsertionIndex: number ): boolean { if (stateEpoch === admissionEpoch) return false if (detached) { @@ -442,46 +475,52 @@ export function createOpSender(deps: OpSenderDeps): OpSender { stateEpoch++ return true } - guardedSettlementNotifier('failure_settling_agent_op_sender')({ - state: 'undeliverable', - ops: minted - }) + // The nested call closed or retargeted the open group: seal these ops at + // their admitted position, ahead of what the nested call enqueued. + sealGroup( + { workflowId: admissionTarget, ops: minted }, + admissionInsertionIndex + ) + stateEpoch++ return true } function admit(operations: GraphOperation[]): void { if (operations.length === 0) return const workflowId = deps.workflowId() - if (workflowId !== lastMintedWorkflowId) { - lastMintedVersion = -1 - lastMintedWorkflowId = workflowId - } - const baseVersion = Math.max(deps.baseVersion(), lastMintedVersion + 1) + const baseVersion = mintBaseVersion(workflowId) const actor = deps.actor() const minted = operations.flatMap((operation, index) => mintWireOps([operation], { actor, baseVersion: baseVersion + index }) ) - lastMintedVersion = baseVersion + minted.length - 1 const admissionTarget = admissionTargetOrSettle(minted, workflowId) if (admissionTarget === null) return - const admissionEpoch = stateEpoch - const admissionAbortGeneration = abortGeneration - // seal() can synchronously re-enter the sender through its settlement - // callback. Do not resurrect an aborted admission or append old-workflow - // ops to state installed by a nested admit(). - if (open?.workflowId !== admissionTarget) seal() - if ( - restoreReentrantAdmission( - minted, - admissionTarget, - admissionEpoch, - admissionAbortGeneration + lastMintedVersions.set(admissionTarget, baseVersion + minted.length - 1) + pumpDeferrals++ + try { + const admissionEpoch = stateEpoch + const admissionAbortGeneration = abortGeneration + const admissionInsertionIndex = queue.length + // seal() can synchronously re-enter the sender through its settlement + // callback. Do not resurrect an aborted admission or append old-workflow + // ops to state installed by a nested admit(). + if (open?.workflowId !== admissionTarget) seal() + if ( + restoreReentrantAdmission( + minted, + admissionTarget, + admissionEpoch, + admissionAbortGeneration, + admissionInsertionIndex + ) ) - ) - return - if (open) for (const op of minted) open.ops.push(op) - else open = { workflowId: admissionTarget, ops: minted } - stateEpoch++ + return + if (open) for (const op of minted) open.ops.push(op) + else open = { workflowId: admissionTarget, ops: minted } + stateEpoch++ + } finally { + releasePumpDeferral() + } } function sealInterruption( @@ -597,13 +636,16 @@ export function createOpSender(deps: OpSenderDeps): OpSender { return ops.length } - function seal(): boolean { - if (!open) return detached - const { workflowId, ops } = open + /** + * Chunk one admission group into wire batches at `insertionIndex`, the queue + * position the group held when it was admitted. + */ + function sealGroup( + { workflowId, ops }: AdmissionGroup, + insertionIndex: number + ): void { const sealAbortGeneration = abortGeneration - const insertionIndex = queue.length - open = null - sealing++ + pumpDeferrals++ try { const chunks = chunkWireOps(ops) const interrupted = sealInterruption(sealAbortGeneration) @@ -612,12 +654,16 @@ export function createOpSender(deps: OpSenderDeps): OpSender { } catch (cause) { recoverSeal(cause, workflowId, ops, sealAbortGeneration, insertionIndex) } finally { - sealing-- - if (sealing === 0 && pumpRequested) { - pumpRequested = false - pump() - } + releasePumpDeferral() } + } + + function seal(): boolean { + if (!open) return detached + const group = open + const insertionIndex = queue.length + open = null + sealGroup(group, insertionIndex) return detached } @@ -707,8 +753,10 @@ export function createOpSender(deps: OpSenderDeps): OpSender { abortAll() { stateEpoch++ abortGeneration++ - lastMintedVersion = -1 - lastMintedWorkflowId = null + // A doc_reset replaces only the bound document (the follower guards on + // isCurrentWorkflow), so only its cursor restarts. + const resetWorkflowId = deps.workflowId() + if (resetWorkflowId !== null) lastMintedVersions.delete(resetWorkflowId) drainOutstanding( guardedSettlementNotifier('failure_settling_agent_op_sender_abort'), 'failure_chunking_agent_op_sender_abort'