From 926f8ace60bd53e9bb47a7131f13e0adbed99f6a Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 17 Sep 2026 20:19:58 -0700 Subject: [PATCH 1/4] test: recover deterministic oracle and replay review checks --- test/graph-invariant-oracle.test.ts | 63 ++++++++++++++++++++++ test/graph-invariant-oracle.ts | 7 +-- test/opid-payload-reuse.regression.test.ts | 5 +- test/pbt-convergence.test.ts | 27 +++++++++- test/reject-no-mutation.regression.test.ts | 61 ++++++++++++--------- test/types/invalid-states.negative.ts | 10 +++- 6 files changed, 141 insertions(+), 32 deletions(-) diff --git a/test/graph-invariant-oracle.test.ts b/test/graph-invariant-oracle.test.ts index 7b06d89b..5147e672 100644 --- a/test/graph-invariant-oracle.test.ts +++ b/test/graph-invariant-oracle.test.ts @@ -46,6 +46,43 @@ describe("checkGraphInvariants test oracle", () => { expect(checkGraphInvariants(doc).filter((item) => item.invariant === "I3")).toHaveLength(2); }); + it("reports a short link tuple as I3 without changing the doc", () => { + const doc = freshDoc(); + linksMap(doc).set("short", ["short", 1, 0, 2]); + const before = Y.encodeStateAsUpdate(doc); + + expect(checkGraphInvariants(doc).filter((item) => item.invariant === "I3")).toEqual([ + { + invariant: "I3", + path: 'links["short"]', + message: "link entry is not a tuple with node endpoints", + }, + ]); + expect(Y.encodeStateAsUpdate(doc)).toEqual(before); + }); + + it("reports negative source and destination slots as I4 without changing the doc", () => { + const doc = freshDoc(); + const links = linksMap(doc); + links.delete("10"); + links.set("negative", ["negative", 1, -1, 2, -2, "X"]); + const before = Y.encodeStateAsUpdate(doc); + + expect(checkGraphInvariants(doc).filter((item) => item.invariant === "I4")).toEqual([ + { + invariant: "I4", + path: 'links["negative"][2]', + message: "source output (1, -1) does not advertise link negative", + }, + { + invariant: "I4", + path: 'links["negative"][4]', + message: "destination input (2, -2) does not carry link negative", + }, + ]); + expect(Y.encodeStateAsUpdate(doc)).toEqual(before); + }); + it("reports links-to-slots disagreement", () => { const doc = freshDoc(); const source = nodesMap(doc).get("1")!; @@ -64,4 +101,30 @@ describe("checkGraphInvariants test oracle", () => { expect(checkGraphInvariants(doc).map((item) => item.invariant)).toContain("I5"); }); + + it("reports exact I5 diagnostics independently of link insertion order", () => { + const diagnosticsFor = (keys: string[]) => { + const doc = freshDoc(); + const links = linksMap(doc); + links.delete("10"); + for (const key of keys) links.set(key, [key, 1, 0, 2, 0, "X"]); + + return checkGraphInvariants(doc).filter((item) => item.invariant === "I5"); + }; + const expected = [ + { + invariant: "I5", + path: 'links["11"]', + message: "input register (2, 0) is also claimed by link 100", + }, + { + invariant: "I5", + path: 'links["2"]', + message: "input register (2, 0) is also claimed by link 100", + }, + ]; + + expect(diagnosticsFor(["100", "11", "2"])).toEqual(expected); + expect(diagnosticsFor(["2", "11", "100"])).toEqual(expected); + }); }); diff --git a/test/graph-invariant-oracle.ts b/test/graph-invariant-oracle.ts index ad5008dd..a5931624 100644 --- a/test/graph-invariant-oracle.ts +++ b/test/graph-invariant-oracle.ts @@ -88,11 +88,12 @@ export function checkGraphInvariants(doc: Y.Doc): GraphInvariantViolation[] { // endpoint-side references. I4 is not attempted when I3 already fails for // that endpoint, which keeps one missing node from producing duplicate noise. const inputClaims = new Map(); - links.forEach((rawTuple, linkMapKey) => { + const linkEntries = [...links.entries()].sort(([a], [b]) => a.localeCompare(b)); + for (const [linkMapKey, rawTuple] of linkEntries) { const path = `links[${JSON.stringify(linkMapKey)}]`; if (!Array.isArray(rawTuple) || rawTuple.length < 5) { violation(violations, "I3", path, "link entry is not a tuple with node endpoints"); - return; + continue; } const tuple = rawTuple as unknown[]; @@ -173,7 +174,7 @@ export function checkGraphInvariants(doc: Y.Doc): GraphInvariantViolation[] { ); } } - }); + } return violations.sort((a, b) => { const invariant = INVARIANT_ORDER[a.invariant] - INVARIANT_ORDER[b.invariant]; diff --git a/test/opid-payload-reuse.regression.test.ts b/test/opid-payload-reuse.regression.test.ts index 733d62a8..17a3189e 100644 --- a/test/opid-payload-reuse.regression.test.ts +++ b/test/opid-payload-reuse.regression.test.ts @@ -2,7 +2,6 @@ import * as Y from "yjs"; import { describe, expect, it } from "vitest"; import { applyOps, mint, project, type SetWidgetOp } from "../src/index.js"; import { canonicalOp } from "../src/applier.js"; -import { sha256Hex } from "../src/digest.js"; import { appliedMap } from "../src/doc.js"; import { loadCatalog, loadLwwVectors } from "./helpers.js"; @@ -69,7 +68,9 @@ describe("regression: op_id reuse with a changed payload (#12)", () => { '"op":"set_widget","op_id":"same0000000000000000000000000000",' + '"stamp":[5,"human:a"],"value":25,"widget":"steps"}'; expect(canonicalOp(firstOp())).toBe(canonical); - expect(appliedMap(doc).get(firstOp().op_id)).toBe(sha256Hex(canonical)); + expect(appliedMap(doc).get(firstOp().op_id)).toBe( + "58cfa9fd689bf1126598dfa798a0bfaa41b45bad2a55edfdcc952afb221f5bd3", + ); // Bounded: 64 hex chars regardless of payload size (schema §4 amendment A8). expect(String(appliedMap(doc).get(firstOp().op_id))).toHaveLength(64); }); diff --git a/test/pbt-convergence.test.ts b/test/pbt-convergence.test.ts index 2518518f..92a9b378 100644 --- a/test/pbt-convergence.test.ts +++ b/test/pbt-convergence.test.ts @@ -20,6 +20,14 @@ interface Scenario { retryMask: boolean[]; } +interface Coverage { + runs: number; + deletes: number; + clears: number; + retries: number; + permutations: number; +} + const scenarioArb: fc.Arbitrary = fc.record({ actors: fc.integer({ min: 1, max: 4 }), nodes: fc.integer({ min: 2, max: 8 }), @@ -127,6 +135,14 @@ function withRetries(ops: Op[], mask: boolean[]): Op[] { return [...adjacent, ...delayed]; } +function assertNonVacuous(coverage: Coverage): void { + expect(coverage.runs).toBe(FC_OPTIONS.numRuns); + expect(coverage.deletes, "no generated operation stream included delete_node").toBeGreaterThan(0); + expect(coverage.clears, "no generated operation stream included clear").toBeGreaterThan(0); + expect(coverage.retries, "no stream passed to applyOps contained a duplicate retry").toBeGreaterThan(0); + expect(coverage.permutations, "no applied permutation differed from forward order").toBeGreaterThan(0); +} + function applyInBatches(doc: Y.Doc, ops: Op[], sizes: number[]): void { let cursor = 0; let batch = 0; @@ -146,22 +162,31 @@ function fork(snapshot: Uint8Array): Y.Doc { describe("property-based convergence and idempotency", () => { it("converges across actor counts, causal permutations, batches, and duplicate retries", () => { + const coverage: Coverage = { runs: 0, deletes: 0, clears: 0, retries: 0, permutations: 0 }; fc.assert( fc.property(scenarioArb, (scenario) => { const causalPhases = phases(scenario); const forward = causalPhases.flat(); const permuted = ordered(causalPhases, scenario.orderKeys); + const retried = withRetries(permuted, scenario.retryMask); const snapshot = Y.encodeStateAsUpdate(mint({ nodes: [], links: [] }, catalog)); const a = fork(snapshot); const b = fork(snapshot); applyInBatches(a, forward, [forward.length || 1]); - applyInBatches(b, withRetries(permuted, scenario.retryMask), scenario.batchSizes); + applyInBatches(b, retried, scenario.batchSizes); + + coverage.runs += 1; + if (forward.some((op) => op.op === "delete_node")) coverage.deletes += 1; + if (forward.some((op) => op.op === "clear")) coverage.clears += 1; + if (new Set(retried.map((op) => op.op_id)).size < retried.length) coverage.retries += 1; + if (permuted.some((op, index) => op.op_id !== forward[index]?.op_id)) coverage.permutations += 1; expect(JSON.stringify(project(b, catalog))).toBe(JSON.stringify(project(a, catalog))); }), FC_OPTIONS, ); + assertNonVacuous(coverage); }); // reset_doc is a DEFERRED op (rejected until un-deferred by amendment), so it diff --git a/test/reject-no-mutation.regression.test.ts b/test/reject-no-mutation.regression.test.ts index 6f85d1dd..9d169518 100644 --- a/test/reject-no-mutation.regression.test.ts +++ b/test/reject-no-mutation.regression.test.ts @@ -1,14 +1,7 @@ import * as Y from "yjs"; import { describe, expect, it } from "vitest"; -import { - applyOps, - mint, - project, - type ConnectOp, - type Op, - type WidgetCatalog, - type WorkflowJSON, -} from "../src/index.js"; +import { applyOps, mint, project } from "../src/index.js"; +import type { ConnectOp, Op, WidgetCatalog, WorkflowJSON } from "../src/index.js"; import { appliedMap } from "../src/doc.js"; import { loadCatalog } from "./helpers.js"; import { checkGraphInvariants } from "./graph-invariant-oracle.js"; @@ -557,21 +550,41 @@ describe("regression: rejected connect ops leave document bytes unchanged (#10)" * the schema list to be updated, which is what the header sentence claims. */ it("§2.5 item 4 (source axis) still diverges — pinning the carve-out", () => { - const projections = [[ - { op: "connect", op_id: opId("cv4a"), actor: "human:x", base_version: 5, stamp: [5, "human:x"], - link_id: 9707, from_node: 300, from_slot: 5, to_node: 700, to_slot: 0, link_type: "IMAGE" }, - { op: "delete_node", op_id: opId("cv4d"), actor: "human:d", base_version: 4, stamp: [4, "human:d"], node_id: 300 }, - ], [ - { op: "delete_node", op_id: opId("cv4d"), actor: "human:d", base_version: 4, stamp: [4, "human:d"], node_id: 300 }, - { op: "connect", op_id: opId("cv4a"), actor: "human:x", base_version: 5, stamp: [5, "human:x"], - link_id: 9707, from_node: 300, from_slot: 5, to_node: 700, to_slot: 0, link_type: "IMAGE" }, - ]].map((order) => { - const doc = mint(convergenceWorkflow, catalog); - for (const op of order) applyOps(doc, [op as unknown as Op], catalog); - return JSON.stringify(project(doc, catalog)); + const bad = { + op: "connect", op_id: opId("cv4a"), actor: "human:x", base_version: 5, stamp: [5, "human:x"], + link_id: 9707, from_node: 300, from_slot: 5, to_node: 700, to_slot: 0, link_type: "IMAGE", + }; + const tail = { + op: "add_node", op_id: opId("cv4t"), actor: "human:x", base_version: 5, stamp: [5, "human:x"], + node_id: 904, class_type: "LoadImage", pos: [0, 0], + node: { id: 904, type: "LoadImage", inputs: [], outputs: [], widgets_values: [], pos: [0, 0] }, + }; + const del = { + op: "delete_node", op_id: opId("cv4d"), actor: "human:d", base_version: 4, + stamp: [4, "human:d"], node_id: 300, + }; + const seed = mint(convergenceWorkflow, catalog); + const snapshot = Y.encodeStateAsUpdate(seed); + const runs = [[[bad, tail], [del]], [[del], [bad, tail]]].map((batches) => { + const doc = new Y.Doc(); + Y.applyUpdate(doc, snapshot); + const outcomes = batches.flatMap((batch) => applyOps(doc, batch as unknown as Op[], catalog).outcomes); + return { outcomes, projection: project(doc, catalog) }; }); - // NOT equal — this is the documented residual, not a passing property. - expect(projections[0]).not.toEqual(projections[1]); + const sourcePresent = runs[0]!; + const sourceDeleted = runs[1]!; + + // Behavioral witnesses: while the source exists the state-dependent slot + // check rejects and aborts the tail; after source deletion the same connect + // is an accepted delete-wins no-op and the tail applies. + expect(sourcePresent.outcomes.map(({ outcome }) => outcome)).toEqual(["rejected", "rejected", "applied"]); + expect(sourcePresent.outcomes[0]).toMatchObject({ outcome: "rejected", reason: { code: "output_slot_missing" } }); + expect(sourcePresent.outcomes[1]).toMatchObject({ outcome: "rejected", reason: { code: "batch_aborted" } }); + expect(sourceDeleted.outcomes.map(({ outcome }) => outcome)).toEqual(["applied", "no-op", "applied"]); + expect(sourcePresent.projection.nodes.some((node) => node.id === 904)).toBe(false); + expect(sourceDeleted.projection.nodes.some((node) => node.id === 904)).toBe(true); + // NOT equal — abort-remainder exposes the documented residual. + expect(sourcePresent.projection).not.toEqual(sourceDeleted.projection); }); it("§2.5 item 5 (destination axis) still diverges — pinning the carve-out", () => { @@ -835,7 +848,7 @@ describe("regression: rejected connect ops leave document bytes unchanged (#10)" op: "set_widget", op_id: opId("uncloneable-sw"), actor: "human:z", base_version: 9, stamp: [9, "human:z"], node_id: 700, widget: "inputcount", value: (() => undefined) as unknown, - } as unknown as ConnectOp; + } as unknown as Op; expect(rejected(applyOps(doc, [op], catalog))).toMatchObject({ code: "malformed_op" }); expect(Buffer.from(Y.encodeStateAsUpdate(doc)).equals(before)).toBe(true); }); diff --git a/test/types/invalid-states.negative.ts b/test/types/invalid-states.negative.ts index 132b201a..3cde71b1 100644 --- a/test/types/invalid-states.negative.ts +++ b/test/types/invalid-states.negative.ts @@ -18,6 +18,7 @@ * the document; `test/invalid-op-states.test.ts` is the matching runtime * audit, including the states the wire still accepts. */ +import type * as Y from "yjs"; import type { AddNodeOp, ClearOp, @@ -33,6 +34,7 @@ import type { SetWidgetOp, TopLevelSetWidgetOp, WireOp, + applyOps, } from "../../src/index.js"; const env = { @@ -194,9 +196,10 @@ const reset: ResetDocOp = { // @ts-expect-error #17: `reset_doc` is deferred; it is a `WireOp`, not an `Op`. const resetAsOp: Op = reset; -declare function applyOpsSignature(ops: Op[]): void; +declare const applyOpsSignature: typeof applyOps; +declare const doc: Y.Doc; // @ts-expect-error #17: the applier cannot be handed an op it always refuses. -applyOpsSignature([reset]); +applyOpsSignature(doc, [reset]); const insertWorkflow: InsertWorkflowOp = { op: "insert_workflow", @@ -204,6 +207,9 @@ const insertWorkflow: InsertWorkflowOp = { workflow: { nodes: [], links: [], definitions: { subgraphs: [] } }, }; const insertWorkflowAsOp: Op = insertWorkflow; +// Positive control: the real public arity plus a valid op must compile, so the +// reset assertion cannot pass merely because every call has the wrong arity. +applyOpsSignature(doc, [insertWorkflow]); // The public vocabulary projection accepts every declared kind and no value // outside the wire union. This is an external-consumer check through index.ts. From aba5c8b21c5f350817382c356de8f3d66f80a371 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 17 Sep 2026 20:37:26 -0700 Subject: [PATCH 2/4] test: make graph diagnostics locale-independent Addresses https://github.com/Comfy-Org/comfy-multi-player/pull/212#discussion_r4043547841 --- test/graph-invariant-oracle.test.ts | 19 ++++++++++++------- test/graph-invariant-oracle.ts | 12 +++++++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/test/graph-invariant-oracle.test.ts b/test/graph-invariant-oracle.test.ts index 5147e672..8556afec 100644 --- a/test/graph-invariant-oracle.test.ts +++ b/test/graph-invariant-oracle.test.ts @@ -102,7 +102,7 @@ describe("checkGraphInvariants test oracle", () => { expect(checkGraphInvariants(doc).map((item) => item.invariant)).toContain("I5"); }); - it("reports exact I5 diagnostics independently of link insertion order", () => { + it("reports code-unit-ordered non-ASCII I5 diagnostics independently of link insertion order", () => { const diagnosticsFor = (keys: string[]) => { const doc = freshDoc(); const links = linksMap(doc); @@ -114,17 +114,22 @@ describe("checkGraphInvariants test oracle", () => { const expected = [ { invariant: "I5", - path: 'links["11"]', - message: "input register (2, 0) is also claimed by link 100", + path: 'links["z"]', + message: "input register (2, 0) is also claimed by link y", }, { invariant: "I5", - path: 'links["2"]', - message: "input register (2, 0) is also claimed by link 100", + path: 'links["ä"]', + message: "input register (2, 0) is also claimed by link y", + }, + { + invariant: "I5", + path: 'links["é"]', + message: "input register (2, 0) is also claimed by link y", }, ]; - expect(diagnosticsFor(["100", "11", "2"])).toEqual(expected); - expect(diagnosticsFor(["2", "11", "100"])).toEqual(expected); + expect(diagnosticsFor(["y", "z", "ä", "é"])).toEqual(expected); + expect(diagnosticsFor(["é", "ä", "z", "y"])).toEqual(expected); }); }); diff --git a/test/graph-invariant-oracle.ts b/test/graph-invariant-oracle.ts index a5931624..6c429654 100644 --- a/test/graph-invariant-oracle.ts +++ b/test/graph-invariant-oracle.ts @@ -25,6 +25,12 @@ function sameLink(a: unknown, b: unknown): boolean { return a === b || (a != null && b != null && linkKey(a) === linkKey(b)); } +function compareCodeUnits(a: string, b: string): number { + if (a < b) return -1; + if (a > b) return 1; + return 0; +} + function violation( out: GraphInvariantViolation[], invariant: GraphInvariant, @@ -88,7 +94,7 @@ export function checkGraphInvariants(doc: Y.Doc): GraphInvariantViolation[] { // endpoint-side references. I4 is not attempted when I3 already fails for // that endpoint, which keeps one missing node from producing duplicate noise. const inputClaims = new Map(); - const linkEntries = [...links.entries()].sort(([a], [b]) => a.localeCompare(b)); + const linkEntries = [...links.entries()].sort(([a], [b]) => compareCodeUnits(a, b)); for (const [linkMapKey, rawTuple] of linkEntries) { const path = `links[${JSON.stringify(linkMapKey)}]`; if (!Array.isArray(rawTuple) || rawTuple.length < 5) { @@ -179,7 +185,7 @@ export function checkGraphInvariants(doc: Y.Doc): GraphInvariantViolation[] { return violations.sort((a, b) => { const invariant = INVARIANT_ORDER[a.invariant] - INVARIANT_ORDER[b.invariant]; if (invariant !== 0) return invariant; - const path = a.path.localeCompare(b.path); - return path !== 0 ? path : a.message.localeCompare(b.message); + const path = compareCodeUnits(a.path, b.path); + return path !== 0 ? path : compareCodeUnits(a.message, b.message); }); } From f6378dc5597457cb1a111400ae4eae12d8168ba6 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 17 Sep 2026 20:37:27 -0700 Subject: [PATCH 3/4] test: require every generated retry to be a no-op Addresses https://github.com/Comfy-Org/comfy-multi-player/pull/212#discussion_r4043547845 --- test/pbt-convergence.test.ts | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/pbt-convergence.test.ts b/test/pbt-convergence.test.ts index 92a9b378..5fb943e8 100644 --- a/test/pbt-convergence.test.ts +++ b/test/pbt-convergence.test.ts @@ -143,15 +143,31 @@ function assertNonVacuous(coverage: Coverage): void { expect(coverage.permutations, "no applied permutation differed from forward order").toBeGreaterThan(0); } -function applyInBatches(doc: Y.Doc, ops: Op[], sizes: number[]): void { +function applyInBatches(doc: Y.Doc, ops: Op[], sizes: number[]): ReturnType["outcomes"] { let cursor = 0; let batch = 0; + const outcomes: ReturnType["outcomes"] = []; while (cursor < ops.length) { const size = sizes[batch++ % sizes.length]!; const result = applyOps(doc, ops.slice(cursor, cursor + size), catalog); expect(result.outcomes.find((outcome) => outcome.outcome === "rejected")).toBeUndefined(); + outcomes.push(...result.outcomes); cursor += size; } + return outcomes; +} + +function assertRetriesAreNoOps(ops: Op[], outcomes: ReturnType["outcomes"]): void { + expect(outcomes).toHaveLength(ops.length); + const seen = new Set(); + for (const [index, op] of ops.entries()) { + const outcome = outcomes[index]!; + expect(outcome.op_id).toBe(op.op_id); + if (seen.has(op.op_id)) { + expect(outcome.outcome, `retry occurrence ${index} for ${op.op} ${op.op_id} was not a no-op`).toBe("no-op"); + } + seen.add(op.op_id); + } } function fork(snapshot: Uint8Array): Y.Doc { @@ -174,7 +190,8 @@ describe("property-based convergence and idempotency", () => { const b = fork(snapshot); applyInBatches(a, forward, [forward.length || 1]); - applyInBatches(b, retried, scenario.batchSizes); + const retryOutcomes = applyInBatches(b, retried, scenario.batchSizes); + assertRetriesAreNoOps(retried, retryOutcomes); coverage.runs += 1; if (forward.some((op) => op.op === "delete_node")) coverage.deletes += 1; From cd8c97b998653d442e844f11710c8d6bf1d2de10 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 17 Sep 2026 20:37:28 -0700 Subject: [PATCH 4/4] test: assert rejected batches preserve bookkeeping Addresses https://github.com/Comfy-Org/comfy-multi-player/pull/212#discussion_r4043547847 --- test/reject-no-mutation.regression.test.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/reject-no-mutation.regression.test.ts b/test/reject-no-mutation.regression.test.ts index 9d169518..f8109dcb 100644 --- a/test/reject-no-mutation.regression.test.ts +++ b/test/reject-no-mutation.regression.test.ts @@ -565,10 +565,21 @@ describe("regression: rejected connect ops leave document bytes unchanged (#10)" }; const seed = mint(convergenceWorkflow, catalog); const snapshot = Y.encodeStateAsUpdate(seed); - const runs = [[[bad, tail], [del]], [[del], [bad, tail]]].map((batches) => { + const runs = [[[bad, tail], [del]], [[del], [bad, tail]]].map((batches, runIndex) => { const doc = new Y.Doc(); Y.applyUpdate(doc, snapshot); - const outcomes = batches.flatMap((batch) => applyOps(doc, batch as unknown as Op[], catalog).outcomes); + const outcomes = batches.flatMap((batch, batchIndex) => { + const beforeRejectedBatch = runIndex === 0 && batchIndex === 0 + ? Buffer.from(Y.encodeStateAsUpdate(doc)) + : null; + const batchOutcomes = applyOps(doc, batch as unknown as Op[], catalog).outcomes; + if (beforeRejectedBatch !== null) { + expect(Buffer.from(Y.encodeStateAsUpdate(doc)).equals(beforeRejectedBatch)).toBe(true); + expect(appliedMap(doc).has(bad.op_id)).toBe(false); + expect(appliedMap(doc).has(tail.op_id)).toBe(false); + } + return batchOutcomes; + }); return { outcomes, projection: project(doc, catalog) }; }); const sourcePresent = runs[0]!;