diff --git a/CHANGELOG.md b/CHANGELOG.md index 425f5b62..233185b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to `@comfyorg/comfy-multi-player` are documented in this fil The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this package uses semantic versioning. +## Unreleased + +### Added + +- Added the standalone `insert_workflow` op and `remapWorkflowIds` helper for + deterministic, collision-safe workflow-template insertion, including + deterministic whole-tree id remapping keyed by the operation envelope id. + ## 0.2.0 - 2026-08-30 ### Changed diff --git a/README.md b/README.md index 541df32c..23e4fd24 100644 --- a/README.md +++ b/README.md @@ -342,7 +342,7 @@ Every op carries the same envelope, minted by its creator before dispatch: } ``` -Six kinds, frozen: +Seven kinds, frozen: | Kind | Payload beyond the envelope | Batchable (authoring) | |---|---|---| @@ -352,6 +352,7 @@ Six kinds, frozen: | `set_widget` | `node_id`, `widget` (name, never index), `value`, optional `old`; an interior write adds `path` AND `inner_widget` together (`InteriorSetWidgetOp`); a promoted HOST write adds `promoted: {value_index, instance_path, host_widgets_values}` instead — a positional write into a subgraph instance's opaque array (schema Amendment A15) | yes | | `delete_node` | `node_id`, `removed_links` | yes | | `clear` | `removed_nodes` | no | +| `insert_workflow` | `workflow` containing top-level `nodes`, `links`, and optional `definitions.subgraphs`; ids must already be remapped collision-free by the minter | no | | `reset_doc` | see [open questions](docs/api-contract-proposal.md) — currently rejected `op_deferred` by this package | no | `FROZEN_OPS`, `DEFERRED_OPS`, and `BATCHABLE_OPS` are exported so you can check @@ -362,7 +363,7 @@ Compile-time assertions in `src/types.ts` pin that `FROZEN_OPS` is exactly is exactly `WireOp["op"]`, and that `BATCHABLE_OPS ⊆ FROZEN_OPS`, so the lists and the unions cannot drift apart silently. -**`Op` vs `WireOp`.** `Op` is what `applyOps` implements — the six kinds it +**`Op` vs `WireOp`.** `Op` is what `applyOps` implements — the seven kinds it can actually apply. `WireOp` is `Op` plus the deferred kinds a conforming peer may legally put on the wire, and it is what `ApplyFailure.op` and the stamp helpers take: a rejected `reset_doc` really does land in `failed.op`, so typing @@ -417,7 +418,7 @@ implementation to replay it with no failures. If you are building a submission surface in front of the applier, that admission layer is where `BATCHABLE_OPS` belongs. `test/batch-policy.test.ts` pins all of this. -The normative definition of the op envelope and the six kinds is +The normative definition of the original op envelope and six kinds is `docs/op-vocabulary-v1.md` in [comfy-cli](https://github.com/Comfy-Org/comfy-cli), which mints these ops on the agent side. The `Op` types here mirror those minted shapes field for field; diff --git a/docs/INVARIANTS.md b/docs/INVARIANTS.md index 567337d0..616b0551 100644 --- a/docs/INVARIANTS.md +++ b/docs/INVARIANTS.md @@ -35,12 +35,12 @@ This is the machine-addressable review log for the package. IDs are stable; do n **Amendment A9 closes three residual write-order holes:** values that clone but cannot be stored by Yjs are refused before `set_widget`, grow, and `add_node` writes; `connect.link_id` must satisfy both Y.Map-value and Y.Array-item domains; and non-iterable `delete_node.removed_links` is refused before the A7 node-presence gate or deletion. **Amendment A10 closes the reference-cycle hole and corrects the shallow write predicate from storability to encodability:** `applyOps` rejects cycles at A8's whole-op depth gate, while the write-site cycle guard independently protects `mint`; top-level `Date` values are refused at the write gate, and `mint` refuses out-of-int64 `BigInt` values because Yjs accepts but does not faithfully encode them. **Amendment A13 classifies every BigInt in an op envelope as `malformed_op` during A8's op-only canonicalization precondition, names its payload path, and therefore rejects before any document write or `op_id` record.** **Amendment A14 shape-validates `connect.link_type` as a string in the op-only precondition, before destination delete-wins or any write; arbitrary and unknown strings remain legal because this is not catalogue membership validation.** **Amendment A11 bounds the remaining breadth and size attack surface:** batches are capped before processing, while A8's whole-envelope canonicalization now caps collection entries and an approximate cost covering strings, keys, binary bytes, containers, and leaves. -The *rejection* half — "a rejected op leaves the doc untouched", stated in the `applyOps` module docstring in `src/applier.ts` and repeated as contract D4 — is swept table-driven in `test/ka4-rejection-byte-identity.test.ts`, one row per reachable rejection code across `add_node`, `set_widget`, `connect`, `connect`+grow, `delete_node`, `clear`, `reset_doc` and the envelope gate, asserting byte-identical `encodeStateAsUpdate` *and* that the `op_id` was not consumed. Completeness is enforced rather than asserted: one test compares the covered codes against `ALL_REJECTION_CODES`, and a second greps `src/applier.ts` for every `new OpRejectedError("…")` so a new code cannot be added without a row. **The four `connect` rejections this used to except are fixed (#34)** and are now ordinary rows in that sweep, asserting byte-identity and `op_id`-preservation like every other code; the block that deliberately asserted the bug is deleted and `KNOWN_KA4_VIOLATIONS` is kept empty so a regression has somewhere to be recorded. +The *rejection* half — "a rejected op leaves the doc untouched", stated in the `applyOps` module docstring in `src/applier.ts` and repeated as contract D4 — is swept table-driven in `test/ka4-rejection-byte-identity.test.ts`, one row per reachable rejection code across `add_node`, `insert_workflow`, `set_widget`, `connect`, `connect`+grow, `delete_node`, `clear`, `reset_doc` and the envelope gate, asserting byte-identical `encodeStateAsUpdate` *and* that the `op_id` was not consumed. Completeness is enforced rather than asserted: one test compares the covered codes against `ALL_REJECTION_CODES`, and a second greps `src/applier.ts` for every `new OpRejectedError("…")` so a new code cannot be added without a row. `insert_workflow` adds collision refusals for nodes and links; both validate before any write. **The four `connect` rejections this used to except are fixed (#34)** and are now ordinary rows in that sweep, asserting byte-identity and `op_id`-preservation like every other code; the block that deliberately asserted the bug is deleted and `KNOWN_KA4_VIOLATIONS` is kept empty so a regression has somewhere to be recorded. ### KA-5 — IDs are collision-free without coordination **Rule:** Use 53-bit random mint; document high-water marks are advisory, never allocators; FE stable IDs must be non-colliding strings. **Why:** Offline peers must create entities without a central allocator. -**Enforced by:** **UNGUARDED — see roadmap**. +**Enforced by:** `insert_workflow` derives scoped node, link, group, and definition ids from the immutable op envelope id plus each original id. It never reads high-water marks or current document state to allocate, so distinct operations remain collision-free and order-independent. General producer randomness remains **UNGUARDED — see roadmap**. ### KA-6 — Raw struct updates flow host → follower one-way only **Rule:** Followers never write the shared doc. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d01e3fcc..b97ca2fd 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -4,6 +4,10 @@ V1 is an op-based Yjs document applier implemented once in the shared `@comfyorg/comfy-multi-player` TypeScript package. The browser and Node doc-host sidecar consume the same git-SHA-pinned package. Semantic ops are the replication unit; the widget catalog is sha256-pinned and fail-closed in the deployed architecture. +ADR-022 adds the seventh implemented kind, `insert_workflow`, for atomic template insertion. It is +standalone-only; the applier deterministically remaps whole-tree ids from the operation id, so +distinct inserts cannot collide and exact replay derives the same ids. + ## Guard gaps - ~~**Positive purity assertion (#22):**~~ **CLOSED.** `scripts/check-purity.mjs` asserts directly that the declared and resolved production dependency roots are exactly `{yjs}`, and `scripts/check-import-graph.mjs` asserts the same per source module via `src-runtime-dep-is-yjs-only`. A cross-language parity guard is still only warranted if a second-language implementation is proposed; the preferred policy remains no second implementation (tracked in the separate bullet below). diff --git a/docs/api-contract-proposal.md b/docs/api-contract-proposal.md index 1921b726..f2a5efac 100644 --- a/docs/api-contract-proposal.md +++ b/docs/api-contract-proposal.md @@ -18,6 +18,14 @@ still cheap. That will not stay true. ## Part 1 — decisions to ratify +### D0. Workflow templates insert as one standalone op + +**Accepted by ADR-022.** `insert_workflow` requires `nodes`; `links`, `groups`, and `definitions` +are optional and default to empty. Producers send raw workflows. The applier remaps every carried +id and internal reference deterministically from the op envelope id plus the original id and graph +scope. Different ops therefore cannot collide or depend on arrival order, while exact replay derives +the same ids and is idempotent. The op is stamped and not batchable. + ### D1. Widgets are addressed by name, not by position **Proposal.** A node's widget values live in a map keyed by widget name. The diff --git a/docs/decisions/ADR-022-insert-workflow-op.md b/docs/decisions/ADR-022-insert-workflow-op.md new file mode 100644 index 00000000..f9a3a52d --- /dev/null +++ b/docs/decisions/ADR-022-insert-workflow-op.md @@ -0,0 +1,42 @@ +# ADR-022: Atomic workflow-template insertion + +- **Status:** Accepted +- **Date:** 2026-09-11 +- **Decider:** Christian +- **Source:** ADR-T8, in-app-agent program TDD (Notion) + +## Context + +Agents and users need to insert a complete workflow template, including subgraph definitions, +without decomposing it into independently observable graph edits. Existing op shapes deliberately +reject definition-bearing payloads and cannot express that transaction safely. + +## Decision + +Add the non-batchable `insert_workflow` op. Its frozen envelope carries one authoritative `workflow` +with required top-level nodes and optional links, groups, and definitions. The applier remaps every +carried node, link, group, and nested-definition id to a deterministic string derived from the op +envelope id, graph scope, id kind, and original id. Producers emit the raw workflow unchanged. + +Internal node/link references and definition-instance types are rewritten to those derived ids. +Distinct insert ops therefore cannot collide, even when they carry the same raw ids; exact replay +derives the same ids and is a byte-identical no-op through the existing `op_id` gate. The operation +is one Yjs transaction and is stamped as its own register. + +## Consequences + +- Template insertion is atomic at the semantic-op boundary and deterministic on replay. +- Producers do not inspect document state or remap ids. +- Definition interiors retain scoped graph namespaces, all derived from the op id. +- Existing op schemas remain closed to definitions. + +## Invariants + +This decision touches KA-1, KA-2, KA-3, KA-4, KA-5, FC-1, FC-3, and FC-4. + +## Glossary + +- **ADR-T8:** the accepted in-app-agent program technical design for workflow insertion. +- **Canonical projection:** stable, key-sorted JSON used as the definition-content identity. +- **Envelope id:** the immutable `op_id` carried by the semantic operation. +- **Producer:** the cloud or CLI caller that emits the raw workflow payload. diff --git a/docs/multiplayer-schema.md b/docs/multiplayer-schema.md index 869a0501..1dda4406 100644 --- a/docs/multiplayer-schema.md +++ b/docs/multiplayer-schema.md @@ -44,6 +44,9 @@ Normative inputs, in precedence order: `fixtures/` (the evidence: three replayable sessions, six LWW vectors, the exported catalog, machine-captured findings). Every DECISION below cites the spike finding that forced it. +3. **ADR-022 / ADR-T8** — the local decision and accepted in-app-agent program + TDD add `insert_workflow` as the seventh implemented op without moving the + pinned comfy-cli vocabulary. --- @@ -2044,3 +2047,26 @@ separate input register still decides whether that identity may occupy the requested destination; losing that gate leaves no tuple or dangling reference. This adds an internal `__stamps` key, not a root-layout change, so `SCHEMA_VERSION` remains 2. + +--- + +## Amendment A19 — 2026-09-11 — atomic workflow-template insertion + +ADR-022 adds the standalone-only `insert_workflow` op. Its payload is one +authoritative workflow containing required top-level `nodes` and optional +`links`, `groups`, and `definitions`. The applier deterministically remaps every +inserted id from the immutable `op_id`, graph scope, id kind, and original id, +then rewrites internal references. Producers emit raw payloads. Distinct ops +cannot collide; exact replay derives the same ids. Duplicate ids within one raw +payload remain collision errors. Malformed link tuples are `malformed_op`. + +Definition ids and every nested definition-interior graph id are remapped in +scoped namespaces. Definition-instance `type` values are rewritten with them, +so a consumer definition id is never used as a live id and no live-definition +conflict path remains. + +The whole merge occurs in one Yjs transaction and claims the +`("insert_workflow", op_id)` stamp target. Exact replay is stopped by the +existing applied-op gate and is byte-identical. This adds graph content but no +new root or node-map layout, so `SCHEMA_VERSION` remains 2. Other op payloads +remain closed to definition-bearing fields. diff --git a/scripts/check-import-graph.mjs b/scripts/check-import-graph.mjs index 8a9652bf..450e32b4 100644 --- a/scripts/check-import-graph.mjs +++ b/scripts/check-import-graph.mjs @@ -59,7 +59,7 @@ const root = process.env.IMPORT_GRAPH_ROOT || dirname(dirname(fileURLToPath(impo * not one per change. See `.agents/checks/import-graph.md` and the config's own * "NO includeOnly" comment. */ -const MIN_MODULES = Number(process.env.IMPORT_GRAPH_MIN_MODULES ?? 10); +const MIN_MODULES = Number(process.env.IMPORT_GRAPH_MIN_MODULES ?? 11); const cli = join(root, "node_modules", ".bin", "depcruise"); const run = spawnSync(cli, ["--output-type", "json", "src"], { diff --git a/src/applier.ts b/src/applier.ts index 18c781b3..e5f363d6 100644 --- a/src/applier.ts +++ b/src/applier.ts @@ -86,8 +86,10 @@ import { appliedMap, apush, arrayItemRefusal, + cloneForMap, countDefinitionInstances, createNodeMap, + definitionsMap, linksMap, mapValueRefusal, mdel, @@ -99,8 +101,11 @@ import { stampsMap, widgetStorageOf, } from "./doc.js"; +import { remapInsertedWorkflowIds } from "./remap.js"; import { sha256Hex } from "./digest.js"; import { CMP_EVENT_SCHEMA_VERSION, emitCmpEvent, type CmpCallContext } from "./events.js"; +import { mint, mintDefinition } from "./mint.js"; +import { project, projectDefinition } from "./project.js"; import { MAX_OP_COST, MAX_OPS_PER_BATCH, @@ -122,6 +127,7 @@ import { type GrowConnectOp, type GrowSpec, type InteriorSetWidgetOp, + type InsertWorkflowOp, type Op, type SetWidgetOp, type StampKey, @@ -441,9 +447,11 @@ function dispatch(doc: Y.Doc, op: Op, catalog?: WidgetCatalog): SuccessfulOutcom return applyDeleteNode(doc, op); case "clear": return applyClear(doc, op); + case "insert_workflow": + return applyInsertWorkflow(doc, op, catalog); default: // Exhaustiveness guard (issue #21): with every `Op` member cased above, - // `op` is `never` here. Add a sixth IMPLEMENTED kind to `Op` and this + // `op` is `never` here. Add a seventh IMPLEMENTED kind to `Op` and this // line stops compiling until it gets a `case`. // // Issue #17 removed the `case "reset_doc"` that used to sit here. It was @@ -459,6 +467,238 @@ function dispatch(doc: Y.Doc, op: Op, catalog?: WidgetCatalog): SuccessfulOutcom } } +// --------------------------------------------------------------------------- +// insert_workflow +// --------------------------------------------------------------------------- + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + return `{${Object.keys(value).sort(codePointCompare).map((key) => `${JSON.stringify(key)}:${canonicalJson((value as Record)[key])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function numericId(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && /^[0-9]+$/.test(value)) return parseInt(value, 10); + return undefined; +} + +function scrubPrivateKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(scrubPrivateKeys); + if (typeof value !== "object" || value === null) return value; + const clean: Record = {}; + for (const [key, child] of Object.entries(value)) { + if (!key.startsWith("__")) clean[key] = scrubPrivateKeys(child); + } + return clean; +} + +function definitionId(value: unknown): string | undefined { + if (typeof value === "string" && value.length > 0) return value; + if (typeof value === "number" && Number.isFinite(value)) return String(value); + return undefined; +} + +function validateDefinitionTree(subgraphs: unknown[], seen = new Set()): void { + for (const candidate of subgraphs) { + if (typeof candidate !== "object" || candidate === null || Array.isArray(candidate)) { + throw new OpRejectedError("malformed_op", "insert_workflow: every subgraph definition must be an object"); + } + const sg = candidate as Record; + const id = definitionId(sg["id"]); + if (id === undefined) { + throw new OpRejectedError("malformed_op", "insert_workflow: every subgraph definition requires a valid id"); + } + if (seen.has(id)) { + throw new OpRejectedError("definition_conflict", `insert_workflow: definition id '${id}' is duplicated in the definition tree`); + } + seen.add(id); + const nested = (sg["definitions"] as { subgraphs?: unknown } | undefined)?.subgraphs; + if (nested !== undefined && !Array.isArray(nested)) { + throw new OpRejectedError("malformed_op", "insert_workflow: nested definitions.subgraphs must be an array"); + } + if (Array.isArray(nested)) validateDefinitionTree(nested, seen); + } +} + +function collectStoredDefinitionIds(definition: Y.Map, ids: Set): void { + ids.add(String(definition.get("id"))); + const container = definition.get("definitions"); + const nested = container instanceof Y.Map ? container.get("subgraphs") : undefined; + if (nested instanceof Y.Map) nested.forEach((child) => { + if (child instanceof Y.Map) collectStoredDefinitionIds(child, ids); + }); +} + +function applyInsertWorkflow(doc: Y.Doc, op: InsertWorkflowOp, catalog?: WidgetCatalog): SuccessfulOutcome { + const workflow = scrubPrivateKeys(op.workflow) as unknown; + if (typeof workflow !== "object" || workflow === null || Array.isArray(workflow)) { + throw new OpRejectedError("malformed_op", "insert_workflow: workflow must be an object"); + } + let wf = workflow as Record; + if (!Array.isArray(wf["nodes"]) || (wf["links"] !== undefined && !Array.isArray(wf["links"]))) { + throw new OpRejectedError("malformed_op", "insert_workflow: nodes and links must be arrays"); + } + const definitions = wf["definitions"]; + if (definitions !== undefined && (typeof definitions !== "object" || definitions === null || Array.isArray(definitions))) { + throw new OpRejectedError("malformed_op", "insert_workflow: definitions must be an object"); + } + const subgraphs = (definitions as { subgraphs?: unknown } | undefined)?.subgraphs; + if (subgraphs !== undefined && !Array.isArray(subgraphs)) { + throw new OpRejectedError("malformed_op", "insert_workflow: definitions.subgraphs must be an array"); + } + if (wf["groups"] !== undefined && !Array.isArray(wf["groups"])) { + throw new OpRejectedError("malformed_op", "insert_workflow: groups must be an array"); + } + validateDefinitionTree((subgraphs as unknown[] | undefined) ?? []); + wf = remapInsertedWorkflowIds(wf as unknown as import("./types.js").WorkflowJSON, op.op_id) as unknown as Record; + const remappedDefinitions = wf["definitions"] as { subgraphs?: unknown[] } | undefined; + const remappedSubgraphs = remappedDefinitions?.subgraphs ?? []; + + const nodes = nodesMap(doc); + const links = linksMap(doc); + const stamps = stampsMap(doc); + const stamp = stampKey(op); + const seenNodes = new Set(); + for (const candidate of wf["nodes"] as unknown[]) { + if (typeof candidate !== "object" || candidate === null || Array.isArray(candidate)) { + throw new OpRejectedError("invalid_node_payload", "insert_workflow: every node must be an object"); + } + const node = candidate as { id?: unknown; type?: unknown }; + if (node.id === undefined || typeof node.type !== "string" || node.type.length === 0) { + throw new OpRejectedError("invalid_node_payload", "insert_workflow: every node requires id and type"); + } + const key = String(node.id); + if (seenNodes.has(key)) { + throw new OpRejectedError("node_id_collision", `insert_workflow: node id '${key}' collides`); + } + const existing = nodes.get(key); + if (existing) { + const incumbent = stamps.get(JSON.stringify(["insert_workflow_node", key])) as StampKey | undefined; + if (incumbent === undefined) { + throw new OpRejectedError("node_id_collision", `insert_workflow: node id '${key}' collides`); + } + if (compareStampKeys(stamp, incumbent) <= 0) return "lww-dropped"; + } + seenNodes.add(key); + } + const seenLinks = new Set(); + const linkWrites: unknown[][] = []; + for (const candidate of (wf["links"] as unknown[] | undefined) ?? []) { + if (!Array.isArray(candidate) || candidate[0] === undefined || candidate[1] === undefined || candidate[3] === undefined) { + throw new OpRejectedError("malformed_op", "insert_workflow: every link must be a tuple with an id and two endpoints"); + } + const key = String(candidate[0]); + if (seenLinks.has(key)) { + throw new OpRejectedError("link_id_collision", `insert_workflow: link id '${key}' collides`); + } + if (links.has(key)) { + const incumbent = stamps.get(JSON.stringify(["insert_workflow_link", key])) as StampKey | undefined; + if (incumbent === undefined) { + throw new OpRejectedError("link_id_collision", `insert_workflow: link id '${key}' collides`); + } + if (compareStampKeys(stamp, incumbent) <= 0) return "lww-dropped"; + } + seenLinks.add(key); + const source = String(candidate[1]); + const target = String(candidate[3]); + if ((!nodes.has(source) && !seenNodes.has(source)) || (!nodes.has(target) && !seenNodes.has(target))) { + continue; + } + linkWrites.push(candidate); + } + + const targetKey = stampTargetKey(op); + const prior = stamps.get(targetKey) as StampKey | undefined; + if (prior != null && compareStampKeys(stamp, prior) <= 0) return "lww-dropped"; + + const cat = catalog ?? { types: {} }; + const defs = definitionsMap(doc); + const storedDefinitionIds = new Set(); + defs.forEach((definition) => collectStoredDefinitionIds(definition, storedDefinitionIds)); + const validateAgainstStored = (candidates: unknown[], topLevel: boolean): void => { + for (const candidate of candidates) { + const definition = candidate as Record; + const id = definitionId(definition["id"])!; + if (storedDefinitionIds.has(id) && (!topLevel || !defs.has(id))) { + throw new OpRejectedError("definition_conflict", `insert_workflow: definition id '${id}' is already used in the definition tree`); + } + if (topLevel && defs.has(id)) continue; // whole-tree canonical comparison below decides idempotence/conflict + const nested = (definition["definitions"] as { subgraphs?: unknown[] } | undefined)?.subgraphs; + if (nested) validateAgainstStored(nested, false); + } + }; + validateAgainstStored(remappedSubgraphs, true); + const definitionWrites: Array<[string, Y.Map]> = []; + for (const candidate of remappedSubgraphs) { + if (typeof candidate !== "object" || candidate === null || Array.isArray(candidate)) { + throw new OpRejectedError("malformed_op", "insert_workflow: every subgraph definition must be an object"); + } + const sg = candidate as Record; + const id = definitionId(sg["id"])!; + const scratch = mint({ nodes: [], links: [], definitions: { subgraphs: [sg] } } as unknown as import("./types.js").WorkflowJSON, cat); + const projected = ((project(scratch, cat).definitions as { subgraphs: Record[] }).subgraphs)[0]!; + const canonical = canonicalJson(projected); + const live = defs.get(id); + if (live) { + if (canonicalJson(projectDefinition(live, cat)) === canonical) continue; + throw new OpRejectedError("definition_conflict", `insert_workflow: definition id '${id}' has different content`); + } + definitionWrites.push([id, mintDefinition(sg, cat)]); + } + + const nodeWrites: Array<[string, unknown, Y.Map]> = []; + for (const candidate of wf["nodes"] as import("./types.js").WorkflowNode[]) { + const node = structuredClone(candidate); + const wv = node.widgets_values; + const entry = catalogEntry(catalog, node.type); + if (!catalog && Array.isArray(wv) && wv.length > 0) { + throw new OpRejectedError("catalog_required", `insert_workflow(${node.type}): positional widgets_values needs a catalog`); + } + rejectUnprojectableWidgets(node.type, wv, entry); + try { + nodeWrites.push([String(node.id), node.id, createNodeMap(node, entry?.widget_order)]); + } catch (err) { + throw new OpRejectedError("invalid_node_payload", `insert_workflow(${node.type}): ${err instanceof Error ? err.message : String(err)}`); + } + } + + for (const [id, definition] of definitionWrites) mset(defs, id, definition); + for (const link of linkWrites) { + const key = String((link as unknown[])[0]); + mset(links, key, cloneForMap(link, "insert_workflow: link")); + mset(stamps, JSON.stringify(["insert_workflow_link", key]), stamp); + } + for (const [key, id, nodeMap] of nodeWrites) { + clearObsoleteWidgetStamps(stamps, key); + mset(nodeMap, NODE_INCARNATION_KEY, `${op.op_id}:${key}`); + mset(nodes, key, nodeMap); + mset(stamps, JSON.stringify(["insert_workflow_node", key]), stamp); + mset(stamps, targetKey, stamp); + reconcileNodeLinkRefs(doc, id, nodeMap); + } + if (nodeWrites.length === 0) mset(stamps, targetKey, stamp); + + const meta = metaMap(doc); + if (Array.isArray(wf["groups"])) { + const currentGroups = Array.isArray(meta.get("groups")) ? meta.get("groups") as unknown[] : []; + const merged = new Map(); + for (const group of [...currentGroups, ...wf["groups"]]) merged.set(canonicalJson(group), group); + mset(meta, "groups", [...merged.entries()].sort(([a], [b]) => codePointCompare(a, b)).map(([, group]) => group)); + } + const currentNode = numericId(meta.get("last_node_id")) ?? 0; + const maxNode = Math.max(currentNode, ...nodeWrites.map(([, id]) => numericId(id) ?? currentNode)); + if (maxNode > currentNode) mset(meta, "last_node_id", maxNode); + const currentLink = numericId(meta.get("last_link_id")) ?? 0; + const maxLink = Math.max(currentLink, ...linkWrites.map((link) => numericId(link[0]) ?? currentLink)); + if (maxLink > currentLink) mset(meta, "last_link_id", maxLink); + return nodeWrites.length > 0 || linkWrites.length > 0 || definitionWrites.length > 0 || ((wf["groups"] as unknown[] | undefined)?.length ?? 0) > 0 + ? "applied" + : "no-op"; +} + // --------------------------------------------------------------------------- // add_node // --------------------------------------------------------------------------- diff --git a/src/collab-trace.ts b/src/collab-trace.ts index 2189a995..540175ba 100644 --- a/src/collab-trace.ts +++ b/src/collab-trace.ts @@ -352,7 +352,17 @@ function assertOpPayload(value: unknown, context: string) { const stamp = assertStamp(required(payload, "stamp", context), `${context}.stamp`); if (stamp[1] !== actor) invalid(`${context}.stamp`, "does not preserve semantic identity"); - if (op === "add_node") { + if (op === "insert_workflow") { + const workflow = asRecord(required(payload, "workflow", context), `${context}.workflow`); + asArray(required(workflow, "nodes", `${context}.workflow`), `${context}.workflow.nodes`); + if (Object.hasOwn(workflow, "links")) asArray(workflow["links"], `${context}.workflow.links`); + if (Object.hasOwn(workflow, "definitions")) { + const definitions = asRecord(workflow["definitions"], `${context}.workflow.definitions`); + if (Object.hasOwn(definitions, "subgraphs")) { + asArray(definitions["subgraphs"], `${context}.workflow.definitions.subgraphs`); + } + } + } else if (op === "add_node") { const nodeId = asNodeId(required(payload, "node_id", context), `${context}.node_id`); const classType = asString(required(payload, "class_type", context), `${context}.class_type`); assertNumberArray(required(payload, "pos", context), `${context}.pos`); diff --git a/src/doc.ts b/src/doc.ts index ca1d6eb4..8def35db 100644 --- a/src/doc.ts +++ b/src/doc.ts @@ -805,11 +805,21 @@ export function createNodeMap(node: WorkflowNode, widgetOrder?: readonly string[ */ export function resolveDefinition(doc: Y.Doc, key: string): Y.Map | null { const defs = definitionsMap(doc); - const byId = defs.get(key); + const all: Y.Map[] = []; + const visit = (definition: Y.Map): void => { + all.push(definition); + const container = definition.get("definitions"); + const nested = container instanceof Y.Map ? container.get("subgraphs") : undefined; + if (nested instanceof Y.Map) nested.forEach((child) => { + if (child instanceof Y.Map) visit(child); + }); + }; + defs.forEach(visit); + const byId = all.find((definition) => String(definition.get("id")) === key); if (byId) return byId; let found: Y.Map | null = null; let count = 0; - defs.forEach((dm) => { + all.forEach((dm) => { if (String(dm.get("name") ?? "") === key) { count++; found = dm; @@ -901,13 +911,19 @@ export function countDefinitionInstances(doc: Y.Doc, defId: string, catalog?: Wi } if (aliases.has(String(node.get("type") ?? ""))) count++; }); - definitionsMap(doc).forEach((dm) => { + const visit = (dm: Y.Map): void => { const inner = dm.get("nodes"); if (inner instanceof Y.Map) { inner.forEach((node: unknown) => { if (node instanceof Y.Map && aliases.has(String(node.get("type") ?? ""))) count++; }); } - }); + const container = dm.get("definitions"); + const nested = container instanceof Y.Map ? container.get("subgraphs") : undefined; + if (nested instanceof Y.Map) nested.forEach((child) => { + if (child instanceof Y.Map) visit(child); + }); + }; + definitionsMap(doc).forEach(visit); return count; } diff --git a/src/index.ts b/src/index.ts index 924b5a40..1f9e0f14 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,7 +2,7 @@ * @comfyorg/comfy-multi-player — shared workflow-document package. * * One implementation of op→doc semantics, used identically by the browser - * and the server doc host. The op vocabulary is frozen at six kinds; the + * and the server doc host. The op vocabulary is frozen at seven kinds; the * normative contract is comfy-cli's `docs/op-vocabulary-v1.md` and the stamp * shapes minted by `comfy_cli/workflow_ops.py` (`_new_op`), both pinned at * comfy-cli commit `7e732242d971daf0d2d30f22f997abfacd78986e` — by SHA and @@ -58,6 +58,7 @@ export { type CmpEventType, } from "./events.js"; export * from "./collab-trace.js"; +export { remapWorkflowIds, type RemapWorkflowIdsOptions } from "./remap.js"; /** * ADR-004 follower read-surface. These helpers expose the established wire * layout only so the frontend follower can consume host updates. They are diff --git a/src/mint.ts b/src/mint.ts index b5cb95e0..303f4e56 100644 --- a/src/mint.ts +++ b/src/mint.ts @@ -138,7 +138,7 @@ function definitionLinkKey(ln: unknown, index: number): string { return `#${String(index)}`; } -function mintDefinition(sg: SubgraphDef, catalog: WidgetCatalog): Y.Map { +export function mintDefinition(sg: SubgraphDef, catalog: WidgetCatalog): Y.Map { const dm = new Y.Map(); for (const [k, v] of Object.entries(sg)) { if (k === "nodes" && Array.isArray(v)) { @@ -167,6 +167,22 @@ function mintDefinition(sg: SubgraphDef, catalog: WidgetCatalog): Y.Map }); dm.set("links", lm); dm.set("link_order", order); + } else if (k === "definitions" && typeof v === "object" && v !== null && !Array.isArray(v)) { + const container = new Y.Map(); + const { subgraphs, ...extra } = v as { subgraphs?: unknown; [key: string]: unknown }; + Object.entries(extra).forEach(([key, value]) => container.set(key, cloneForMap(value, `mint: definition.definitions.${key}`))); + if (Array.isArray(subgraphs)) { + const nested = new Y.Map>(); + const order: string[] = []; + for (const child of subgraphs as SubgraphDef[]) { + const key = String(child.id); + order.push(key); + nested.set(key, mintDefinition(child, catalog)); + } + container.set("subgraphs", nested); + container.set("subgraph_order", order); + } + dm.set("definitions", container); } else { dm.set(k, cloneForMap(v, `mint: definition.${k}`)); } diff --git a/src/project.ts b/src/project.ts index 1052926b..c62fd28c 100644 --- a/src/project.ts +++ b/src/project.ts @@ -214,7 +214,7 @@ function tryProjectNode(value: unknown, catalog: WidgetCatalog): WorkflowNode | } /** Definition Y.Map → subgraph definition JSON, interior nodes/links in mint order. */ -function projectDefinition(dm: Y.Map, catalog: WidgetCatalog): Record { +export function projectDefinition(dm: Y.Map, catalog: WidgetCatalog): Record { const out: Record = Object.create(null) as Record; dm.forEach((v, k) => { if (k === "node_order" || k === "link_order") return; // internal order registers @@ -227,6 +227,20 @@ function projectDefinition(dm: Y.Map, catalog: WidgetCatalog): Record v.has(id)).map((id) => structuredClone(v.get(id))); + } else if (k === "definitions" && v instanceof Y.Map) { + const nestedOut: Record = Object.create(null) as Record; + v.forEach((nestedValue, nestedKey) => { + if (nestedKey === "subgraph_order") return; + if (nestedKey === "subgraphs" && nestedValue instanceof Y.Map) { + const order = (v.get("subgraph_order") as string[] | undefined) ?? [...nestedValue.keys()].sort(); + nestedOut["subgraphs"] = order + .filter((id) => nestedValue.has(id)) + .map((id) => projectDefinition(nestedValue.get(id) as Y.Map, catalog)); + } else { + nestedOut[nestedKey] = structuredClone(nestedValue); + } + }); + out[k] = nestedOut; } else { out[k] = structuredClone(v); } diff --git a/src/remap.ts b/src/remap.ts new file mode 100644 index 00000000..f50d15d9 --- /dev/null +++ b/src/remap.ts @@ -0,0 +1,142 @@ +import type { WorkflowJSON, WorkflowNode } from "./types.js"; + +export interface RemapWorkflowIdsOptions { + nodeIdStart: number; + linkIdStart: number; +} + +function derivedId(opId: string, scope: string, kind: string, original: unknown): string { + return `insert:${opId}:${scope}:${kind}:${encodeURIComponent(JSON.stringify(original))}`; +} + +function remapGraph(graph: Record, opId: string, scope: string, definitionIds: Map): void { + const nodes = graph["nodes"] as WorkflowNode[]; + const links = (graph["links"] as unknown[] | undefined) ?? []; + const nodeIds = new Map(); + const linkIds = new Map(); + for (const node of nodes) nodeIds.set(node.id, derivedId(opId, scope, "node", node.id)); + for (const link of links) { + if (Array.isArray(link)) linkIds.set(link[0], derivedId(opId, scope, "link", link[0])); + else if (typeof link === "object" && link !== null) { + const record = link as { id?: unknown }; + linkIds.set(record.id, derivedId(opId, scope, "link", record.id)); + } + } + + for (const node of nodes) { + node.id = nodeIds.get(node.id)!; + if (definitionIds.has(node.type)) node.type = definitionIds.get(node.type)!; + if (Array.isArray(node.inputs)) { + for (const input of node.inputs) { + if (typeof input === "object" && input !== null && "link" in input) { + const record = input as { link?: unknown }; + if (linkIds.has(record.link)) record.link = linkIds.get(record.link); + } + } + } + if (Array.isArray(node.outputs)) { + for (const output of node.outputs) { + if (typeof output === "object" && output !== null && Array.isArray((output as { links?: unknown }).links)) { + const record = output as { links: unknown[] }; + record.links = record.links.map((id) => linkIds.get(id) ?? id); + } + } + } + } + graph["links"] = links.map((link) => { + if (Array.isArray(link)) { + link[0] = linkIds.get(link[0]) ?? link[0]; + link[1] = nodeIds.get(link[1]) ?? link[1]; + link[3] = nodeIds.get(link[3]) ?? link[3]; + } else if (typeof link === "object" && link !== null) { + const record = link as { id?: unknown; origin_id?: unknown; target_id?: unknown }; + record.id = linkIds.get(record.id) ?? record.id; + record.origin_id = nodeIds.get(record.origin_id) ?? record.origin_id; + record.target_id = nodeIds.get(record.target_id) ?? record.target_id; + } + return link; + }); + if (Array.isArray(graph["groups"])) { + graph["groups"] = (graph["groups"] as unknown[]).map((group) => { + if (typeof group === "object" && group !== null && !Array.isArray(group)) { + const record = group as { id?: unknown }; + if (record.id !== undefined) record.id = derivedId(opId, scope, "group", record.id); + } + return group; + }); + } +} + +/** Deterministically remap every id carried by an insertion op, including nested definition graphs. */ +export function remapInsertedWorkflowIds(wf: WorkflowJSON, opId: string): WorkflowJSON { + const out = structuredClone(wf) as WorkflowJSON & Record; + out.links ??= []; + out.groups ??= []; + const subgraphs = (out.definitions?.subgraphs ?? []) as Array>; + const definitionIds = new Map(); + const collect = (definitions: Array>, scope: string): void => { + for (const definition of definitions) { + const original = String(definition["id"]); + const remapped = derivedId(opId, scope, "definition", original); + definitionIds.set(original, remapped); + const nested = (definition["definitions"] as { subgraphs?: Array> } | undefined)?.subgraphs ?? []; + collect(nested, `${scope}/definition:${encodeURIComponent(JSON.stringify(original))}`); + } + }; + collect(subgraphs, "root"); + const rewrite = (definitions: Array>, scope: string): void => { + for (const definition of definitions) { + const original = String(definition["id"]); + definition["id"] = definitionIds.get(original)!; + remapGraph(definition, opId, `${scope}/definition:${encodeURIComponent(JSON.stringify(original))}`, definitionIds); + const nested = (definition["definitions"] as { subgraphs?: Array> } | undefined)?.subgraphs ?? []; + rewrite(nested, `${scope}/definition:${encodeURIComponent(JSON.stringify(original))}`); + } + }; + remapGraph(out, opId, "root", definitionIds); + rewrite(subgraphs, "root"); + return out; +} + +/** Remap only the top-level graph namespace; definition interiors are independent graphs. */ +export function remapWorkflowIds(wf: WorkflowJSON, opts: RemapWorkflowIdsOptions): WorkflowJSON { + const out = structuredClone(wf); + const nodeIds = new Map(); + const linkIds = new Map(); + + out.nodes.forEach((node, index) => nodeIds.set(node.id, opts.nodeIdStart + index)); + out.links.forEach((link, index) => { + if (Array.isArray(link)) linkIds.set(link[0], opts.linkIdStart + index); + }); + + out.nodes = out.nodes.map((node): WorkflowNode => { + const remapped = node as WorkflowNode; + remapped.id = nodeIds.get(node.id)!; + if (Array.isArray(remapped.inputs)) { + for (const input of remapped.inputs) { + if (typeof input === "object" && input !== null && "link" in input) { + const record = input as { link?: unknown }; + if (linkIds.has(record.link)) record.link = linkIds.get(record.link); + } + } + } + if (Array.isArray(remapped.outputs)) { + for (const output of remapped.outputs) { + if (typeof output === "object" && output !== null && Array.isArray((output as { links?: unknown }).links)) { + const record = output as { links: unknown[] }; + record.links = record.links.map((id) => linkIds.get(id) ?? id); + } + } + } + return remapped; + }); + + out.links = out.links.map((link) => { + if (!Array.isArray(link)) return link; + link[0] = linkIds.get(link[0]) ?? link[0]; + link[1] = nodeIds.get(link[1]) ?? link[1]; + link[3] = nodeIds.get(link[3]) ?? link[3]; + return link; + }); + return out; +} diff --git a/src/stamps.ts b/src/stamps.ts index 8e40df80..3d623d8b 100644 --- a/src/stamps.ts +++ b/src/stamps.ts @@ -112,6 +112,12 @@ export function writeTarget(op: WireOp): unknown[] { return ["input", String(op.to_node), op.to_slot]; case "disconnect": return ["input", String(op.to_node), op.to_slot]; + case "insert_workflow": + // One insertion is one register: the op is idempotent by `op_id` (exact + // replay is a no-op via the applied-set gate), and two distinct + // insertions never contend because the minter allocates fresh ids + // (ADR-022). Keyed by op_id so a stamp lookup is per-insertion. + return ["insert_workflow", op.op_id]; case "clear": case "reset_doc": // Whole-document ops: no scalar register to contest, so the target is @@ -121,7 +127,7 @@ export function writeTarget(op: WireOp): unknown[] { return [op.op]; default: // Exhaustiveness guard (issue #21): with every `WireOp` member cased - // above — the five `Op` kinds plus the deferred `reset_doc` — `op` is + // above — the six `Op` kinds plus the deferred `reset_doc` — `op` is // `never` here, so adding a kind to EITHER union fails `tsc` at this // line until it is given a write target. // diff --git a/src/types.ts b/src/types.ts index 4ac9e737..6dd1d6ae 100644 --- a/src/types.ts +++ b/src/types.ts @@ -33,7 +33,15 @@ export const LEGACY_NODE_INCARNATION = "0"; // --------------------------------------------------------------------------- /** The implemented op kinds. `apply` rejects anything else loudly. */ -export const FROZEN_OPS = ["add_node", "connect", "disconnect", "set_widget", "delete_node", "clear"] as const; +export const FROZEN_OPS = [ + "add_node", + "connect", + "disconnect", + "set_widget", + "delete_node", + "clear", + "insert_workflow", +] as const; /** Defined by the vocabulary but deferred (§1.6): rejected until un-deferred by amendment. */ export const DEFERRED_OPS = ["reset_doc"] as const; @@ -130,7 +138,7 @@ export interface OpBase { } // --------------------------------------------------------------------------- -// The seven declared op kinds: six implemented (`Op`) plus the deferred +// The eight declared op kinds: seven implemented (`Op`) plus the deferred // `reset_doc` (`DeferredOp`); together `WireOp`. "Frozen" now means // implemented — `FROZEN_OPS` is pinned to `Op["op"]` exactly (issue #17). // --------------------------------------------------------------------------- @@ -147,6 +155,27 @@ export interface AddNodeOp extends OpBase { node: WorkflowNode; } +/** + * Merge a workflow template (nodes, links, `definitions.subgraphs`) into an + * existing doc in one transaction (ADR-022; agent-subgraph TDD V1.5). + * + * The applier remaps every carried id deterministically from `op_id` and the + * original id. Exact replay therefore chooses the same ids, while distinct + * insert ops cannot collide or depend on document state. `links`, `groups`, + * and `definitions` are optional and default to empty. + */ +export interface InsertWorkflowOp extends OpBase { + op: "insert_workflow"; + /** The template to merge — AUTHORITATIVE, nodes inserted verbatim after id validation. */ + workflow: { + nodes: WorkflowNode[]; + links?: unknown[]; + groups?: unknown[]; + definitions?: { subgraphs?: unknown[]; [key: string]: unknown }; + [key: string]: unknown; + }; +} + /** Autogrow slot descriptor carried by a `connect` (vocabulary §1.2 / §8.4). */ export interface GrowSpec { /** Requested slot name (`images.image0` autogrow shape, or bare `image_2` for the inputcount family). */ @@ -427,7 +456,8 @@ export type Op = | DisconnectOp | SetWidgetOp | DeleteNodeOp - | ClearOp; + | ClearOp + | InsertWorkflowOp; /** * A kind the vocabulary declares but this package refuses to apply @@ -543,6 +573,7 @@ export interface WorkflowNode { export interface WorkflowJSON { nodes: WorkflowNode[]; links: unknown[]; + definitions?: { subgraphs?: unknown[]; [key: string]: unknown }; groups?: unknown[]; extra?: Record; [key: string]: unknown; diff --git a/test/batch-policy.test.ts b/test/batch-policy.test.ts index 04adcc5f..b0dc9b5a 100644 --- a/test/batch-policy.test.ts +++ b/test/batch-policy.test.ts @@ -42,8 +42,8 @@ import { loadCatalog, loadSession, sessionFiles } from "./helpers.js"; const repoRoot = dirname(dirname(fileURLToPath(import.meta.url))); const batchable = new Set(BATCHABLE_OPS); -/** The two kinds the vocabulary calls standalone-only (§1.5, §1.6). */ -const STANDALONE_ONLY = ["clear", "reset_doc"]; +/** Kinds the vocabulary or ADR-022 calls standalone-only. */ +const STANDALONE_ONLY = ["clear", "insert_workflow", "reset_doc"]; const catalog: WidgetCatalog = { types: { Sampler: { widget_order: ["steps"] } } }; const base: WorkflowJSON = { diff --git a/test/convergence.test.ts b/test/convergence.test.ts index 1a17d4ee..8e8e74b9 100644 --- a/test/convergence.test.ts +++ b/test/convergence.test.ts @@ -53,6 +53,8 @@ function touchedNodes(op: WireOp): string[] { return [String(op.from_node), String(op.to_node)]; case "disconnect": return [String(op.to_node)]; + case "insert_workflow": + return op.workflow.nodes.map((node) => String(node.id)); case "clear": case "delete_node": case "reset_doc": diff --git a/test/insert-workflow.test.ts b/test/insert-workflow.test.ts new file mode 100644 index 00000000..18f7551a --- /dev/null +++ b/test/insert-workflow.test.ts @@ -0,0 +1,433 @@ +/** + * `insert_workflow` — merge a template workflow (nodes, links, + * `definitions.subgraphs`) into an existing doc in ONE transaction. + * + * TDD V1.5 (in-app-agent program, ADR-T8): today a template with subgraphs can + * only reach a live doc through a whole-doc reseed, because every other op + * schema rejects definition-bearing fields (KA-11). This op carries the + * template as an opaque payload the applier merges under the same guards the + * six existing kinds obey: collision-free ids (validated BEFORE any mutation, + * KA-4), definition dedupe/fork by content hash, exact-replay no-op through + * the op_id gate, and byte-identical doc on rejection. + * + * ID allocation belongs to the applier and is derived from the immutable op + * id plus each raw id, never from mutable document state. + */ +import { describe, expect, it } from "vitest"; +import * as Y from "yjs"; + +import { appliedMap } from "../src/doc.js"; +import { + OPAQUE_WIDGETS_KEY, + applyOps, + mint, + project, + remapWorkflowIds, + type Op, + type WidgetCatalog, + type WorkflowJSON, +} from "../src/index.js"; +import { appliedOpIds, noOpIds, rejectedOutcomeWithIndex } from "./apply-result-helpers.js"; + +const catalog: WidgetCatalog = { + types: { + Src: { widget_order: [] }, + Sink: { widget_order: ["inputcount"] }, + KSampler: { widget_order: ["seed", "steps"] }, + Inner: { widget_order: ["text"] }, + }, +}; + +function baseWorkflow(): WorkflowJSON { + return { + last_node_id: 7, + last_link_id: 7, + nodes: [ + { id: 1, type: "Src" }, + { id: 2, type: "Src" }, + { id: 3, type: "Sink", inputs: [{ name: "in", link: 7 }], widgets_values: [1] }, + { id: 4, type: "KSampler", widgets_values: [1, 2] }, + { id: 6, type: "def-1" }, + ], + links: [[7, 2, 0, 3, 0, "X"]], + definitions: { + subgraphs: [{ id: "def-1", name: "D", nodes: [{ id: 27, type: "Inner", widgets_values: ["t"] }], links: [] }], + }, + } as unknown as WorkflowJSON; +} + +/** A template with its OWN definition `def-2` and ids already disjoint from the base. */ +function template(): WorkflowJSON { + return { + nodes: [ + { id: 100, type: "Src", pos: [0, 0] }, + { id: 101, type: "def-2", inputs: [{ name: "a", link: 200 }] }, + ], + links: [[200, 100, 0, 101, 0, "X"]], + definitions: { + subgraphs: [{ id: "def-2", name: "E", nodes: [{ id: 5, type: "Inner", widgets_values: ["u"] }], links: [] }], + }, + } as unknown as WorkflowJSON; +} + +let seq = 0; +function env(): Pick { + return { + op_id: ("i" + String(seq++).padStart(4, "0")).padEnd(32, "0"), + actor: "a", + base_version: 1, + stamp: [1, "a"] as [number, string], + }; +} + +function insertOp(workflow: unknown, overrides: Partial = {}): Op { + return { ...env(), op: "insert_workflow", workflow, ...overrides } as unknown as Op; +} + +const bytes = (doc: Y.Doc): Buffer => Buffer.from(Y.encodeStateAsUpdate(doc)); + +function ids(wf: WorkflowJSON): unknown[] { + return (wf.nodes ?? []).map((n) => n.id); +} +function linkIds(wf: WorkflowJSON): unknown[] { + return (wf.links ?? []).map((l) => (l as unknown[])[0]); +} +function defIds(wf: WorkflowJSON): string[] { + const defs = (wf as { definitions?: { subgraphs?: { id: unknown }[] } }).definitions; + return (defs?.subgraphs ?? []).map((s) => String(s.id)); +} + +describe("insert_workflow: happy path", () => { + it("merges template nodes, links and definitions into the doc in one op", () => { + const doc = mint(baseWorkflow(), catalog); + const op = insertOp(template()); + const result = applyOps(doc, [op], catalog); + + expect(appliedOpIds(result)).toEqual([op.op_id]); + const wf = project(doc, catalog); + expect(ids(wf).slice(0, 5)).toEqual([1, 2, 3, 4, 6]); + expect(ids(wf).slice(5).every((id) => typeof id === "string" && id.includes(op.op_id))).toBe(true); + expect(linkIds(wf)[1]).toEqual(expect.stringContaining(op.op_id)); + expect(defIds(wf)[1]).toEqual(expect.stringContaining(op.op_id)); + // Inserted nodes project with their widgets resolved through the catalog. + const n100 = wf.nodes!.find((n) => n.pos?.[0] === 0)!; + expect(n100.type).toBe("Src"); + expect(n100.pos).toEqual([0, 0]); + const n101 = wf.nodes!.find((n) => n.type === defIds(wf)[1])!; + expect((n101.inputs?.[0] as { link?: unknown } | undefined)?.link).toBe(linkIds(wf)[1]); + expect(wf.last_node_id).toBe(7); + expect(wf.last_link_id).toBe(7); + }); + + it("is an exact-replay no-op through the op_id gate (no second copy)", () => { + const doc = mint(baseWorkflow(), catalog); + const op = insertOp(template()); + applyOps(doc, [op], catalog); + const before = bytes(doc); + const again = applyOps(doc, [op], catalog); + expect(noOpIds(again)).toEqual([op.op_id]); + expect(bytes(doc).equals(before)).toBe(true); + expect(ids(project(doc, catalog))).toHaveLength(7); + }); + + it("remaps a template definition even when its raw id matches live content", () => { + const doc = mint(baseWorkflow(), catalog); + const tpl = template(); + (tpl as { definitions: { subgraphs: unknown[] } }).definitions.subgraphs = [ + { id: "def-1", name: "D", nodes: [{ id: 27, type: "Inner", widgets_values: ["t"] }], links: [] }, + ]; + tpl.nodes![1] = { id: 101, type: "def-1", inputs: [{ name: "a", link: 200 }] }; + const op = insertOp(tpl); + const result = applyOps(doc, [op], catalog); + expect(appliedOpIds(result)).toEqual([op.op_id]); + const wf = project(doc, catalog); + expect(defIds(wf)).toHaveLength(2); + expect(defIds(wf)[1]).toContain(op.op_id); + expect(wf.nodes!.find((n) => n.id !== 6 && n.type === defIds(wf)[1])).toBeDefined(); + }); + + it("preserves opaque widgets_values for uncatalogued inserted nodes", () => { + const doc = mint(baseWorkflow(), catalog); + const tpl = { nodes: [{ id: 300, type: "Note", widgets_values: ["hello"] }], links: [] } as unknown as WorkflowJSON; + applyOps(doc, [insertOp(tpl)], catalog); + const wf = project(doc, catalog); + expect(wf.nodes!.find((n) => n.type === "Note")!.widgets_values).toEqual(["hello"]); + expect(OPAQUE_WIDGETS_KEY).toBeTruthy(); + }); + + it("accepts nodes without optional links, groups, or definitions", () => { + const doc = mint(baseWorkflow(), catalog); + const op = insertOp({ nodes: [{ id: 1, type: "Src", title: "minimal" }] }); + + expect(applyOps(doc, [op], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + expect(project(doc, catalog).nodes!.find((node) => node.title === "minimal")?.id).toEqual(expect.stringContaining(op.op_id)); + }); + + it("merges groups deterministically without dropping existing groups", () => { + const seeded = { ...baseWorkflow(), groups: [{ title: "Base" }] } as WorkflowJSON; + const doc = mint(seeded, catalog); + applyOps(doc, [insertOp({ nodes: [], links: [], groups: [{ title: "Inserted" }] })], catalog); + + expect(project(doc, catalog).groups).toEqual([{ title: "Base" }, { title: "Inserted" }]); + }); + + it("scrubs private __ keys recursively from every inserted payload branch", () => { + const doc = mint(baseWorkflow(), catalog); + applyOps(doc, [insertOp({ + nodes: [{ id: 100, type: "Src", properties: { visible: true, __definition_digest: "node-secret" } }], + links: [], + groups: [{ title: "G", nested: { __private: "group-secret", visible: true } }], + definitions: { + subgraphs: [{ id: "def-2", nodes: [], links: [], metadata: { __definition_digest: "def-secret", visible: true } }], + }, + })], catalog); + + const json = JSON.stringify(project(doc, catalog)); + expect(json).not.toContain("__definition_digest"); + expect(json).not.toContain("__private"); + expect(json).toContain('"visible":true'); + }); + + it("makes colliding concurrent inserts converge in either legal arrival order", () => { + const seed = Y.encodeStateAsUpdate(mint(baseWorkflow(), catalog)); + const run = (ops: Op[]): WorkflowJSON => { + const doc = new Y.Doc(); + Y.applyUpdate(doc, seed); + for (const op of ops) applyOps(doc, [op], catalog); + return project(doc, catalog); + }; + const lower = insertOp( + { nodes: [{ id: 100, type: "Src", title: "lower" }, { id: 101, type: "Sink" }], links: [[200, 100, 0, 101, 0, "lower"]] }, + { op_id: "a".repeat(32), stamp: [2, "a"] }, + ); + const higher = insertOp( + { nodes: [{ id: 100, type: "Src", title: "higher" }, { id: 101, type: "Sink" }], links: [[200, 100, 0, 101, 0, "higher"]] }, + { op_id: "b".repeat(32), stamp: [3, "b"] }, + ); + + expect(run([lower, higher])).toEqual(run([higher, lower])); + expect(run([lower, higher]).nodes!.filter((node) => node.title === "lower" || node.title === "higher")).toHaveLength(2); + expect(run([lower, higher]).links!.filter((link) => ["lower", "higher"].includes((link as unknown[])[5] as string))).toHaveLength(2); + }); + + it("keeps valid inserted content when a carried link endpoint was concurrently deleted", () => { + const seed = Y.encodeStateAsUpdate(mint(baseWorkflow(), catalog)); + const insert = insertOp({ + nodes: [{ id: 100, type: "Src", title: "inserted source" }, { id: 101, type: "Sink", title: "inserted sink" }], + links: [[200, 100, 0, 101, 0, "valid"], [201, 100, 0, 1, 0, "missing endpoint"]], + groups: [{ title: "Inserted group" }], + definitions: { subgraphs: [{ id: "inserted-def", nodes: [], links: [] }] }, + }); + const remove = { + ...env(), op: "delete_node", node_id: 1, removed_links: [], stamp: [2, "b"], + } as unknown as Op; + const run = (ops: Op[]): WorkflowJSON => { + const doc = new Y.Doc(); + Y.applyUpdate(doc, seed); + for (const op of ops) applyOps(doc, [op], catalog); + return project(doc, catalog); + }; + + const deleteThenInsert = run([remove, insert]); + const insertThenDelete = run([insert, remove]); + expect(deleteThenInsert).toEqual(insertThenDelete); + expect(deleteThenInsert.nodes!.filter((node) => typeof node.title === "string" && node.title.startsWith("inserted"))).toHaveLength(2); + expect(deleteThenInsert.links!.filter((link) => (link as unknown[])[5] === "valid")).toHaveLength(1); + expect(deleteThenInsert.links!.some((link) => (link as unknown[])[5] === "missing endpoint")).toBe(false); + expect(deleteThenInsert.groups).toContainEqual({ title: "Inserted group" }); + expect(defIds(deleteThenInsert).some((id) => id.includes(insert.op_id))).toBe(true); + }); + + it("stores nested definitions as addressable maps for later interior set_widget", () => { + const doc = mint(baseWorkflow(), catalog); + const nested = { id: "nested-def", nodes: [{ id: 9, type: "Inner", widgets_values: ["before"] }], links: [] }; + const outer = { + id: "outer-def", name: "Outer", nodes: [{ id: 8, type: "nested-def" }], links: [], definitions: { subgraphs: [nested] }, + }; + applyOps(doc, [insertOp({ nodes: [{ id: 100, type: "outer-def" }], links: [], definitions: { subgraphs: [outer] } })], catalog); + const projected = project(doc, catalog) as { nodes: Array<{ id: string; type: string }>; definitions: { subgraphs: Array> } }; + const projectedOuter = projected.definitions.subgraphs.find((definition) => definition["name"] === "Outer") ?? projected.definitions.subgraphs.find((definition) => definition["id"] !== "def-1")!; + const projectedNested = (projectedOuter["definitions"] as { subgraphs: Array> }).subgraphs[0]!; + const outerNode = (projectedOuter["nodes"] as Array<{ id: string }>)[0]!; + const innerNode = (projectedNested["nodes"] as Array<{ id: string }>)[0]!; + const host = projected.nodes.find((node) => node.type === projectedOuter["id"])!; + const edit = { ...env(), op: "set_widget", node_id: host.id, path: [host.id, outerNode.id, innerNode.id], inner_widget: "text", value: "after" } as unknown as Op; + + expect(applyOps(doc, [edit], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + const defs = (project(doc, catalog) as { definitions: { subgraphs: Array> } }).definitions.subgraphs; + const updatedOuter = defs.find((definition) => definition.id === projectedOuter["id"]) as { + definitions: { subgraphs: Array<{ nodes: Array<{ widgets_values: unknown[] }> }> }; + }; + expect(updatedOuter.definitions.subgraphs[0]!.nodes[0]!.widgets_values).toEqual(["after"]); + }); +}); + +describe("insert_workflow: rejection (KA-4 byte identity, op_id absent from applied)", () => { + const cases: { name: string; workflow: unknown; code: string; withCatalog: boolean }[] = [ + { + name: "duplicate node id inside the template itself", + workflow: { nodes: [{ id: 100, type: "Src" }, { id: 100, type: "Src" }], links: [] }, + code: "node_id_collision", + withCatalog: true, + }, + { name: "missing workflow", workflow: undefined, code: "malformed_op", withCatalog: true }, + { name: "non-object workflow", workflow: "nope", code: "malformed_op", withCatalog: true }, + { name: "nodes is not an array", workflow: { nodes: {}, links: [] }, code: "malformed_op", withCatalog: true }, + { + name: "node without id/type", + workflow: { nodes: [{ pos: [0, 0] }], links: [] }, + code: "invalid_node_payload", + withCatalog: true, + }, + { + name: "positional widgets with no catalog", + workflow: { nodes: [{ id: 100, type: "KSampler", widgets_values: [1, 2] }], links: [] }, + code: "catalog_required", + withCatalog: false, + }, + ]; + + for (const c of cases) { + it(`rejects: ${c.name} → ${c.code}`, () => { + const doc = mint(baseWorkflow(), catalog); + const before = bytes(doc); + const op = insertOp(c.workflow); + const result = applyOps(doc, [op], c.withCatalog ? catalog : undefined); + expect(rejectedOutcomeWithIndex(result)!.code).toBe(c.code); + expect(bytes(doc).equals(before)).toBe(true); + expect(appliedMap(doc).has(op.op_id)).toBe(false); + }); + } + + it("remaps raw ids that collide with live ids without touching incumbents", () => { + const doc = mint(baseWorkflow(), catalog); + const before = bytes(doc); + const wf = { nodes: [{ id: 100, type: "Src" }, { id: 101, type: "Src" }, { id: 4, type: "Src" }], links: [] }; + expect(applyOps(doc, [insertOp(wf)], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + expect(bytes(doc).equals(before)).toBe(false); + expect(project(doc, catalog).nodes!.find((node) => node.id === 4)?.type).toBe("KSampler"); + expect(ids(project(doc, catalog))).toHaveLength(8); + }); + + it("routes dangling link endpoints through the existing unknown-node no-op path", () => { + const doc = mint(baseWorkflow(), catalog); + const op = insertOp({ nodes: [], links: [[200, 100, 0, 999, 0, "X"]] }); + const result = applyOps(doc, [op], catalog); + + expect(result.outcomes[0]).toMatchObject({ outcome: "no-op" }); + expect(appliedMap(doc).has(op.op_id)).toBe(true); + }); + + it("rejects an id-less nested subgraph atomically", () => { + const doc = mint(baseWorkflow(), catalog); + const before = bytes(doc); + const op = insertOp({ + nodes: [{ id: 100, type: "Src" }], + links: [], + definitions: { + subgraphs: [{ id: "outer", nodes: [], links: [], definitions: { subgraphs: [{ nodes: [], links: [] }] } }], + }, + }); + + expect(rejectedOutcomeWithIndex(applyOps(doc, [op], catalog))!.code).toBe("malformed_op"); + expect(bytes(doc).equals(before)).toBe(true); + expect(appliedMap(doc).has(op.op_id)).toBe(false); + }); + + it("remaps a nested definition id already used in the live tree", () => { + const doc = mint(baseWorkflow(), catalog); + const op = insertOp({ + nodes: [{ id: 100, type: "outer" }], links: [], + definitions: { + subgraphs: [{ id: "outer", nodes: [], links: [], definitions: { subgraphs: [{ id: "def-1", nodes: [], links: [] }] } }], + }, + }); + + expect(applyOps(doc, [op], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + }); + + it("inserts same-raw-id definitions identically in either arrival order", () => { + const seed = Y.encodeStateAsUpdate(mint(baseWorkflow(), catalog)); + const makeDoc = (): Y.Doc => { + const doc = new Y.Doc(); + Y.applyUpdate(doc, seed); + return doc; + }; + const makeConflict = (name: string, nodeId: number, opId: string): Op => + insertOp( + { + nodes: [{ id: nodeId, type: "def-1" }], + links: [], + definitions: { subgraphs: [{ id: "def-1", name, nodes: [], links: [] }] }, + }, + { op_id: opId }, + ); + const first = makeConflict("A", 100, "a".repeat(32)); + const second = makeConflict("B", 101, "b".repeat(32)); + const docA = makeDoc(); + const docB = makeDoc(); + + expect(applyOps(docA, [first], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + expect(applyOps(docA, [second], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + expect(applyOps(docB, [second], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + expect(applyOps(docB, [first], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + expect(project(docA, catalog)).toEqual(project(docB, catalog)); + expect(defIds(project(docA, catalog))).toHaveLength(3); + }); + + it("never overwrites an occupied legacy fork-shaped id", () => { + const wf = baseWorkflow() as WorkflowJSON & { definitions: { subgraphs: Record[] } }; + wf.definitions.subgraphs.push({ id: "def-1-deadbeef", name: "Occupied", nodes: [], links: [] }); + const doc = mint(wf, catalog); + const before = bytes(doc); + const op = insertOp({ + nodes: [{ id: 100, type: "def-1" }], + links: [], + definitions: { subgraphs: [{ id: "def-1", name: "Different", nodes: [], links: [] }] }, + }); + + expect(applyOps(doc, [op], catalog).outcomes[0]).toMatchObject({ outcome: "applied" }); + expect(bytes(doc).equals(before)).toBe(false); + const definitions = (project(doc, catalog) as { definitions: { subgraphs: { id: string; name: string }[] } }).definitions.subgraphs; + expect(definitions.find((definition) => definition.id === "def-1-deadbeef")!.name).toBe("Occupied"); + }); +}); + +describe("remapWorkflowIds (minter-side helper)", () => { + it("shifts node and link ids and rewrites every reference, leaving definitions untouched", () => { + const out = remapWorkflowIds(template(), { nodeIdStart: 8, linkIdStart: 8 }); + expect(ids(out)).toEqual([8, 9]); + expect(linkIds(out)).toEqual([8]); + const link = out.links![0] as unknown[]; + expect(link.slice(0, 5)).toEqual([8, 8, 0, 9, 0]); + expect(out.nodes![1]!.inputs).toEqual([{ name: "a", link: 8 }]); + expect(defIds(out)).toEqual(["def-2"]); + // Interior definition ids are a separate namespace: unchanged. + const inner = (out as { definitions: { subgraphs: { nodes: { id: unknown }[] }[] } }).definitions.subgraphs[0]!; + expect(inner.nodes[0]!.id).toBe(5); + }); + + it("rewrites output links arrays and does not mutate its input", () => { + const tpl = { + nodes: [ + { id: 1, type: "Src", outputs: [{ name: "o", links: [1, 2] }] }, + { id: 2, type: "Sink", inputs: [{ name: "in", link: 1 }] }, + { id: 3, type: "Sink", inputs: [{ name: "in", link: 2 }] }, + ], + links: [ + [1, 1, 0, 2, 0, "X"], + [2, 1, 0, 3, 0, "X"], + ], + } as unknown as WorkflowJSON; + const snapshot = structuredClone(tpl); + const out = remapWorkflowIds(tpl, { nodeIdStart: 10, linkIdStart: 20 }); + expect(tpl).toEqual(snapshot); + expect(ids(out)).toEqual([10, 11, 12]); + expect(out.nodes![0]!.outputs).toEqual([{ name: "o", links: [20, 21] }]); + expect(out.nodes![2]!.inputs).toEqual([{ name: "in", link: 21 }]); + expect(out.links).toEqual([ + [20, 10, 0, 11, 0, "X"], + [21, 10, 0, 12, 0, "X"], + ]); + }); +}); diff --git a/test/ka4-rejection-byte-identity.test.ts b/test/ka4-rejection-byte-identity.test.ts index 309a3bfa..f4165321 100644 --- a/test/ka4-rejection-byte-identity.test.ts +++ b/test/ka4-rejection-byte-identity.test.ts @@ -187,6 +187,30 @@ const CASES: Row[] = [ }) as unknown as Op, }, + // ---- insert_workflow ---------------------------------------------------- + { + kind: "insert_workflow", + why: "duplicate raw node ids are ambiguous before deterministic remapping", + code: "node_id_collision", + build: () => ({ op: "insert_workflow", ...env(), workflow: { nodes: [{ id: 1, type: "Src" }, { id: 1, type: "Src" }] } }) as Op, + }, + { + kind: "insert_workflow", + why: "duplicate raw link ids are ambiguous before deterministic remapping", + code: "link_id_collision", + build: () => ({ op: "insert_workflow", ...env(), workflow: { nodes: [], links: [[7, 2, 0, 3, 0, "X"], [7, 2, 0, 3, 0, "X"]] } }) as Op, + }, + { + kind: "insert_workflow", + why: "duplicate raw definition ids are ambiguous before deterministic remapping", + code: "definition_conflict", + build: () => ({ + op: "insert_workflow", + ...env(), + workflow: { nodes: [], definitions: { subgraphs: [{ id: "d", nodes: [], links: [] }, { id: "d", nodes: [], links: [] }] } }, + }) as Op, + }, + // ---- set_widget ---------------------------------------------------------- { kind: "set_widget", @@ -633,6 +657,9 @@ const ALL_REJECTION_CODES = [ "op_deferred", "catalog_required", "invalid_node_payload", + "node_id_collision", + "link_id_collision", + "definition_conflict", "unknown_widget", "opaque_widgets", "widget_out_of_range", diff --git a/test/permutation/full-op-pool.permutation.test.ts b/test/permutation/full-op-pool.permutation.test.ts index ffe50d6b..30764625 100644 --- a/test/permutation/full-op-pool.permutation.test.ts +++ b/test/permutation/full-op-pool.permutation.test.ts @@ -45,8 +45,8 @@ const PRECONDITIONS = [ "interior-or-inputcount", "promoted-or-autogrow", ] as const; -const PAIR_EXECUTIONS = 150_528; -const SAMPLED_RUNS = 24_736; +const PAIR_EXECUTIONS = 196_608; +const SAMPLED_RUNS = 1_696; const SAMPLED_EXECUTIONS = SAMPLED_RUNS * 2; const TOTAL_EXECUTIONS = PAIR_EXECUTIONS + SAMPLED_EXECUTIONS; const SAMPLE_SEED = 0x4f70504; @@ -215,6 +215,8 @@ function makeOp( return { ...env, op: "delete_node", node_id: side === 0 ? 10 : 20, removed_links: [80, 100, 101] }; case "clear": return { ...env, op: "clear", removed_nodes: side === 0 ? [10, 40, 57] : [20, 40, 57] }; + case "insert_workflow": + return { ...env, op: "insert_workflow", workflow: { nodes: [node(140 + serial, "Aux", [], [], [value])], links: [] } }; case "reset_doc": return { ...env, op: "reset_doc", workflow: { nodes: [], links: [] } }; } diff --git a/test/readonly-surface.test.ts b/test/readonly-surface.test.ts index 5c28b272..5810d87b 100644 --- a/test/readonly-surface.test.ts +++ b/test/readonly-surface.test.ts @@ -730,6 +730,7 @@ describe("read-only surface — classification", () => { // exposes no shared-document write, replay, or merge-policy implementation. "COLLAB_TRACE_SCHEMA", "assertCollabReplayTraceV1", + "remapWorkflowIds", ]; const READ_SURFACE: readonly string[] = [ "readGraph", diff --git a/test/types/invalid-states.negative.ts b/test/types/invalid-states.negative.ts index 2c70f777..1e350d2b 100644 --- a/test/types/invalid-states.negative.ts +++ b/test/types/invalid-states.negative.ts @@ -25,6 +25,7 @@ import type { ConnectOp, DeleteNodeOp, GrowConnectOp, + InsertWorkflowOp, InteriorSetWidgetOp, Op, ResetDocOp, @@ -196,6 +197,13 @@ declare function applyOpsSignature(ops: Op[]): void; // @ts-expect-error #17: the applier cannot be handed an op it always refuses. applyOpsSignature([reset]); +const insertWorkflow: InsertWorkflowOp = { + op: "insert_workflow", + ...env, + workflow: { nodes: [], links: [], definitions: { subgraphs: [] } }, +}; +const insertWorkflowAsOp: Op = insertWorkflow; + // --------------------------------------------------------------------------- // Positive controls — these MUST compile, or the gate above is vacuous // ---------------------------------------------------------------------------