diff --git a/comfy_cli/workflow_ops.py b/comfy_cli/workflow_ops.py index 28635f2e9..80ae9880c 100644 --- a/comfy_cli/workflow_ops.py +++ b/comfy_cli/workflow_ops.py @@ -52,11 +52,13 @@ from __future__ import annotations import copy +import hashlib import json import math import random import re import uuid +from decimal import Decimal from typing import Any from comfy_cli import layout @@ -74,6 +76,215 @@ # frontend counter ids), always inside JS Number.MAX_SAFE_INTEGER. _ID_FLOOR = 1 << 40 +# Mirrored EXACTLY from comfy-multi-player's `src/limits.ts` at +# 221b2b2e470c79afaface24b1edcb53cdafc8922 (`MAX_PAYLOAD_DEPTH` 64, +# `MAX_COLLECTION_ENTRIES` 4096, `MAX_OP_COST` 262_144) — cited by SHA, not by +# branch, because the whole point is that the two implementations agree. +# +# These are not a local policy choice. CMP's A8 canonicalizer applies the same +# budget to the whole envelope on EVERY apply, before its dedupe gate +# (`applier.ts`: `applyOps` -> `opDigest` -> `canonicalJson` -> `opBoundsRefusal`), +# so "the digest is computed on every apply" is equally true there and a looser +# bound HERE rescues nothing — it only moves where an oversized op fails, from a +# local `malformed_op` to a `malformed_op` from the shared package after +# comfy-cli already accepted the payload. Re-deriving a second, looser budget is +# exactly the validation drift the one shared op package exists to prevent. If a +# real edit ever needs more room, widen it in comfy-multi-player and move this +# pin; never widen it here alone. +_MAX_OP_PAYLOAD_DEPTH = 64 +_MAX_OP_COLLECTION_ENTRIES = 4096 +_MAX_OP_COST = 262_144 + +#: Any surrogate code point in a Python ``str`` is a LONE surrogate — astral +#: characters are single code points there — so matching the whole block is +#: enough to find the ones that cannot be UTF-8 encoded. +_LONE_SURROGATE = re.compile("[\ud800-\udfff]") + + +def _es_number(value: float) -> str: + """Render a float exactly as ECMAScript ``Number::toString`` does. + + The canonical form has to be byte-identical on both sides of the wire, and + the counterpart implementation (comfy-multi-player schema Amendment A8) + builds it with ``JSON.stringify``. Python's own float repr disagrees with + that on three shapes a FLOAT widget produces constantly — an integral float + (``20.0`` vs ``20``), negative zero (``-0.0`` vs ``0``) and an exponent + (``1e-07`` vs ``1e-7``) — so digesting Python's spelling would reject an + identical op that had merely been relayed through JS as ``op_id_reuse``, and + no float payload would ever digest the same on the two sides. + + ``repr`` is still the input: it is the shortest round-tripping decimal, the + same digit string ECMAScript's algorithm starts from. Only the rendering of + that digit string differs, and this reimplements it: the ECMAScript + ``Number::toString`` algorithm, including its ``k``/``n``/``s`` case + analysis for when to use fixed versus exponential notation. + """ + if value == 0: # also folds -0.0, which ECMAScript renders as "0" + return "0" + if value < 0: + return "-" + _es_number(-value) + _, raw_digits, exponent = Decimal(repr(value)).as_tuple() + digits = list(raw_digits) + # ECMAScript picks the `s` that is NOT divisible by 10, so trailing zeros + # move into the exponent instead of into the digit string. + while len(digits) > 1 and digits[-1] == 0: + digits.pop() + exponent += 1 + s = "".join(str(d) for d in digits) + k = len(s) + n = exponent + k + if k <= n <= 21: + return s + "0" * (n - k) + if 0 < n <= 21: + return f"{s[:n]}.{s[n:]}" + if -6 < n <= 0: + return "0." + "0" * -n + s + e = n - 1 + mantissa = s if k == 1 else f"{s[0]}.{s[1:]}" + return f"{mantissa}e{'+' if e >= 0 else '-'}{abs(e)}" + + +def _canonical_string(value: str) -> str: + """One JSON string literal, escaped the way ``JSON.stringify`` escapes it. + + ``json.dumps`` already escapes exactly the code points ``JSON.stringify`` + does, so the two agree on every well-formed string. They disagree on a lone + surrogate — which ``json.loads('"\\ud800"')`` happily produces: ES2019 + well-formed stringify emits it as a ``\\udXXX`` escape, Python emits it raw. + A raw surrogate cannot be UTF-8 encoded at all, so leaving it would make the + digest die with a bare ``UnicodeEncodeError`` outside the ``malformed_op:`` + contract callers classify errors by. Escaping it both matches ECMAScript and + keeps the encode in :func:`_op_digest` total. + """ + literal = json.dumps(value, ensure_ascii=False) + if _LONE_SURROGATE.search(literal): + literal = _LONE_SURROGATE.sub(lambda m: f"\\u{ord(m.group()):04x}", literal) + return literal + + +def _canonical_key(key: Any) -> str: + """An object key as its string form, matching ECMAScript property coercion.""" + if isinstance(key, str): + return key + if isinstance(key, bool): # before int: bool is an int subclass + return "true" if key else "false" + if isinstance(key, int): + return str(key) + if isinstance(key, float): + if math.isnan(key): + return "NaN" + if math.isinf(key): + return "Infinity" if key > 0 else "-Infinity" + return _es_number(key) + if key is None: + return "null" + raise ValueError(f"malformed_op: object key of type {type(key).__name__} is not canonical JSON") + + +def _canonical_value(value: Any, depth: int, budget: list[int]) -> str: + if depth > _MAX_OP_PAYLOAD_DEPTH: + raise ValueError(f"payload_too_deep: op payload nests deeper than {_MAX_OP_PAYLOAD_DEPTH} levels") + budget[0] += 1 + if value is None: + budget[0] += 8 + rendered = "null" + elif isinstance(value, bool): # before int: bool is an int subclass + budget[0] += 8 + rendered = "true" if value else "false" + elif isinstance(value, int): + budget[0] += 8 + rendered = str(value) + elif isinstance(value, float): + budget[0] += 8 + # ``JSON.stringify`` renders every non-finite number as ``null``, so + # doing the same is what makes the two canonical forms agree. It also + # un-breaks overwriting a widget whose CURRENT value is NaN/Infinity + # (Python's ``json.load`` accepts both, and the op captures it in + # ``old``), which the previous ``allow_nan=False`` rejected outright. + rendered = _es_number(value) if math.isfinite(value) else "null" + elif isinstance(value, str): + budget[0] += len(value) + rendered = _canonical_string(value) + elif isinstance(value, dict): + if len(value) > _MAX_OP_COLLECTION_ENTRIES: + raise ValueError(f"malformed_op: object exceeds {_MAX_OP_COLLECTION_ENTRIES} entries") + budget[0] += 4 + members = [] + for key in value: + name = _canonical_key(key) + budget[0] += len(name) + members.append((name, value[key])) + # Unicode code point order at every depth: Python sorts `str` that way. + members.sort(key=lambda member: member[0]) + rendered = ( + "{" + ",".join(f"{_canonical_string(n)}:{_canonical_value(v, depth + 1, budget)}" for n, v in members) + "}" + ) + elif isinstance(value, list | tuple): + if len(value) > _MAX_OP_COLLECTION_ENTRIES: + raise ValueError(f"malformed_op: array exceeds {_MAX_OP_COLLECTION_ENTRIES} entries") + budget[0] += 4 + rendered = "[" + ",".join(_canonical_value(child, depth + 1, budget) for child in value) + "]" + else: + raise ValueError(f"malformed_op: op payload is not canonical JSON: {type(value).__name__} is not serializable") + if budget[0] > _MAX_OP_COST: + raise ValueError(f"malformed_op: payload exceeds the {_MAX_OP_COST}-unit cost budget") + return rendered + + +def _canonical_op(op: dict) -> str: + """Compact JSON over the whole envelope: keys sorted by Unicode code point + at every depth, array order preserved, numbers rendered as ECMAScript + renders them so the digest matches the CMP A8 counterpart's.""" + return _canonical_value(op, 0, [0]) + + +def _op_digest(op: dict) -> str: + return hashlib.sha256(_canonical_op(op).encode("utf-8")).hexdigest() + + +def _applied_digest(op: dict) -> str | None: + """The digest :func:`apply_op` would have recorded for ``op``, or ``None`` + when the payload is not canonical, so nothing recorded one.""" + try: + return _op_digest(op) + except ValueError: + return None + + +def _rebind_op_digest(workflow: dict, op: dict, applied: str | None) -> None: + """Re-bind ``op_id`` to the digest of the *enriched* envelope. + + ``_set_widget_impl`` finishes a promoted / legacy-primitive op AFTER + :func:`apply_op` returns — the host's materialized ``host_widgets_values`` + can only be read back out of the document the write produced — so the ledger + otherwise holds the pre-enrichment digest while the caller broadcasts the + enriched payload, and a redelivery of what was actually sent is rejected as + ``op_id_reuse``. + + Nothing happens in the two cases where correcting it would be wrong: + + * the op was DEFERRED, or was a duplicate, so the ledger entry either does + not exist or belongs to a different payload. Detected by comparing against + the digest ``apply_op`` actually wrote for *this* call rather than by + trusting the key's presence. + * the enriched envelope is not canonical. ``host_widgets_values`` comes out + of the document, so it can carry a NaN/Infinity, a lone surrogate or a + volume the op payload never did. The document is ALREADY written here, so + raising would report failure for an op that was applied: the caller never + gets it back to broadcast and ``set_widget``'s ``ValueError`` fallback can + attempt a second write. The pre-enrichment digest stays instead, and a + redelivery of the uncanonical envelope is refused in :func:`apply_op`'s + duplicate branch — before it can mutate anything. + """ + if applied is None: + return + digests = workflow.get("_applied_op_digests") + if not isinstance(digests, dict) or digests.get(op["op_id"]) != applied: + return + enriched = _applied_digest(op) + if enriched is not None: + digests[op["op_id"]] = enriched + def mint_id() -> int: """A leaderless, collision-free, int-typed identity for a node or link.""" @@ -1400,6 +1611,9 @@ def _set_widget_impl( if warnings: op["warnings"] = warnings workflow = apply_op(workflow, op, graph) + # Captured before the enrichment below mutates ``op``: it is how + # ``_rebind_op_digest`` recognizes the ledger entry as this call's own. + applied_digest = _applied_digest(op) if target.repair is not None: # The slot only exists after the repair (and the instance may sit # on a freshly forked definition): read it back from the document. @@ -1414,6 +1628,7 @@ def _set_widget_impl( # resolution is the pre-fork sibling, not the written one. written = _engine._resolve_node_path(workflow, list(target.segments), _engine._subgraph_defs_by_id(workflow)) op["promoted"]["host_widgets_values"] = _promoted.host_widgets_values(written) + _rebind_op_digest(workflow, op, applied_digest) return workflow, op if target.kind == "legacy_primitive": # A frontend-only PrimitiveNode has no catalog entry, so every @@ -1435,7 +1650,9 @@ def _set_widget_impl( **extra, ) workflow = apply_op(workflow, op, graph) + applied_digest = _applied_digest(op) op["promoted"]["host_widgets_values"] = list(src.get("widgets_values") or []) + _rebind_op_digest(workflow, op, applied_digest) return workflow, op if target.kind == "top" and target.redirected_from is not None: node_id = target.node.get("id") @@ -3326,11 +3543,57 @@ def unknown_size(class_type: Any) -> list[float] | None: def apply_op(workflow: dict, op: dict, graph) -> dict: """Replay one op onto ``workflow`` in place and return it. Idempotent: an - op whose ``op_id`` was already applied is a no-op.""" - had_applied_ops = "_applied_ops" in workflow - applied = workflow.setdefault("_applied_ops", []) - if op["op_id"] in applied: + op whose ``op_id`` was already applied is a no-op only for one payload.""" + # Validate the replay bookkeeping BEFORE anything mutates. All three values + # arrive from untrusted JSON — ``_applied_op_digests`` and ``_applied_ops`` + # are stripped on write but never checked on load, and ``op_id`` is whatever + # the envelope carried — and all three are indexed by the tail below. Left + # unchecked, a null/list/string ledger or a list-valued ``op_id`` raises + # ``TypeError``/``AttributeError`` AFTER the handler has written the + # document and appended to ``_applied_ops``, outside the rollback: a + # half-committed op whose deferred cleanup never ran. + op_id = op["op_id"] + if not isinstance(op_id, str): + raise ValueError(f"malformed_op: op_id must be a string, got {type(op_id).__name__}") + # Membership, not truthiness: an explicit ``null`` is PRESENT, and the + # ``setdefault`` in the tail below would hand back that ``None`` rather than + # a fresh dict. + if "_applied_op_digests" in workflow and not isinstance(workflow["_applied_op_digests"], dict): + raise ValueError("malformed_op: _applied_op_digests must be an object keyed by op_id") + if "_applied_ops" in workflow and not isinstance(workflow["_applied_ops"], list): + raise ValueError("malformed_op: _applied_ops must be an array of op_ids") + recorded_digests = workflow.get("_applied_op_digests") + recorded_ops = workflow.get("_applied_ops") + if op_id in (recorded_ops or ()): + recorded = (recorded_digests or {}).get(op_id) + if recorded is None: + # A legacy list-only ledger keeps its historical duplicate + # behaviour, and deliberately WITHOUT digesting the payload first: + # a replay whose envelope is not canonical must still be the + # documented no-op rather than a ``malformed_op``. + return workflow + if recorded != _op_digest(op): + raise ValueError(f"op_id_reuse: op_id {op_id!r} was already applied with a different payload") return workflow + # Digest BEFORE the ``setdefault`` below, so a first-seen op that + # canonicalization rejects leaves a document that had no ledger exactly as + # it found it — which is what the amendment's "rejects before mutation" + # wording promises. + digest = _op_digest(op) + # A deferred ``set_widget`` never reached ``_applied_ops`` — it waits in + # ``_deferred_widget_ops`` until a selector reveals its target — so gating + # on the applied list alone lets a SECOND op carrying the same ``op_id`` and + # a different payload straight through: it overwrites the pending entry (or + # applies at once), and the tail's ``deferred.pop`` then drops the original. + # No ``op_id_reuse`` is raised, so two replicas that received the pair in + # different orders diverge silently. + queued = workflow.get("_deferred_widget_ops") + waiting = queued.get(op_id) if isinstance(queued, dict) else None + # Cannot raise: a queued op was canonicalized on the way in. + if isinstance(waiting, dict) and _op_digest(waiting) != digest: + raise ValueError(f"op_id_reuse: op_id {op_id!r} is already deferred with a different payload") + had_applied_ops = "_applied_ops" in workflow + workflow.setdefault("_applied_ops", []) kind = op["op"] if kind != "insert_workflow" and "definitions" in op: raise ValueError(f"malformed_op: {kind} does not accept definitions") @@ -3388,10 +3651,11 @@ def apply_op(workflow: dict, op: dict, graph) -> dict: # binding above is stale for that kind and the reset's own op_id would be # written into a discarded list. Re-read, so a re-delivered reset_doc is a # no-op rather than a second wipe. - workflow.setdefault("_applied_ops", []).append(op["op_id"]) + workflow.setdefault("_applied_ops", []).append(op_id) + workflow.setdefault("_applied_op_digests", {})[op_id] = digest deferred = workflow.get("_deferred_widget_ops") if isinstance(deferred, dict): - deferred.pop(op["op_id"], None) + deferred.pop(op_id, None) if not deferred: workflow.pop("_deferred_widget_ops", None) if kind == "set_widget": @@ -3413,9 +3677,23 @@ def _retry_deferred_widget_ops(workflow: dict, graph) -> None: pending.pop(op_id, None) progressed = True continue + # Bind the id to its canonical payload here too: a deferred write + # reaches ``_applied_ops`` through this loop and never through + # ``apply_op``'s tail, so without this the reuse gate would read it + # as a legacy list-only entry and silently no-op a changed retry. + # Digested BEFORE the write, so an uncanonical queued payload — only + # reachable from a hand-edited document, since ``apply_op`` + # canonicalizes on the way in — is dropped like the non-dict entry + # above instead of raising half-way through its own apply. + digest = _applied_digest(op) + if digest is None: + pending.pop(op_id, None) + progressed = True + continue if _apply_set_widget(workflow, op, graph) is False: continue workflow.setdefault("_applied_ops", []).append(op_id) + workflow.setdefault("_applied_op_digests", {})[op_id] = digest pending.pop(op_id, None) progressed = True if not progressed: @@ -4114,6 +4392,7 @@ def _apply_reset_doc(workflow: dict, op: dict) -> None: workflow["last_node_id"] = 0 workflow["last_link_id"] = 0 workflow["_applied_ops"] = [] + workflow["_applied_op_digests"] = {} workflow["_widget_stamps"] = {} @@ -4259,6 +4538,7 @@ def canonical(workflow: dict) -> dict: """ w = copy.deepcopy(workflow) w.pop("_applied_ops", None) + w.pop("_applied_op_digests", None) w.pop("_widget_stamps", None) w.pop("_pending_dynamic_connects", None) w.pop("_deferred_widget_ops", None) @@ -4366,6 +4646,7 @@ def strip_internal(workflow: dict) -> dict: file writes, ``--stdout`` and batch output alike. """ workflow.pop("_applied_ops", None) + workflow.pop("_applied_op_digests", None) workflow.pop("_widget_stamps", None) workflow.pop("_pending_dynamic_connects", None) workflow.pop("_deferred_widget_ops", None) diff --git a/docs/op-vocabulary-v1.md b/docs/op-vocabulary-v1.md index cbb176a92..34b047e6d 100644 --- a/docs/op-vocabulary-v1.md +++ b/docs/op-vocabulary-v1.md @@ -354,11 +354,15 @@ local model can need it. * Every op carries `op_id`: uuid4 hex, minted by the **creator, before dispatch** (`_new_op`). Receivers never regenerate or rewrite an `op_id`. * `apply_op` records applied `op_id`s in the document's `_applied_ops` list and - drops any op whose `op_id` is already there. **Uniqueness scope is + their canonical digests in `_applied_op_digests`. An identical retry is a + no-op; the same `op_id` carrying a different canonical payload rejects with + `op_id_reuse`. A legacy document with only the list keeps its historical + duplicate behavior. See **Amendment v1.11** (§20) for the canonical form, + the bounds and the ledger contract. **Uniqueness scope is PER-WORKFLOW**: the same `op_id` can exist in two different workflow documents without interaction; within one document each op applies exactly once. -* `_applied_ops` (and `_widget_stamps`) are apply-time bookkeeping, stripped - before serialization to disk (`strip_internal`). +* `_applied_ops`, `_applied_op_digests` (and `_widget_stamps`) are apply-time + bookkeeping, stripped before serialization to disk (`strip_internal`). ## 3. Conflict rules @@ -1290,3 +1294,109 @@ No operation kind is added or removed. `FROZEN_OPS`, `DEFERRED_OPS`, and `BATCHABLE_OPS` are unchanged. Downstream consumers that move their pinned vocabulary SHA to v1.10 must preserve `subgraph_reuse` conflict metadata even though replay itself continues to insert `op.node` opaquely. + +## 20. Amendment v1.11 — 2026-10-24 (canonical op identity: `op_id` binds one payload) + +**`op_id` identifies one immutable semantic-op envelope, not just one delivery.** +Before this amendment `apply_op` dropped any op whose `op_id` was already in +`_applied_ops` without looking at the payload, so a sender that reused an id +with a changed payload had its second edit silently discarded — and two +replicas that received the pair in different orders ended up with different +documents and no error on either side. This mirrors comfy-multi-player schema +**Amendment A8**; the two implementations must agree byte-for-byte on the +canonical form, because the digest is the only thing that distinguishes an +identical retry from reuse. + +### 20.1 The canonical form + +Compact JSON over the **whole envelope**: + +* object keys sorted by **Unicode code point** at every depth; array order + preserved; no insignificant whitespace (`,`/`:` separators, no spaces). The + same caveat as 8.1 applies: for ASCII, JS UTF-16 `<` agrees with code-point + order, but above the Basic Multilingual Plane it does not, so an + implementation must sort by code point and not rely on + `Object.keys(...).sort()`'s default comparator. Every key an op envelope + carries today is ASCII, so this only bites a dict widget whose own keys are + not; +* **numbers rendered as ECMAScript renders them** (`Number::toString`), not as + Python's float repr does. An integral float is `20`, not `20.0`; negative zero + is `0`; an exponent is `1e-7`, not `1e-07`. This is load-bearing rather than + cosmetic: the counterpart builds the canonical form with `JSON.stringify`, so + without it a FLOAT widget op relayed through JS digests differently on the two + sides and an identical redelivery is rejected as `op_id_reuse`; +* **every non-finite number renders as `null`**, which is what `JSON.stringify` + does. The alternative — rejecting the op — made a widget whose *current* value + is `NaN`/`Infinity` impossible to overwrite, because the op captures that + value in `old` (and `json.load` accepts both spellings on the way in); +* a **lone surrogate** renders as a `\udXXX` escape, matching ES2019 + well-formed `JSON.stringify`. Emitted raw it cannot be UTF-8 encoded at all, + so the digest died with a bare `UnicodeEncodeError` outside the + `malformed_op:` prefix callers classify errors by. + +The ledger records the lowercase SHA-256 hex digest of that string. + +### 20.2 The ledger contract + +* `_applied_op_digests` maps `op_id` → digest, beside `_applied_ops`. Both are + stripped by `canonical()` and `strip_internal()`, and `reset_doc` clears both + as one history barrier. +* **Rejects before mutation.** `op_id`, `_applied_ops` and `_applied_op_digests` + all arrive from untrusted JSON and are all validated before any handler runs: + a non-string `op_id`, or a ledger that is not an object/array, is + `malformed_op`. So is a payload canonicalization refuses. A document that + arrived without a ledger is never left carrying an empty one by a rejected op. +* **A deferred op is already bound.** A `set_widget` waiting in + `_deferred_widget_ops` for a selector to reveal its target has not reached + `_applied_ops`, so the gate checks the deferred queue too; a second payload on + that `op_id` rejects instead of overwriting the pending entry. +* **The enriched envelope is what is bound.** A promoted or `legacy_primitive` + `set_widget` only learns the host's materialized `host_widgets_values` by + reading back the document the write produced, so the ledger entry is corrected + to the envelope the caller actually broadcasts. If that enriched envelope is + not canonical the earlier digest stands: the document is already written, and + reporting failure for an applied op is worse than holding a digest a + redelivery will refuse in the pre-mutation branch. +* **Legacy documents are untouched.** One carrying `_applied_ops` and no + `_applied_op_digests` keeps the historical drop-the-duplicate behavior, and + deliberately without digesting the replayed payload first — a non-canonical + replay must still be the documented no-op rather than a new error. + +### 20.3 Bounds + +Canonicalization is bounded to **64 levels of nesting**, **4,096 entries per +collection** and **262,144 approximate cost units**. These are not chosen here: +they are mirrored exactly from `comfy-multi-player`'s `src/limits.ts` at +`221b2b2e470c79afaface24b1edcb53cdafc8922` (`MAX_PAYLOAD_DEPTH`, +`MAX_COLLECTION_ENTRIES`, `MAX_OP_COST`), cited by SHA rather than by branch. + +They exist only so a hostile or runaway payload cannot pin a replica inside +canonicalization, and every ordinary op has to pass. "The digest is computed on +every apply, not only on a reuse comparison" is true of **both** sides — CMP's +A8 canonicalizer charges the whole envelope in `applyOps` before its dedupe gate +— so a looser bound on one side rescues no edit. It only moves where an +oversized op fails, from a local `malformed_op` to a `malformed_op` from the +shared package after this side already accepted the payload. A comfy-cli-only +budget was briefly carried here (1,048,576 / 33,554,432) and is withdrawn as +exactly the cross-implementation validation drift one shared op package exists +to prevent. If a real edit needs more room, widen it in `comfy-multi-player` and +move this pin. + +Measured headroom at the shared values: the widest op this vocabulary mints from +a real workflow fixture is ~1.6K cost units over 13 entries, under 1% of the +budget. The three edits previously said to need a looser bound bind only well +past anything observed — `clear`/`reset_doc` at more than 4,096 nodes in one +document, a hub `delete_node` at more than 4,096 incident links on one node, and +a widget string at 131,064 characters when the op also captures an equal-sized +`old` (262,133 without one). The largest workflow in the comfy-cli fixtures has +11 nodes. + +The asymmetric case is the one to keep in mind: `{value: 25}` followed by the +same `op_id` carrying `{value: 30}` rejects, while the same payload with its keys +in a different insertion order is an ordinary no-op. + +**No change to §§1, 3-8.** No op kind was added, removed, or re-scoped; +`FROZEN_OPS` / `DEFERRED_OPS` / `BATCHABLE_OPS` are untouched. §2's identity +bullet now cites this section. Downstream repos pinning this document by SHA +move the SHA together with their comfy-cli pin, and must move their CMP A8 pin +with it. diff --git a/tests/comfy_cli/command/test_workflow_edit.py b/tests/comfy_cli/command/test_workflow_edit.py index deb73b52a..7395b5a3e 100644 --- a/tests/comfy_cli/command/test_workflow_edit.py +++ b/tests/comfy_cli/command/test_workflow_edit.py @@ -977,6 +977,7 @@ def test_repair_replays_identically_on_a_fresh_copy(self): replayed = workflow_ops.apply_op(copy.deepcopy(base), op, _graph()) for w in (applied, replayed): w.pop("_applied_ops", None) + w.pop("_applied_op_digests", None) w.pop("_widget_stamps", None) assert json.dumps(replayed, sort_keys=True) == json.dumps(applied, sort_keys=True) diff --git a/tests/comfy_cli/command/test_workflow_edit_legacy_proxy.py b/tests/comfy_cli/command/test_workflow_edit_legacy_proxy.py index beab7ef45..168269187 100644 --- a/tests/comfy_cli/command/test_workflow_edit_legacy_proxy.py +++ b/tests/comfy_cli/command/test_workflow_edit_legacy_proxy.py @@ -69,6 +69,7 @@ def _interior(wf: dict, instance_id: Any, inner_id: Any) -> dict: def _stripped(wf: dict) -> str: w = copy.deepcopy(wf) w.pop("_applied_ops", None) + w.pop("_applied_op_digests", None) w.pop("_widget_stamps", None) return json.dumps(w, sort_keys=True) diff --git a/tests/comfy_cli/command/test_workflow_edit_promoted.py b/tests/comfy_cli/command/test_workflow_edit_promoted.py index d84e61931..fd4de09b7 100644 --- a/tests/comfy_cli/command/test_workflow_edit_promoted.py +++ b/tests/comfy_cli/command/test_workflow_edit_promoted.py @@ -95,6 +95,7 @@ def test_promoted_address_writes_the_host_value(graph): assert "path" not in op assert op["promoted"]["value_index"] == 1 assert op["promoted"]["host_widgets_values"] == expected + assert workflow_ops.apply_op(wf, op, graph) is wf def test_interior_address_of_a_promoted_widget_is_redirected_to_the_host(graph): @@ -442,6 +443,7 @@ def test_legacy_primitive_write_carries_a_positional_payload(graph): assert op["promoted"] == {"value_index": 0, "instance_path": [str(legacy)], "host_widgets_values": [512, "fixed"]} replayed = workflow_ops.apply_op(copy.deepcopy(base), op, graph) assert _node(replayed, legacy)["widgets_values"] == [512, "fixed"] + assert workflow_ops.apply_op(wf, op, graph) is wf def test_later_connect_to_a_materialized_promoted_input_stays_on_the_declared_register(graph): diff --git a/tests/comfy_cli/test_dynamic_combo_other_option_widget.py b/tests/comfy_cli/test_dynamic_combo_other_option_widget.py index fcedd81da..fa91a98f8 100644 --- a/tests/comfy_cli/test_dynamic_combo_other_option_widget.py +++ b/tests/comfy_cli/test_dynamic_combo_other_option_widget.py @@ -565,3 +565,71 @@ def test_missing_nested_selector_uses_the_active_outer_options_default(): assert "model.mode='quick'" in msg, msg assert "'detailed'" in msg, msg assert wf["nodes"][0]["widgets_values"] == before + + +def test_a_deferred_write_is_bound_to_its_canonical_payload_once_applied(graph): + """A deferred write reaches ``_applied_ops`` through the retry loop, not + through ``apply_op``'s tail, so the op-id/payload binding (op-vocabulary + Amendment v1.10) has to be recorded there too. Without it the reuse gate + sees a list-only entry and silently no-ops a changed-payload retry.""" + wf, node_id = _fresh(graph) + hidden_write = workflow_ops._new_op( + "set_widget", + "agent-a", + 1, + node_id=node_id, + widget="model.prompt_expansion_mode", + value="quality", + ) + workflow_ops.apply_op(wf, hidden_write, graph) + assert hidden_write["op_id"] in wf["_deferred_widget_ops"] + + wf, _ = workflow_ops.set_widget(wf, graph, node_id, "model", "MiniMax H3 Max", actor="agent-b") + + assert hidden_write["op_id"] in wf["_applied_ops"] + assert wf["_applied_op_digests"][hidden_write["op_id"]] == workflow_ops._op_digest(hidden_write) + + # An identical redelivery stays a no-op; the same id with a changed payload + # is refused rather than dropped. + assert workflow_ops.apply_op(wf, copy.deepcopy(hidden_write), graph) is wf + changed = copy.deepcopy(hidden_write) + changed["value"] = "balanced" + before = copy.deepcopy(wf) + with pytest.raises(ValueError, match="^op_id_reuse:"): + workflow_ops.apply_op(wf, changed, graph) + assert wf == before + + +def test_a_second_payload_on_a_deferred_op_id_is_refused_rather_than_overwriting_it(graph): + """The reuse gate used to read ``_applied_ops`` only. A deferred write sits + in ``_deferred_widget_ops`` and nowhere else, so a second op carrying the + same ``op_id`` and a different payload went straight past the gate: it + overwrote the pending entry (or applied at once), and the tail's + ``deferred.pop`` then dropped the original. No ``op_id_reuse`` was raised, + so two replicas that received the pair in different orders diverged. + """ + wf, node_id = _fresh(graph) + hidden_write = workflow_ops._new_op( + "set_widget", + "agent-a", + 1, + node_id=node_id, + widget="model.prompt_expansion_mode", + value="quality", + ) + workflow_ops.apply_op(wf, hidden_write, graph) + assert hidden_write["op_id"] in wf["_deferred_widget_ops"] + + changed = copy.deepcopy(hidden_write) + changed["value"] = "balanced" + before = copy.deepcopy(wf) + with pytest.raises(ValueError, match="^op_id_reuse:"): + workflow_ops.apply_op(wf, changed, graph) + assert wf == before + # The original is still the queued payload, so revealing the target applies + # what was actually sent first. + assert wf["_deferred_widget_ops"][hidden_write["op_id"]]["value"] == "quality" + + # An identical redelivery while deferred stays a no-op on the queue. + workflow_ops.apply_op(wf, copy.deepcopy(hidden_write), graph) + assert wf["_deferred_widget_ops"][hidden_write["op_id"]]["value"] == "quality" diff --git a/tests/comfy_cli/test_op_id_reuse_contract.py b/tests/comfy_cli/test_op_id_reuse_contract.py new file mode 100644 index 000000000..b982740a7 --- /dev/null +++ b/tests/comfy_cli/test_op_id_reuse_contract.py @@ -0,0 +1,384 @@ +"""``op_id`` is bound to one canonical payload (op-vocabulary Amendment v1.10 / CMP A8). + +The graph here is a real :class:`~comfy_cli.cql.engine.Graph` built from a +one-node ``object_info``, not a hand-rolled stub. ``apply_op`` -> ``set_widget`` +goes through the catalog (``graph.node`` for the schema, ``widget_order_for_node`` +for the positional slot), so a stub that answers only the widget order cannot +reach the write at all. +""" + +from __future__ import annotations + +import copy +import json +from typing import Any + +import pytest + +from comfy_cli import workflow_ops as ops +from comfy_cli.cql.engine import Graph + +_OBJECT_INFO: dict[str, Any] = { + "KSampler": { + "input": {"required": {"steps": ["INT", {"default": 20, "min": 1, "max": 10000}]}}, + "input_order": {"required": ["steps"]}, + "output": ["LATENT"], + "output_name": ["LATENT"], + "category": "sampling", + "display_name": "KSampler", + "python_module": "nodes", + }, +} + + +@pytest.fixture +def graph() -> Graph: + return Graph.from_object_info(_OBJECT_INFO) + + +def _workflow(): + return {"nodes": [{"id": 7, "type": "KSampler", "widgets_values": [20], "inputs": [], "outputs": []}], "links": []} + + +def _op(value=25): + return { + "op": "set_widget", + "op_id": "same0000000000000000000000000000", + "actor": "human:a", + "base_version": 5, + "stamp": [5, "human:a"], + "node_id": 7, + "widget": "steps", + "value": value, + } + + +def test_changed_payload_reusing_op_id_rejects_without_mutation(graph): + workflow = _workflow() + ops.apply_op(workflow, _op(25), graph) + before = copy.deepcopy(workflow) + with pytest.raises(ValueError, match="^op_id_reuse:"): + ops.apply_op(workflow, _op(30), graph) + assert workflow == before + + +def test_canonical_digest_matches_cmp_a8_vector_and_key_order_is_immaterial(graph): + workflow = _workflow() + first = _op() + ops.apply_op(workflow, first, graph) + assert ( + workflow["_applied_op_digests"][first["op_id"]] + == "d395ba92dd991c07c0f790b974b3d4f34e3b29c71a26621634d75de31040a431" + ) + reordered = dict(reversed(list(first.items()))) + assert ops.apply_op(workflow, reordered, graph) is workflow + + +def test_legacy_list_only_replay_returns_before_digesting_payload(graph): + workflow = _workflow() + workflow["_applied_ops"] = [_op()["op_id"]] + replay = _op() + replay["value"] = object() + assert ops.apply_op(workflow, replay, graph) is workflow + + +def test_canonical_ignores_replay_digest_metadata(graph): + workflow = _workflow() + applied = copy.deepcopy(workflow) + ops.apply_op(applied, _op(), graph) + applied["nodes"] = copy.deepcopy(workflow["nodes"]) + assert ops.canonical(applied) == ops.canonical(workflow) + + +# --- Amendment v1.10: what the canonical form has to agree with ------------- + + +def test_numbers_are_rendered_the_way_ecmascript_renders_them(): + """The counterpart implementation (CMP A8) canonicalizes with + ``JSON.stringify``, so the digest only matches cross-implementation if the + numbers are spelled ECMAScript's way. Python's own float repr disagrees on + exactly the shapes a FLOAT widget produces: an integral float, negative + zero, and an exponent. + """ + assert ops._canonical_op({"v": 20.0}) == '{"v":20}' + assert ops._canonical_op({"v": -0.0}) == '{"v":0}' + assert ops._canonical_op({"v": 1e-7}) == '{"v":1e-7}' + assert ops._canonical_op({"v": 1e-6}) == '{"v":0.000001}' + assert ops._canonical_op({"v": 1e20}) == '{"v":100000000000000000000}' + assert ops._canonical_op({"v": 1e21}) == '{"v":1e+21}' + assert ops._canonical_op({"v": 1.5}) == '{"v":1.5}' + assert ops._canonical_op({"v": 5e-324}) == '{"v":5e-324}' + + +def test_object_keys_sort_by_code_point_not_by_utf16_code_unit(): + """§8.1's caveat applies to the key sort too: for ASCII, JS UTF-16 ``<`` + agrees with code-point order, but above the Basic Multilingual Plane it does + not. ``Object.keys(...).sort()`` puts U+1F600 before U+FF00; code-point + order is the other way round, and code-point order is what the amendment + pins. Only a dict widget whose own keys are non-ASCII can reach this, since + every key an op envelope carries is ASCII. + """ + assert ops._canonical_op({"\U0001f600": 1, "\uff00": 2}) == '{"\uff00":2,"\U0001f600":1}' + + +def test_a_float_widget_relayed_through_js_digests_the_same(graph): + """A FLOAT op whose value is integral comes back from a JS relay as an int. + Both spellings have to digest identically, or the identical redelivery is + rejected as ``op_id_reuse``. + """ + workflow = _workflow() + as_float = _op(20.0) + ops.apply_op(workflow, as_float, graph) + relayed = _op(20) # what `JSON.parse(JSON.stringify(op))` yields + assert ops._op_digest(relayed) == ops._op_digest(as_float) + assert ops.apply_op(workflow, relayed, graph) is workflow + + +def test_non_finite_numbers_canonicalize_as_null_like_json_stringify(graph): + """``json.load`` accepts NaN/Infinity, so a document can hold one and an op + captures it in ``old`` or in the host's ``host_widgets_values``. The earlier + ``allow_nan=False`` made every such write raise; ``JSON.stringify`` renders + them as ``null``, and matching that is both cross-implementation correct and + what un-breaks overwriting a NaN widget. + """ + assert ops._canonical_op({"v": float("nan")}) == '{"v":null}' + assert ops._canonical_op({"v": float("inf")}) == '{"v":null}' + assert ops._canonical_op({"v": [float("-inf")]}) == '{"v":[null]}' + + workflow = _workflow() + workflow["nodes"][0]["widgets_values"] = [float("nan")] + workflow, op = ops.set_widget(workflow, graph, 7, "steps", 30) + assert workflow["nodes"][0]["widgets_values"] == [30] + assert workflow["_applied_op_digests"][op["op_id"]] == ops._op_digest(op) + + +def test_a_lone_surrogate_digests_instead_of_dying_on_the_utf8_encode(): + """``json.loads('"\\ud800"')`` produces a lone surrogate, which cannot be + UTF-8 encoded at all. ES2019 well-formed stringify escapes it; so do we, so + the digest stays total instead of raising a bare ``UnicodeEncodeError`` + outside the ``malformed_op:`` contract callers classify errors by. + """ + op = {"v": json.loads('"\\ud800"')} + assert ops._canonical_op(op) == '{"v":"\\ud800"}' + assert len(ops._op_digest(op)) == 64 + + +def test_the_bounds_mirror_the_shared_op_package_values(): + """These are MIRRORED from the shared op package, not chosen here. + + ``comfy-multi-player``'s ``src/limits.ts`` at + ``221b2b2e470c79afaface24b1edcb53cdafc8922`` fixes ``MAX_PAYLOAD_DEPTH`` at + 64, ``MAX_COLLECTION_ENTRIES`` at 4096 and ``MAX_OP_COST`` at 262144, and + its A8 canonicalizer charges the whole envelope on EVERY apply, before the + dedupe gate (``applyOps`` -> ``opDigest`` -> ``canonicalJson`` -> + ``opBoundsRefusal``). So a looser bound here cannot rescue an oversized + edit; it only lets comfy-cli accept a payload the shared package then + refuses as ``malformed_op``. Widen it there and move this pin — never here + alone. + """ + assert ops._MAX_OP_PAYLOAD_DEPTH == 64 + assert ops._MAX_OP_COLLECTION_ENTRIES == 4096 + assert ops._MAX_OP_COST == 262_144 + + +def test_ordinary_large_payloads_are_not_rejected_by_the_canonicalization_bounds(): + """The bounds exist to stop a runaway payload, not to cap a real edit, and + at the shared values they still clear every real edit by two orders of + magnitude. The widest op the frozen vocabulary mints from an actual + workflow fixture in this repo measures ~1.6K cost units and 13 entries. + + The three edits cited as needing a looser bound are exercised here at sizes + far past anything observed: a ``clear`` enumerating a thousand node ids, a + hub ``delete_node`` enumerating a thousand incident links, and a 60K-char + prompt value whose op also captures an equal-sized ``old``. + """ + assert len(ops._op_digest({"op": "clear", "removed_nodes": list(range(1_000))})) == 64 + assert len(ops._op_digest({"op": "delete_node", "node_id": 7, "removed_links": list(range(1_000))})) == 64 + assert len(ops._op_digest({"op": "set_widget", "value": "x" * 60_000, "old": "y" * 60_000})) == 64 + + +def test_the_shared_bounds_bind_only_past_any_observed_workflow(): + """Where the mirrored bounds actually bite, pinned so the next reader does + not have to re-derive it before deciding whether an edit is really capped. + + ``removed_nodes``/``removed_links`` are flat lists, so the ENTRY cap is what + binds them, at 4,097 — a workflow with more than 4,096 nodes, or a single + node carrying more than 4,096 incident links. The largest workflow in this + repo's fixtures has 11 nodes. A widget string is charged by length, so the + COST budget binds it, and an op that captures an equal-sized ``old`` halves + the headroom: 131,064 characters for the bare value/old pair, a little less + once the envelope's own fields are charged. That is a 128 KiB string in one + widget. + """ + assert len(ops._op_digest({"removed_nodes": [0] * ops._MAX_OP_COLLECTION_ENTRIES})) == 64 + with pytest.raises(ValueError, match="^malformed_op: array exceeds"): + ops._op_digest({"removed_nodes": [0] * (ops._MAX_OP_COLLECTION_ENTRIES + 1)}) + + assert len(ops._op_digest({"value": "x" * 131_064, "old": "y" * 131_064})) == 64 + with pytest.raises(ValueError, match="cost budget$"): + ops._op_digest({"value": "x" * 131_065, "old": "y" * 131_065}) + + +def test_canonicalization_still_refuses_a_runaway_payload(): + nested: Any = "leaf" + for _ in range(ops._MAX_OP_PAYLOAD_DEPTH + 2): + nested = [nested] + with pytest.raises(ValueError, match="^payload_too_deep:"): + ops._op_digest({"v": nested}) + with pytest.raises(ValueError, match="^malformed_op: array exceeds"): + ops._op_digest({"v": [0] * (ops._MAX_OP_COLLECTION_ENTRIES + 1)}) + # Cost-only overflow: 64 entries is well inside the entry cap, so this + # reaches the cost branch rather than tripping the entry branch first. + with pytest.raises(ValueError, match="cost budget$"): + ops._op_digest({"v": ["x" * 4096] * 64}) + with pytest.raises(ValueError, match="^malformed_op: op payload is not canonical JSON:"): + ops._op_digest({"v": object()}) + + +# --- Amendment v1.10: rejecting before mutation ----------------------------- + + +def test_a_rejected_first_seen_op_leaves_no_ledger_behind(graph): + """ "Rejects before mutation" has to include the ledger itself: a document + that arrived without ``_applied_ops`` must not come back carrying an empty + one because canonicalization refused the op. + """ + workflow = _workflow() + before = copy.deepcopy(workflow) + deep = _op() + deep["value"] = [["x"] * 1] + for _ in range(ops._MAX_OP_PAYLOAD_DEPTH + 2): + deep["value"] = [deep["value"]] + with pytest.raises(ValueError, match="^payload_too_deep:"): + ops.apply_op(workflow, deep, graph) + assert workflow == before + assert "_applied_ops" not in workflow + + +@pytest.mark.parametrize( + ("ledger", "message"), + [ + (None, "^malformed_op: _applied_op_digests must be an object"), + ([], "^malformed_op: _applied_op_digests must be an object"), + ("{}", "^malformed_op: _applied_op_digests must be an object"), + ], +) +def test_a_malformed_digest_ledger_is_refused_before_the_handler_runs(graph, ledger, message): + """The ledger is stripped on write but never checked on load. Indexing it in + ``apply_op``'s tail raised ``TypeError`` after the handler had already + written the document and appended to ``_applied_ops`` — outside the + rollback, so the op stayed half-committed. + """ + workflow = _workflow() + workflow["_applied_op_digests"] = ledger + before = copy.deepcopy(workflow) + with pytest.raises(ValueError, match=message): + ops.apply_op(workflow, _op(), graph) + assert workflow == before + + +def test_a_truthy_non_dict_ledger_is_refused_on_the_reuse_branch_too(graph): + workflow = _workflow() + applied = _op() + ops.apply_op(workflow, applied, graph) + workflow["_applied_op_digests"] = "not a mapping" + with pytest.raises(ValueError, match="^malformed_op: _applied_op_digests must be an object"): + ops.apply_op(workflow, applied, graph) + + +@pytest.mark.parametrize("op_id", [["a"], {"a": 1}, 7, None]) +def test_a_non_string_op_id_is_refused_before_the_handler_runs(graph, op_id): + workflow = _workflow() + before = copy.deepcopy(workflow) + bad = _op() + bad["op_id"] = op_id + with pytest.raises(ValueError, match="^malformed_op: op_id must be a string"): + ops.apply_op(workflow, bad, graph) + assert workflow == before + + +def test_a_malformed_applied_ops_list_is_refused_before_the_handler_runs(graph): + workflow = _workflow() + workflow["_applied_ops"] = {"not": "a list"} + before = copy.deepcopy(workflow) + with pytest.raises(ValueError, match="^malformed_op: _applied_ops must be an array"): + ops.apply_op(workflow, _op(), graph) + assert workflow == before + + +# --- Amendment v1.10: the enriched envelope --------------------------------- + + +def _primitive_canvas(value="old prompt") -> dict: + """A frontend-only ``PrimitiveNode``: no catalog entry, so ``set_widget`` + takes the ``legacy_primitive`` path that enriches the op with the host's + positional array AFTER ``apply_op`` has already recorded a digest. + """ + return { + "nodes": [ + { + "id": 51, + "type": "PrimitiveNode", + "outputs": [{"name": "STRING", "type": "STRING", "links": [], "widget": {"name": "text"}}], + "widgets_values": [value], + } + ], + "links": [], + } + + +def test_the_ledger_binds_the_enriched_envelope_the_caller_broadcasts(graph): + """``_set_widget_impl`` finishes the op after ``apply_op`` returns, because + ``host_widgets_values`` can only be read back out of the document the write + produced. If the ledger kept the pre-enrichment digest, a redelivery of the + payload that was actually broadcast would be rejected as ``op_id_reuse``. + """ + workflow, op = ops.set_widget(_primitive_canvas(), graph, 51, "value", "a lighthouse at dusk") + assert op["promoted"]["host_widgets_values"] == ["a lighthouse at dusk"] + assert workflow["_applied_op_digests"][op["op_id"]] == ops._op_digest(op) + # The broadcast envelope round-trips as the documented no-op... + assert ops.apply_op(workflow, copy.deepcopy(op), graph) is workflow + # ...and the same id with a changed payload is still refused. + changed = copy.deepcopy(op) + changed["value"] = "a lighthouse at dawn" + with pytest.raises(ValueError, match="^op_id_reuse:"): + ops.apply_op(workflow, changed, graph) + + +def test_a_nan_host_widget_no_longer_fails_an_op_that_already_applied(graph): + """The enriched digest runs after the write has committed, so raising there + reported failure for an op the document had already accepted: the caller + never got it back to broadcast and ``set_widget``'s ``ValueError`` fallback + could attempt a second write. + """ + workflow, op = ops.set_widget(_primitive_canvas(float("nan")), graph, 51, "value", "lighthouse") + assert workflow["nodes"][0]["widgets_values"] == ["lighthouse"] + assert workflow["_applied_op_digests"][op["op_id"]] == ops._op_digest(op) + + +def test_rebinding_a_digest_is_inert_when_the_entry_is_not_this_calls_own(): + """The three cases ``_rebind_op_digest`` must leave alone: no ledger at all + (the op deferred, so nothing recorded it), an entry belonging to a different + payload (a duplicate), and an uncanonical enriched envelope (the document is + already written, so raising is worse than keeping the older digest). + """ + op = _op() + applied = ops._op_digest(op) + enriched = dict(op, promoted={"host_widgets_values": ["x"]}) + + no_ledger: dict = {} + ops._rebind_op_digest(no_ledger, enriched, applied) + assert no_ledger == {} + + someone_elses = {"_applied_op_digests": {op["op_id"]: "a" * 64}} + ops._rebind_op_digest(someone_elses, enriched, applied) + assert someone_elses["_applied_op_digests"][op["op_id"]] == "a" * 64 + + uncanonical = dict(enriched, promoted={"host_widgets_values": [object()]}) + ours = {"_applied_op_digests": {op["op_id"]: applied}} + ops._rebind_op_digest(ours, uncanonical, applied) + assert ours["_applied_op_digests"][op["op_id"]] == applied + + ours = {"_applied_op_digests": {op["op_id"]: applied}} + ops._rebind_op_digest(ours, enriched, applied) + assert ours["_applied_op_digests"][op["op_id"]] == ops._op_digest(enriched)