From 3d210ab00d48659c9fe94f3524d925fc24e75bc7 Mon Sep 17 00:00:00 2001 From: Glary-Bot Date: Sun, 18 Oct 2026 23:16:54 +0000 Subject: [PATCH 1/5] Fix MTPHead unpack crash in prefetch queue cleanup (CORE-644) --- comfy/model_prefetch.py | 13 +++-- comfy/text_encoders/qwen35.py | 2 +- .../comfy_test/model_prefetch_cleanup_test.py | 54 +++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 tests-unit/comfy_test/model_prefetch_cleanup_test.py diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 2b5cd042aed..5a2b133323d 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -155,15 +155,18 @@ def cleanup_prefetch_queues(): global MALLOC_GRAPH_USED cleanup_malloc_graph() - for queue in PREFETCH_QUEUES: + queues = PREFETCH_QUEUES + # Reset before walking so a failing entry can't leave stale queues behind for every later node. + PREFETCH_QUEUES = [] + for queue in queues: for entry in queue: - if entry is None or not isinstance(entry, tuple): + # Only pinned slots are (offload_stream, (module, comfy_modules)); slots not yet + # reached still hold the raw module or module group passed to make_prefetch_queue. + if not isinstance(entry, tuple) or len(entry) != 2 or not isinstance(entry[1], tuple): continue - _, prefetch_state = entry - prefetched_module, comfy_modules = prefetch_state + prefetched_module, comfy_modules = entry[1] if comfy_modules is not None: cleanup_prefetched_modules(prefetched_module, comfy_modules) - PREFETCH_QUEUES = [] GRAPH_WARMED_MODULES.clear() if MALLOC_GRAPH_USED: logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 0a23688241b..7396f81a979 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -872,7 +872,7 @@ def draft(token, hidden, p): draft_freqs.copy_(freqs_at(p)) host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed) queue = comfy.model_prefetch.make_prefetch_queue( - [None if host_rows else embed, (head, self.mtp)], device, + [None if host_rows else embed, [head, self.mtp]], device, {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars}) def embed_core(): diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py new file mode 100644 index 00000000000..84e1e665dd5 --- /dev/null +++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py @@ -0,0 +1,54 @@ +import pytest +import torch + +from comfy.cli_args import args as cli_args + +if not torch.cuda.is_available(): + cli_args.cpu = True + +import comfy.model_prefetch as model_prefetch # noqa: E402 + + +class MTPHead(torch.nn.Module): + pass + + +@pytest.fixture(autouse=True) +def _reset_queues(): + model_prefetch.PREFETCH_QUEUES = [] + yield + model_prefetch.PREFETCH_QUEUES = [] + + +def _prefetched(module): + module._prefetch = {"signature": None} + return (None, (module, [module])) + + +@pytest.mark.parametrize("pending_unit", [tuple, list]) +def test_cleanup_skips_unconsumed_grouped_unit(pending_unit): + # Qwen3.5 MTP draft queue interrupted after the embed pop: the embed slot has + # been pinned, the grouped (head, mtp) slot has not been reached yet. + embed = torch.nn.Embedding(4, 4) + head = torch.nn.Linear(4, 4) + queue = [_prefetched(embed), pending_unit((head, MTPHead())), None] + model_prefetch.PREFETCH_QUEUES = [queue] + + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + assert not hasattr(embed, "_prefetch") + + +def test_cleanup_resets_queues_when_entry_cleanup_fails(monkeypatch): + def boom(module, comfy_modules): + raise RuntimeError("unpin failed") + + monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", boom) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(torch.nn.Linear(4, 4)), None]] + + with pytest.raises(RuntimeError): + model_prefetch.cleanup_prefetch_queues() + # A failed cleanup must not poison every later node on the worker. + assert model_prefetch.PREFETCH_QUEUES == [] + model_prefetch.cleanup_prefetch_queues() From d033958deb28f5f72f55abaa37336fdb8fe21431 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 21:10:18 +0000 Subject: [PATCH 2/5] Test that prefetch cleanup skips an unreached grouped head slot Covers the window between popping the consumed slot and pinning the next one, where the queue head is still the raw (lm_head, MTPHead) group. Master's head-only cleanup still raises the CORE-644 TypeError there. --- tests-unit/comfy_test/model_prefetch_cleanup_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py index 84e1e665dd5..24868085e6b 100644 --- a/tests-unit/comfy_test/model_prefetch_cleanup_test.py +++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py @@ -52,3 +52,15 @@ def boom(module, comfy_modules): # A failed cleanup must not poison every later node on the worker. assert model_prefetch.PREFETCH_QUEUES == [] model_prefetch.cleanup_prefetch_queues() + + +@pytest.mark.parametrize("pending_unit", [tuple, list]) +def test_cleanup_skips_unreached_grouped_head(pending_unit): + # Interrupted between popping the consumed slot and pinning the next one: the + # head of the queue is still the raw (head, mtp) group, not a pinned slot. + queue = [pending_unit((torch.nn.Linear(4, 4), MTPHead())), None] + model_prefetch.PREFETCH_QUEUES = [queue] + + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] From 520869e9c871132e898b11d43b371f298f3e1303 Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 29 Oct 2026 21:19:34 +0000 Subject: [PATCH 3/5] Keep unpinning other prefetch queues when one queue's cleanup fails cleanup_prefetch_queues() now resets the queue list before walking it, so an exception part way through would drop the remaining queues without unpinning them. Clean up every queue, finish the graph-state reset, then re-raise the first error. --- comfy/model_prefetch.py | 10 +++++++++- .../comfy_test/model_prefetch_cleanup_test.py | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 4c40a61a933..b1aef073ff7 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -160,6 +160,7 @@ def cleanup_prefetch_queues(): queues = PREFETCH_QUEUES # Reset before walking so a failing entry can't leave stale queues behind for every later node. PREFETCH_QUEUES = [] + first_error = None for queue in queues: # Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later # entries, and a head that was not reached yet, still hold the raw module or module group. @@ -167,13 +168,20 @@ def cleanup_prefetch_queues(): if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple): continue prefetched_module, comfy_modules = head[1] - cleanup_prefetched_modules(prefetched_module, comfy_modules) + try: + cleanup_prefetched_modules(prefetched_module, comfy_modules) + except Exception as e: + # Keep unpinning the other queues; report the first failure once they are done. + if first_error is None: + first_error = e GRAPH_WARMED_MODULES.clear() if MALLOC_GRAPH_USED: logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES) MALLOC_GRAPH_BREAKS = 0 MALLOC_GRAPH_ROGUES = 0 MALLOC_GRAPH_USED = False + if first_error is not None: + raise first_error def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None): malloc_graph = MALLOC_GRAPHS.get(threading.get_ident()) diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py index 24868085e6b..7396db23a62 100644 --- a/tests-unit/comfy_test/model_prefetch_cleanup_test.py +++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py @@ -64,3 +64,23 @@ def test_cleanup_skips_unreached_grouped_head(pending_unit): model_prefetch.cleanup_prefetch_queues() assert model_prefetch.PREFETCH_QUEUES == [] + + +def test_cleanup_unpins_later_queues_when_an_earlier_one_fails(monkeypatch): + real_cleanup = model_prefetch.cleanup_prefetched_modules + failing = torch.nn.Linear(4, 4) + later = torch.nn.Linear(4, 4) + + def cleanup(module, comfy_modules): + if module is failing: + raise RuntimeError("unpin failed") + real_cleanup(module, comfy_modules) + + monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", cleanup) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(failing), None], [_prefetched(later), None]] + + with pytest.raises(RuntimeError, match="unpin failed"): + model_prefetch.cleanup_prefetch_queues() + # The first failure is re-raised, but only after every other queue was cleaned up. + assert not hasattr(later, "_prefetch") + assert model_prefetch.PREFETCH_QUEUES == [] From 2522c20266def47d7567f196412fc15a9fdb3173 Mon Sep 17 00:00:00 2001 From: Glary-Bot Date: Thu, 29 Oct 2026 21:41:30 +0000 Subject: [PATCH 4/5] Always clear _prefetch markers when prefetched module cleanup fails --- comfy/model_prefetch.py | 33 ++++++++++++++----- .../comfy_test/model_prefetch_cleanup_test.py | 25 ++++++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index b1aef073ff7..57848584836 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -123,20 +123,35 @@ def pin_modules(comfy_modules, device, dtype=None): return offload_stream, fully_faulted def cleanup_prefetched_modules(module, comfy_modules): + # Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight + # skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error. + first_error = None for s in comfy_modules: prefetch = getattr(s, "_prefetch", None) if prefetch is None: continue - for param_key in ("weight", "bias"): - lowvram_fn = getattr(s, param_key + "_lowvram_function", None) - if lowvram_fn is not None: - lowvram_fn.clear_prepared() - if prefetch["signature"] is not None: - comfy_aimdo.model_vbar.vbar_unpin(s._v) - delattr(s, "_prefetch") + try: + for param_key in ("weight", "bias"): + lowvram_fn = getattr(s, param_key + "_lowvram_function", None) + if lowvram_fn is not None: + lowvram_fn.clear_prepared() + if prefetch["signature"] is not None: + comfy_aimdo.model_vbar.vbar_unpin(s._v) + except Exception as e: + if first_error is None: + first_error = e + finally: + delattr(s, "_prefetch") if getattr(module, "_v_block_faulted", False): - comfy_aimdo.model_vbar.vbar_unpin(module._v_block) - del module._v_block_faulted + try: + comfy_aimdo.model_vbar.vbar_unpin(module._v_block) + except Exception as e: + if first_error is None: + first_error = e + finally: + del module._v_block_faulted + if first_error is not None: + raise first_error def _drop_graph(module): graph = getattr(module, "_comfy_graph", None) diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py index 7396db23a62..3dacbc2518a 100644 --- a/tests-unit/comfy_test/model_prefetch_cleanup_test.py +++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py @@ -84,3 +84,28 @@ def cleanup(module, comfy_modules): # The first failure is re-raised, but only after every other queue was cleaned up. assert not hasattr(later, "_prefetch") assert model_prefetch.PREFETCH_QUEUES == [] + + +def test_failed_unpin_still_clears_prefetch_markers(monkeypatch): + # A module whose unpin raised must not keep _prefetch, or a later cast would skip + # the fresh cast and reuse the stale transfer state. The rest of the group is still cleaned. + calls = [] + + def vbar_unpin(v): + calls.append(v) + if v == "first": + raise RuntimeError("unpin failed") + + monkeypatch.setattr(model_prefetch.comfy_aimdo.model_vbar, "vbar_unpin", vbar_unpin) + first = torch.nn.Linear(4, 4) + second = torch.nn.Linear(4, 4) + for module, v in ((first, "first"), (second, "second")): + module._v = v + module._prefetch = {"signature": "sig"} + model_prefetch.PREFETCH_QUEUES = [[(None, (first, [first, second])), None]] + + with pytest.raises(RuntimeError, match="unpin failed"): + model_prefetch.cleanup_prefetch_queues() + assert calls == ["first", "second"] + assert not hasattr(first, "_prefetch") + assert not hasattr(second, "_prefetch") From acc3b5aff14005814cde6788f768443a96c27730 Mon Sep 17 00:00:00 2001 From: bymyself Date: Sat, 31 Oct 2026 05:44:44 +0000 Subject: [PATCH 5/5] Clean prefetch queues when malloc graph cleanup fails --- comfy/model_prefetch.py | 5 ++++- .../comfy_test/model_prefetch_cleanup_test.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 57848584836..75dfaf786b3 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -171,11 +171,14 @@ def cleanup_prefetch_queues(): global MALLOC_GRAPH_ROGUES global MALLOC_GRAPH_USED - cleanup_malloc_graph() queues = PREFETCH_QUEUES # Reset before walking so a failing entry can't leave stale queues behind for every later node. PREFETCH_QUEUES = [] first_error = None + try: + cleanup_malloc_graph() + except Exception as e: + first_error = e for queue in queues: # Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later # entries, and a head that was not reached yet, still hold the raw module or module group. diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py index 3dacbc2518a..4ccaa71474c 100644 --- a/tests-unit/comfy_test/model_prefetch_cleanup_test.py +++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py @@ -109,3 +109,20 @@ def vbar_unpin(v): assert calls == ["first", "second"] assert not hasattr(first, "_prefetch") assert not hasattr(second, "_prefetch") + + +def test_malloc_graph_cleanup_failure_still_cleans_queues(monkeypatch): + # After a GPU failure the graph abort can raise too. Queues and _prefetch markers must not + # survive into the next node, and the graph error still reaches the caller. + def boom(): + raise RuntimeError("graph abort failed") + + monkeypatch.setattr(model_prefetch, "cleanup_malloc_graph", boom) + module = torch.nn.Linear(4, 4) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(module), None]] + + with pytest.raises(RuntimeError, match="graph abort failed"): + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + assert not hasattr(module, "_prefetch")