diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 6cfe9921f46..75dfaf786b3 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -123,20 +123,35 @@ def pin_modules(comfy_modules, device, dtype=None): return offload_stream, fully_faulted def cleanup_prefetched_modules(module, comfy_modules): + # Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight + # skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error. + first_error = None for s in comfy_modules: prefetch = getattr(s, "_prefetch", None) if prefetch is None: continue - for param_key in ("weight", "bias"): - lowvram_fn = getattr(s, param_key + "_lowvram_function", None) - if lowvram_fn is not None: - lowvram_fn.clear_prepared() - if prefetch["signature"] is not None: - comfy_aimdo.model_vbar.vbar_unpin(s._v) - delattr(s, "_prefetch") + try: + for param_key in ("weight", "bias"): + lowvram_fn = getattr(s, param_key + "_lowvram_function", None) + if lowvram_fn is not None: + lowvram_fn.clear_prepared() + if prefetch["signature"] is not None: + comfy_aimdo.model_vbar.vbar_unpin(s._v) + except Exception as e: + if first_error is None: + first_error = e + finally: + delattr(s, "_prefetch") if getattr(module, "_v_block_faulted", False): - comfy_aimdo.model_vbar.vbar_unpin(module._v_block) - del module._v_block_faulted + try: + comfy_aimdo.model_vbar.vbar_unpin(module._v_block) + except Exception as e: + if first_error is None: + first_error = e + finally: + del module._v_block_faulted + if first_error is not None: + raise first_error def _drop_graph(module): graph = getattr(module, "_comfy_graph", None) @@ -156,19 +171,35 @@ def cleanup_prefetch_queues(): global MALLOC_GRAPH_ROGUES global MALLOC_GRAPH_USED - cleanup_malloc_graph() - for queue in PREFETCH_QUEUES: - # Only the head has been prefetched; later entries are raw module groups. - if queue[0] is not None: - _, (prefetched_module, comfy_modules) = queue[0] - cleanup_prefetched_modules(prefetched_module, comfy_modules) + queues = PREFETCH_QUEUES + # Reset before walking so a failing entry can't leave stale queues behind for every later node. PREFETCH_QUEUES = [] + first_error = None + try: + cleanup_malloc_graph() + except Exception as e: + first_error = e + for queue in queues: + # Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later + # entries, and a head that was not reached yet, still hold the raw module or module group. + head = queue[0] if queue else None + if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple): + continue + prefetched_module, comfy_modules = head[1] + try: + cleanup_prefetched_modules(prefetched_module, comfy_modules) + except Exception as e: + # Keep unpinning the other queues; report the first failure once they are done. + if first_error is None: + first_error = e GRAPH_WARMED_MODULES.clear() if MALLOC_GRAPH_USED: logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES) MALLOC_GRAPH_BREAKS = 0 MALLOC_GRAPH_ROGUES = 0 MALLOC_GRAPH_USED = False + if first_error is not None: + raise first_error def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None): malloc_graph = MALLOC_GRAPHS.get(threading.get_ident()) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 4719c52dd2d..d4f241c5f20 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -998,7 +998,7 @@ def draft(token, hidden, p): draft_freqs.copy_(freqs_at(p)) host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed) queue = comfy.model_prefetch.make_prefetch_queue( - [None if host_rows else embed, (head, self.mtp)], device, + [None if host_rows else embed, [head, self.mtp]], device, {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars}) def embed_core(): diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py new file mode 100644 index 00000000000..4ccaa71474c --- /dev/null +++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py @@ -0,0 +1,128 @@ +import pytest +import torch + +from comfy.cli_args import args as cli_args + +if not torch.cuda.is_available(): + cli_args.cpu = True + +import comfy.model_prefetch as model_prefetch # noqa: E402 + + +class MTPHead(torch.nn.Module): + pass + + +@pytest.fixture(autouse=True) +def _reset_queues(): + model_prefetch.PREFETCH_QUEUES = [] + yield + model_prefetch.PREFETCH_QUEUES = [] + + +def _prefetched(module): + module._prefetch = {"signature": None} + return (None, (module, [module])) + + +@pytest.mark.parametrize("pending_unit", [tuple, list]) +def test_cleanup_skips_unconsumed_grouped_unit(pending_unit): + # Qwen3.5 MTP draft queue interrupted after the embed pop: the embed slot has + # been pinned, the grouped (head, mtp) slot has not been reached yet. + embed = torch.nn.Embedding(4, 4) + head = torch.nn.Linear(4, 4) + queue = [_prefetched(embed), pending_unit((head, MTPHead())), None] + model_prefetch.PREFETCH_QUEUES = [queue] + + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + assert not hasattr(embed, "_prefetch") + + +def test_cleanup_resets_queues_when_entry_cleanup_fails(monkeypatch): + def boom(module, comfy_modules): + raise RuntimeError("unpin failed") + + monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", boom) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(torch.nn.Linear(4, 4)), None]] + + with pytest.raises(RuntimeError): + model_prefetch.cleanup_prefetch_queues() + # A failed cleanup must not poison every later node on the worker. + assert model_prefetch.PREFETCH_QUEUES == [] + model_prefetch.cleanup_prefetch_queues() + + +@pytest.mark.parametrize("pending_unit", [tuple, list]) +def test_cleanup_skips_unreached_grouped_head(pending_unit): + # Interrupted between popping the consumed slot and pinning the next one: the + # head of the queue is still the raw (head, mtp) group, not a pinned slot. + queue = [pending_unit((torch.nn.Linear(4, 4), MTPHead())), None] + model_prefetch.PREFETCH_QUEUES = [queue] + + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + + +def test_cleanup_unpins_later_queues_when_an_earlier_one_fails(monkeypatch): + real_cleanup = model_prefetch.cleanup_prefetched_modules + failing = torch.nn.Linear(4, 4) + later = torch.nn.Linear(4, 4) + + def cleanup(module, comfy_modules): + if module is failing: + raise RuntimeError("unpin failed") + real_cleanup(module, comfy_modules) + + monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", cleanup) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(failing), None], [_prefetched(later), None]] + + with pytest.raises(RuntimeError, match="unpin failed"): + model_prefetch.cleanup_prefetch_queues() + # The first failure is re-raised, but only after every other queue was cleaned up. + assert not hasattr(later, "_prefetch") + assert model_prefetch.PREFETCH_QUEUES == [] + + +def test_failed_unpin_still_clears_prefetch_markers(monkeypatch): + # A module whose unpin raised must not keep _prefetch, or a later cast would skip + # the fresh cast and reuse the stale transfer state. The rest of the group is still cleaned. + calls = [] + + def vbar_unpin(v): + calls.append(v) + if v == "first": + raise RuntimeError("unpin failed") + + monkeypatch.setattr(model_prefetch.comfy_aimdo.model_vbar, "vbar_unpin", vbar_unpin) + first = torch.nn.Linear(4, 4) + second = torch.nn.Linear(4, 4) + for module, v in ((first, "first"), (second, "second")): + module._v = v + module._prefetch = {"signature": "sig"} + model_prefetch.PREFETCH_QUEUES = [[(None, (first, [first, second])), None]] + + with pytest.raises(RuntimeError, match="unpin failed"): + model_prefetch.cleanup_prefetch_queues() + assert calls == ["first", "second"] + assert not hasattr(first, "_prefetch") + assert not hasattr(second, "_prefetch") + + +def test_malloc_graph_cleanup_failure_still_cleans_queues(monkeypatch): + # After a GPU failure the graph abort can raise too. Queues and _prefetch markers must not + # survive into the next node, and the graph error still reaches the caller. + def boom(): + raise RuntimeError("graph abort failed") + + monkeypatch.setattr(model_prefetch, "cleanup_malloc_graph", boom) + module = torch.nn.Linear(4, 4) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(module), None]] + + with pytest.raises(RuntimeError, match="graph abort failed"): + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + assert not hasattr(module, "_prefetch")