Skip to content
61 changes: 46 additions & 15 deletions comfy/model_prefetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,20 +123,35 @@ def pin_modules(comfy_modules, device, dtype=None):
return offload_stream, fully_faulted

def cleanup_prefetched_modules(module, comfy_modules):
# Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight
# skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error.
first_error = None
for s in comfy_modules:
prefetch = getattr(s, "_prefetch", None)
if prefetch is None:
continue
for param_key in ("weight", "bias"):
lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
if lowvram_fn is not None:
lowvram_fn.clear_prepared()
if prefetch["signature"] is not None:
comfy_aimdo.model_vbar.vbar_unpin(s._v)
delattr(s, "_prefetch")
try:
for param_key in ("weight", "bias"):
lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
if lowvram_fn is not None:
lowvram_fn.clear_prepared()
if prefetch["signature"] is not None:
comfy_aimdo.model_vbar.vbar_unpin(s._v)
except Exception as e:
if first_error is None:
first_error = e
finally:
delattr(s, "_prefetch")
if getattr(module, "_v_block_faulted", False):
comfy_aimdo.model_vbar.vbar_unpin(module._v_block)
del module._v_block_faulted
try:
comfy_aimdo.model_vbar.vbar_unpin(module._v_block)
except Exception as e:
if first_error is None:
first_error = e
finally:
del module._v_block_faulted
if first_error is not None:
raise first_error

def _drop_graph(module):
graph = getattr(module, "_comfy_graph", None)
Expand All @@ -156,19 +171,35 @@ def cleanup_prefetch_queues():
global MALLOC_GRAPH_ROGUES
global MALLOC_GRAPH_USED

cleanup_malloc_graph()
for queue in PREFETCH_QUEUES:
# Only the head has been prefetched; later entries are raw module groups.
if queue[0] is not None:
_, (prefetched_module, comfy_modules) = queue[0]
cleanup_prefetched_modules(prefetched_module, comfy_modules)
queues = PREFETCH_QUEUES
# Reset before walking so a failing entry can't leave stale queues behind for every later node.
PREFETCH_QUEUES = []
first_error = None
try:
cleanup_malloc_graph()
except Exception as e:
first_error = e
for queue in queues:
# Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later
# entries, and a head that was not reached yet, still hold the raw module or module group.
head = queue[0] if queue else None
if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple):
continue
prefetched_module, comfy_modules = head[1]
try:
cleanup_prefetched_modules(prefetched_module, comfy_modules)
except Exception as e:
# Keep unpinning the other queues; report the first failure once they are done.
if first_error is None:
first_error = e
Comment thread
christian-byrne marked this conversation as resolved.
Resolved
GRAPH_WARMED_MODULES.clear()
if MALLOC_GRAPH_USED:
logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES)
MALLOC_GRAPH_BREAKS = 0
MALLOC_GRAPH_ROGUES = 0
MALLOC_GRAPH_USED = False
if first_error is not None:
raise first_error

def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None):
malloc_graph = MALLOC_GRAPHS.get(threading.get_ident())
Expand Down
2 changes: 1 addition & 1 deletion comfy/text_encoders/qwen35.py
Original file line number Diff line number Diff line change
Expand Up @@ -998,7 +998,7 @@ def draft(token, hidden, p):
draft_freqs.copy_(freqs_at(p))
host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed)
queue = comfy.model_prefetch.make_prefetch_queue(
[None if host_rows else embed, (head, self.mtp)], device,
[None if host_rows else embed, [head, self.mtp]], device,
{"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars})

def embed_core():
Expand Down
128 changes: 128 additions & 0 deletions tests-unit/comfy_test/model_prefetch_cleanup_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
import pytest
import torch

from comfy.cli_args import args as cli_args

if not torch.cuda.is_available():
cli_args.cpu = True

import comfy.model_prefetch as model_prefetch # noqa: E402


class MTPHead(torch.nn.Module):
pass


@pytest.fixture(autouse=True)
def _reset_queues():
model_prefetch.PREFETCH_QUEUES = []
yield
model_prefetch.PREFETCH_QUEUES = []


def _prefetched(module):
module._prefetch = {"signature": None}
return (None, (module, [module]))


@pytest.mark.parametrize("pending_unit", [tuple, list])
def test_cleanup_skips_unconsumed_grouped_unit(pending_unit):
# Qwen3.5 MTP draft queue interrupted after the embed pop: the embed slot has
# been pinned, the grouped (head, mtp) slot has not been reached yet.
embed = torch.nn.Embedding(4, 4)
head = torch.nn.Linear(4, 4)
queue = [_prefetched(embed), pending_unit((head, MTPHead())), None]
model_prefetch.PREFETCH_QUEUES = [queue]

model_prefetch.cleanup_prefetch_queues()

assert model_prefetch.PREFETCH_QUEUES == []
assert not hasattr(embed, "_prefetch")


def test_cleanup_resets_queues_when_entry_cleanup_fails(monkeypatch):
def boom(module, comfy_modules):
raise RuntimeError("unpin failed")

monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", boom)
model_prefetch.PREFETCH_QUEUES = [[_prefetched(torch.nn.Linear(4, 4)), None]]

with pytest.raises(RuntimeError):
model_prefetch.cleanup_prefetch_queues()
# A failed cleanup must not poison every later node on the worker.
assert model_prefetch.PREFETCH_QUEUES == []
model_prefetch.cleanup_prefetch_queues()


@pytest.mark.parametrize("pending_unit", [tuple, list])
def test_cleanup_skips_unreached_grouped_head(pending_unit):
# Interrupted between popping the consumed slot and pinning the next one: the
# head of the queue is still the raw (head, mtp) group, not a pinned slot.
queue = [pending_unit((torch.nn.Linear(4, 4), MTPHead())), None]
model_prefetch.PREFETCH_QUEUES = [queue]

model_prefetch.cleanup_prefetch_queues()

assert model_prefetch.PREFETCH_QUEUES == []


def test_cleanup_unpins_later_queues_when_an_earlier_one_fails(monkeypatch):
real_cleanup = model_prefetch.cleanup_prefetched_modules
failing = torch.nn.Linear(4, 4)
later = torch.nn.Linear(4, 4)

def cleanup(module, comfy_modules):
if module is failing:
raise RuntimeError("unpin failed")
real_cleanup(module, comfy_modules)

monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", cleanup)
model_prefetch.PREFETCH_QUEUES = [[_prefetched(failing), None], [_prefetched(later), None]]

with pytest.raises(RuntimeError, match="unpin failed"):
model_prefetch.cleanup_prefetch_queues()
# The first failure is re-raised, but only after every other queue was cleaned up.
assert not hasattr(later, "_prefetch")
assert model_prefetch.PREFETCH_QUEUES == []


def test_failed_unpin_still_clears_prefetch_markers(monkeypatch):
# A module whose unpin raised must not keep _prefetch, or a later cast would skip
# the fresh cast and reuse the stale transfer state. The rest of the group is still cleaned.
calls = []

def vbar_unpin(v):
calls.append(v)
if v == "first":
raise RuntimeError("unpin failed")

monkeypatch.setattr(model_prefetch.comfy_aimdo.model_vbar, "vbar_unpin", vbar_unpin)
first = torch.nn.Linear(4, 4)
second = torch.nn.Linear(4, 4)
for module, v in ((first, "first"), (second, "second")):
module._v = v
module._prefetch = {"signature": "sig"}
model_prefetch.PREFETCH_QUEUES = [[(None, (first, [first, second])), None]]

with pytest.raises(RuntimeError, match="unpin failed"):
model_prefetch.cleanup_prefetch_queues()
assert calls == ["first", "second"]
assert not hasattr(first, "_prefetch")
assert not hasattr(second, "_prefetch")


def test_malloc_graph_cleanup_failure_still_cleans_queues(monkeypatch):
# After a GPU failure the graph abort can raise too. Queues and _prefetch markers must not
# survive into the next node, and the graph error still reaches the caller.
def boom():
raise RuntimeError("graph abort failed")

monkeypatch.setattr(model_prefetch, "cleanup_malloc_graph", boom)
module = torch.nn.Linear(4, 4)
model_prefetch.PREFETCH_QUEUES = [[_prefetched(module), None]]

with pytest.raises(RuntimeError, match="graph abort failed"):
model_prefetch.cleanup_prefetch_queues()

assert model_prefetch.PREFETCH_QUEUES == []
assert not hasattr(module, "_prefetch")
Loading