Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
a649493
Track every VBAR block used by a decode graph
rattus128 Oct 21, 2026
f2031e4
Make writable models independent of the model they serve
rattus128 Oct 24, 2026
2effc04
Move the dynamic VRAM check for Qwen Image 2.1 caches into the model
rattus128 Oct 24, 2026
8a22606
Release writable models after the node
rattus128 Oct 24, 2026
a742657
Add model states
rattus128 Nov 1, 2026
021eb96
Use model states for Qwen autoregressive caches
rattus128 Oct 24, 2026
88ae8bf
Support resuming Qwen generation from saved cache state
rattus128 Oct 24, 2026
f323576
Support anonymous files on other POSIX platforms
rattus128 Oct 24, 2026
cf157b6
Add Open Model State node
rattus128 Oct 26, 2026
4263238
Add Save Model State node
rattus128 Oct 26, 2026
3db50de
Generate text into model states
rattus128 Nov 1, 2026
855d358
Add tests for model states
rattus128 Nov 1, 2026
02020d8
Apply review suggestions to the model state nodes
rattus128 Nov 2, 2026
aa9fcaa
Continue a saved Qwen3.5 state with one batched prefill
synap5e Nov 4, 2026
7885ceb
Add a prompt state cache for Qwen3.5 TextGenerate
synap5e Nov 4, 2026
eeab416
Accept older prefix_cache mode names and add LLMPrefixCacheStats
synap5e Nov 4, 2026
876878f
Let the prompt state cache use the int8 KV cache
synap5e Nov 4, 2026
aace0b7
Fix MTPHead unpack crash in prefetch queue cleanup (CORE-644) (#17180)
christian-byrne Nov 1, 2026
af6b54e
Register w3a8_int8 and w5a8_int8 grouped INT8 formats
synap5e Nov 4, 2026
512bb0f
Pin comfy-kitchen to #303's head (34bc583) for the w3/w5 formats
synap5e Nov 5, 2026
21f6ba1
Merge a continued int8 prefill's rows per key/value head group
synap5e Nov 5, 2026
8e66106
Prefill a long prompt in pieces of at most 4096 rows
synap5e Nov 5, 2026
8d94c64
Prefill the prompt state cache's segments through Qwen35.prefill
synap5e Nov 5, 2026
998d7f1
Don't query device memory from /system_stats while a prompt runs
synap5e Nov 5, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions comfy/anonymous_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
def create_anonymous_file(size=0):
"""Return an unbuffered binary file owned by the caller, initially sized in bytes.

Uses Linux memfd or a Windows temporary-cache/delete-on-close file.
Uses Linux memfd, a Windows temporary-cache/delete-on-close file, or an
unlinked temporary file elsewhere.
Windows may still write to disk; Linux memfd may be swapped.
Initial sizing is sparse with zero-filled holes; Windows requires a filesystem
supporting sparse files. Keep the file open through all I/O and close
Expand All @@ -47,7 +48,7 @@ def create_anonymous_file(size=0):
raise OSError(errno, os.strerror(errno))
file = open(fd, "w+b", buffering=0)
else:
raise NotImplementedError("anonymous files require Linux or Windows")
file = tempfile.TemporaryFile(buffering=0)

try:
if os.name == "nt":
Expand Down
2 changes: 1 addition & 1 deletion comfy/model_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -2800,7 +2800,7 @@ def get_writable_model(self, conds, noise_shape, model_options, device):
transformer_options = model_options.get("transformer_options", {})
options = transformer_options.get("qwen_image21_cache", {})
model = self.diffusion_model
if options.get("device", "auto") != "auto" or not model.can_cache_prefix(transformer_options):
if options.get("device", "auto") != "auto" or not self.current_patcher.is_dynamic() or not model.can_cache_prefix(transformer_options):
return None

heads = model.transformer_blocks[0].attn.heads
Expand Down
13 changes: 13 additions & 0 deletions comfy/model_management.py
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,9 @@ def cuda_device_context(device):
if prev is not None:
torch.cuda.set_device(prev)

# Held while a prompt executes: a dynamic VRAM page-out can hold torch's allocator lock and hand the GIL to another thread, where torch.cuda.memory_stats then deadlocks.
model_execution_lock = threading.Lock()

def get_total_memory(dev=None, torch_total_too=False):
global directml_enabled
if dev is None:
Expand Down Expand Up @@ -634,6 +637,8 @@ def get_torch_device_name(device):

current_loaded_models: list[LoadedModel] = []

EPHEMERAL_MODELS = []

DIRTY_MMAPS = set()

PIN_PRESSURE_HYSTERESIS = 256 * 1024 * 1024
Expand Down Expand Up @@ -1080,6 +1085,14 @@ def archive_model_dtypes(model):
setattr(module, f"{buf_name}_comfy_model_dtype", buf.dtype)


def register_ephemeral_model(state):
"""Keep a model state loaded until the node ends, then release whatever was not flushed."""
EPHEMERAL_MODELS.append(state)

def cleanup_ephemeral_models():
while EPHEMERAL_MODELS:
EPHEMERAL_MODELS.pop().release()

def cleanup_models():
to_delete = []
for i in range(len(current_loaded_models)):
Expand Down
58 changes: 17 additions & 41 deletions comfy/model_patcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from __future__ import annotations

import collections
import contextlib
import inspect
import logging
import math
Expand All @@ -38,6 +37,7 @@
import comfy.model_prefetch
import comfy.ops
import comfy.patcher_extension
import comfy.storage
import comfy.utils
import comfy_aimdo.host_buffer
from comfy.comfy_types import UnetWrapperFunction
Expand Down Expand Up @@ -1404,18 +1404,6 @@ def get_injections(self, key: str):
def set_additional_models(self, key: str, models: list['ModelPatcher']):
self.additional_models[key] = models

@contextlib.contextmanager
def writable_models(self, conds, noise_shape, model_options):
with contextlib.ExitStack() as stack:
for patcher in [self] + self.get_additional_models_with_key("multigpu"):
if not patcher.is_dynamic():
continue
model = patcher.model.get_writable_model(conds, noise_shape, model_options, patcher.load_device)
if model is not None:
with comfy.model_management.cuda_device_context(patcher.load_device):
stack.enter_context(patcher.writable_model(model))
yield

def remove_additional_models(self, key: str):
if key in self.additional_models:
self.additional_models.pop(key)
Expand Down Expand Up @@ -1834,34 +1822,22 @@ def register_load_device(self, device):
def is_dynamic(self):
return True

@contextlib.contextmanager
def writable_model(self, model):
state, file = comfy.utils.create_safetensors(model.state_dict())
patcher = None
try:
model.load_state_dict(state, assign=True)
patcher = ModelPatcherDynamic(model, self.load_device, torch.device("cpu"), fast_disk=self.fast_disk)
self.set_additional_models("writable", [patcher])
# Loading reverses this list; the writable model gets priority over its owner.
comfy.model_management.load_models_gpu([patcher, self])
patcher.set_in_use_by_current_prompt(True)
yield patcher
finally:
with comfy.model_management.cuda_device_context(self.load_device):
# The final forward may still be using the allocation graph's pages.
comfy.model_management.synchronize()
comfy.model_prefetch.cleanup_prefetch_queues()
comfy.model_prefetch.cleanup_malloc_graph()
if patcher is not None:
self.remove_additional_models("writable")
vbar = model.dynamic_vbars.pop(patcher.load_device, None)
if vbar is not None:
vbar.close()
for module in model.modules():
for pin in getattr(module, "_pins", {}).values():
pin["dirty"] = False
comfy.model_management.unload_model_and_clones(patcher, unload_additional_models=False)
file.close()
def discard_writes(self):
"""Forget pinned writes so unpinning does not flush them to the backing file."""
pin_state = self.model.dynamic_pins[self.load_device]
for subset in ("weights", "weights-loaded", "weights-fast"):
for module, _ in pin_state[subset][1]:
module._pins[subset]["dirty"] = False

def flush_writes(self):
"""Write every value of a writable model back to its backing file without evicting it."""
for m in self.model.modules():
if hasattr(m, "_v") and m._v_signature is not None:
m.writeback(m._v_weight)
pin_state = self.model.dynamic_pins[self.load_device]
for subset in ("weights", "weights-loaded", "weights-fast"):
for module, _ in pin_state[subset][1]:
comfy.pinned_memory.flush_pin(module, module._pins[subset])

def set_in_use_by_current_prompt(self, in_use):
self.model.dynamic_pins[self.load_device]["current_prompt"] = in_use
Expand Down
104 changes: 75 additions & 29 deletions comfy/model_prefetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,20 +123,52 @@ def pin_modules(comfy_modules, device, dtype=None):
return offload_stream, fully_faulted

def cleanup_prefetched_modules(module, comfy_modules):
# Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight
# skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error.
first_error = None
for s in comfy_modules:
prefetch = getattr(s, "_prefetch", None)
if prefetch is None:
continue
for param_key in ("weight", "bias"):
lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
if lowvram_fn is not None:
lowvram_fn.clear_prepared()
if prefetch["signature"] is not None:
comfy_aimdo.model_vbar.vbar_unpin(s._v)
delattr(s, "_prefetch")
if getattr(module, "_v_block_faulted", False):
comfy_aimdo.model_vbar.vbar_unpin(module._v_block)
del module._v_block_faulted
try:
for param_key in ("weight", "bias"):
lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
if lowvram_fn is not None:
lowvram_fn.clear_prepared()
if prefetch["signature"] is not None:
comfy_aimdo.model_vbar.vbar_unpin(s._v)
except Exception as e:
if first_error is None:
first_error = e
finally:
delattr(s, "_prefetch")
for block in getattr(module, "_v_blocks_faulted", ()):
try:
comfy_aimdo.model_vbar.vbar_unpin(block)
except Exception as e:
if first_error is None:
first_error = e
if hasattr(module, "_v_blocks_faulted"):
del module._v_blocks_faulted
if first_error is not None:
raise first_error

def fault_graph_blocks(module, blocks):
signatures = []
pinned = []
try:
for block in blocks:
signature = comfy_aimdo.model_vbar.vbar_fault(block)
if signature is None:
return None
pinned.append(block)
signatures.append(signature)
module._v_blocks_faulted = pinned
pinned = []
return signatures
finally:
for block in pinned:
comfy_aimdo.model_vbar.vbar_unpin(block)

def _drop_graph(module):
graph = getattr(module, "_comfy_graph", None)
Expand All @@ -156,19 +188,35 @@ def cleanup_prefetch_queues():
global MALLOC_GRAPH_ROGUES
global MALLOC_GRAPH_USED

cleanup_malloc_graph()
for queue in PREFETCH_QUEUES:
# Only the head has been prefetched; later entries are raw module groups.
if queue[0] is not None:
_, (prefetched_module, comfy_modules) = queue[0]
cleanup_prefetched_modules(prefetched_module, comfy_modules)
queues = PREFETCH_QUEUES
# Reset before walking so a failing entry can't leave stale queues behind for every later node.
PREFETCH_QUEUES = []
first_error = None
try:
cleanup_malloc_graph()
except Exception as e:
first_error = e
for queue in queues:
# Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later
# entries, and a head that was not reached yet, still hold the raw module or module group.
head = queue[0] if queue else None
if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple):
continue
prefetched_module, comfy_modules = head[1]
try:
cleanup_prefetched_modules(prefetched_module, comfy_modules)
except Exception as e:
# Keep unpinning the other queues; report the first failure once they are done.
if first_error is None:
first_error = e
GRAPH_WARMED_MODULES.clear()
if MALLOC_GRAPH_USED:
logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES)
MALLOC_GRAPH_BREAKS = 0
MALLOC_GRAPH_ROGUES = 0
MALLOC_GRAPH_USED = False
if first_error is not None:
raise first_error

def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None):
malloc_graph = MALLOC_GRAPHS.get(threading.get_ident())
Expand Down Expand Up @@ -197,14 +245,16 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap
malloc_graph.resume()
GRAPH_CAPTURE_STREAMS[device] = capture_stream

signature = None
prefetch = queue[1]
prefetch_modules = prefetch if isinstance(prefetch, (list, tuple)) else (prefetch,)
blocks = [root._v_block for root in prefetch_modules if getattr(root, "_v_block", None) is not None] if enable_graph else []
signatures = None
graph_hit = False
graph = getattr(module, "_comfy_graph", None) if enable_graph else None
if graph is not None:
signature = comfy_aimdo.model_vbar.vbar_fault(module._v_block)
if signature is not None:
module._v_block_faulted = True
graph_hit = comfy_aimdo.model_vbar.vbar_signature_compare(signature, graph["signature"])
signatures = fault_graph_blocks(module, blocks)
if signatures is not None:
graph_hit = all(comfy_aimdo.model_vbar.vbar_signature_compare(a, b) for a, b in zip(signatures, graph["signatures"]))

if malloc_graph is not None and malloc_scope is not None:
if malloc_graph.iterate(malloc_scope if module is not None and not graph_hit else None):
Expand All @@ -225,10 +275,8 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap
return

fully_faulted = False
prefetch = queue[0]
if prefetch is not None:
comfy_modules = []
prefetch_modules = prefetch if isinstance(prefetch, (list, tuple)) else (prefetch,)
for root in prefetch_modules:
for s in root.modules():
if hasattr(s, "_v"):
Expand All @@ -241,11 +289,9 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap

if core is not None:
if enable_graph and fully_faulted and module in GRAPH_WARMED_MODULES:
if signature is None:
signature = comfy_aimdo.model_vbar.vbar_fault(module._v_block)
if signature is not None:
module._v_block_faulted = True
if signature is not None:
if signatures is None:
signatures = fault_graph_blocks(module, blocks)
if signatures is not None:
_drop_graph(module)
malloc_graph.pause()
graph = torch.cuda.CUDAGraph()
Expand All @@ -264,7 +310,7 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap
malloc_graph.resume(sync=True)
comfy.model_management.current_stream(device).wait_stream(capture_stream)
graph.replay()
module._comfy_graph = {"graph": graph, "signature": signature}
module._comfy_graph = {"graph": graph, "signatures": signatures}
malloc_graph._comfy_cuda_graph_modules.add(module)
return
if capture_stream is None:
Expand Down
Loading
Loading