From a6494934feb1914a65ff066a7dd3a707437188e3 Mon Sep 17 00:00:00 2001 From: Rattus Date: Wed, 21 Oct 2026 23:05:18 +1000 Subject: [PATCH 01/24] Track every VBAR block used by a decode graph --- comfy/model_prefetch.py | 48 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 6cfe9921f46..022685125d2 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -134,9 +134,27 @@ def cleanup_prefetched_modules(module, comfy_modules): if prefetch["signature"] is not None: comfy_aimdo.model_vbar.vbar_unpin(s._v) delattr(s, "_prefetch") - if getattr(module, "_v_block_faulted", False): - comfy_aimdo.model_vbar.vbar_unpin(module._v_block) - del module._v_block_faulted + for block in getattr(module, "_v_blocks_faulted", ()): + comfy_aimdo.model_vbar.vbar_unpin(block) + if hasattr(module, "_v_blocks_faulted"): + del module._v_blocks_faulted + +def fault_graph_blocks(module, blocks): + signatures = [] + pinned = [] + try: + for block in blocks: + signature = comfy_aimdo.model_vbar.vbar_fault(block) + if signature is None: + return None + pinned.append(block) + signatures.append(signature) + module._v_blocks_faulted = pinned + pinned = [] + return signatures + finally: + for block in pinned: + comfy_aimdo.model_vbar.vbar_unpin(block) def _drop_graph(module): graph = getattr(module, "_comfy_graph", None) @@ -197,14 +215,16 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap malloc_graph.resume() GRAPH_CAPTURE_STREAMS[device] = capture_stream - signature = None + prefetch = queue[1] + prefetch_modules = prefetch if isinstance(prefetch, (list, tuple)) else (prefetch,) + blocks = [root._v_block for root in prefetch_modules if getattr(root, "_v_block", None) is not None] if enable_graph else [] + signatures = None graph_hit = False graph = getattr(module, "_comfy_graph", None) if enable_graph else None if graph is not None: - signature = comfy_aimdo.model_vbar.vbar_fault(module._v_block) - if signature is not None: - module._v_block_faulted = True - graph_hit = comfy_aimdo.model_vbar.vbar_signature_compare(signature, graph["signature"]) + signatures = fault_graph_blocks(module, blocks) + if signatures is not None: + graph_hit = all(comfy_aimdo.model_vbar.vbar_signature_compare(a, b) for a, b in zip(signatures, graph["signatures"])) if malloc_graph is not None and malloc_scope is not None: if malloc_graph.iterate(malloc_scope if module is not None and not graph_hit else None): @@ -225,10 +245,8 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap return fully_faulted = False - prefetch = queue[0] if prefetch is not None: comfy_modules = [] - prefetch_modules = prefetch if isinstance(prefetch, (list, tuple)) else (prefetch,) for root in prefetch_modules: for s in root.modules(): if hasattr(s, "_v"): @@ -241,11 +259,9 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap if core is not None: if enable_graph and fully_faulted and module in GRAPH_WARMED_MODULES: - if signature is None: - signature = comfy_aimdo.model_vbar.vbar_fault(module._v_block) - if signature is not None: - module._v_block_faulted = True - if signature is not None: + if signatures is None: + signatures = fault_graph_blocks(module, blocks) + if signatures is not None: _drop_graph(module) malloc_graph.pause() graph = torch.cuda.CUDAGraph() @@ -264,7 +280,7 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap malloc_graph.resume(sync=True) comfy.model_management.current_stream(device).wait_stream(capture_stream) graph.replay() - module._comfy_graph = {"graph": graph, "signature": signature} + module._comfy_graph = {"graph": graph, "signatures": signatures} malloc_graph._comfy_cuda_graph_modules.add(module) return if capture_stream is None: From f2031e45b958b2e9432c4ff4d7fbb6b7583ad5e2 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sat, 24 Oct 2026 20:36:10 +1000 Subject: [PATCH 02/24] Make writable models independent of the model they serve A writable model is its own model patcher. It loads after the model it serves, so load_models_gpu([patcher]) alone gives it VBAR priority without nesting it as an additional model, and its storage policy comes from where anonymous files live rather than from that model's checkpoint. --- comfy/model_patcher.py | 60 +++++++++++++++++++++--------------------- comfy/storage.py | 9 +++++++ 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index 83b92edb563..2ba12272ecb 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -38,6 +38,7 @@ import comfy.model_prefetch import comfy.ops import comfy.patcher_extension +import comfy.storage import comfy.utils import comfy_aimdo.host_buffer from comfy.comfy_types import UnetWrapperFunction @@ -1413,7 +1414,7 @@ def writable_models(self, conds, noise_shape, model_options): model = patcher.model.get_writable_model(conds, noise_shape, model_options, patcher.load_device) if model is not None: with comfy.model_management.cuda_device_context(patcher.load_device): - stack.enter_context(patcher.writable_model(model)) + stack.enter_context(writable_model(model, patcher.load_device)) yield def remove_additional_models(self, key: str): @@ -1834,35 +1835,6 @@ def register_load_device(self, device): def is_dynamic(self): return True - @contextlib.contextmanager - def writable_model(self, model): - state, file = comfy.utils.create_safetensors(model.state_dict()) - patcher = None - try: - model.load_state_dict(state, assign=True) - patcher = ModelPatcherDynamic(model, self.load_device, torch.device("cpu"), fast_disk=self.fast_disk) - self.set_additional_models("writable", [patcher]) - # Loading reverses this list; the writable model gets priority over its owner. - comfy.model_management.load_models_gpu([patcher, self]) - patcher.set_in_use_by_current_prompt(True) - yield patcher - finally: - with comfy.model_management.cuda_device_context(self.load_device): - # The final forward may still be using the allocation graph's pages. - comfy.model_management.synchronize() - comfy.model_prefetch.cleanup_prefetch_queues() - comfy.model_prefetch.cleanup_malloc_graph() - if patcher is not None: - self.remove_additional_models("writable") - vbar = model.dynamic_vbars.pop(patcher.load_device, None) - if vbar is not None: - vbar.close() - for module in model.modules(): - for pin in getattr(module, "_pins", {}).values(): - pin["dirty"] = False - comfy.model_management.unload_model_and_clones(patcher, unload_additional_models=False) - file.close() - def set_in_use_by_current_prompt(self, in_use): self.model.dynamic_pins[self.load_device]["current_prompt"] = in_use @@ -2272,3 +2244,31 @@ def get_non_dynamic_delegate(self): CoreModelPatcher = ModelPatcher + + +@contextlib.contextmanager +def writable_model(model, device): + """Load a writable model over its own anonymous backing file for the duration of the block.""" + state, file = comfy.utils.create_safetensors(model.state_dict()) + patcher = None + try: + model.load_state_dict(state, assign=True) + patcher = ModelPatcherDynamic(model, device, torch.device("cpu"), fast_disk=comfy.storage.anonymous_fast_disk()) + comfy.model_management.load_models_gpu([patcher]) + patcher.set_in_use_by_current_prompt(True) + yield patcher + finally: + with comfy.model_management.cuda_device_context(device): + # The final forward may still be using the allocation graph's pages. + comfy.model_management.synchronize() + comfy.model_prefetch.cleanup_prefetch_queues() + comfy.model_prefetch.cleanup_malloc_graph() + if patcher is not None: + vbar = model.dynamic_vbars.pop(device, None) + if vbar is not None: + vbar.close() + for module in model.modules(): + for pin in getattr(module, "_pins", {}).values(): + pin["dirty"] = False + comfy.model_management.unload_model_and_clones(patcher, unload_additional_models=False) + file.close() diff --git a/comfy/storage.py b/comfy/storage.py index 2f40250e5c4..cba18697e1a 100644 --- a/comfy/storage.py +++ b/comfy/storage.py @@ -3,6 +3,7 @@ import os import platform import re +import tempfile import comfy_aimdo.storage from comfy.cli_args import args @@ -80,6 +81,14 @@ def fast_storage(path): return None +@functools.lru_cache(maxsize=None) +def anonymous_fast_disk(): + # Linux anonymous files are memfds and already live in RAM. + if os.name != "nt": + return not args.disable_fast_disk + return model_fast_disk([tempfile.gettempdir()]) + + def annotate_state_dict(state_dict, path): path = os.path.realpath(path) for value in state_dict.values(): From 2effc043a26c50a5d755fc6574c69d46e53366ba Mon Sep 17 00:00:00 2001 From: Rattus Date: Sat, 24 Oct 2026 20:36:22 +1000 Subject: [PATCH 03/24] Move the dynamic VRAM check for Qwen Image 2.1 caches into the model Whether a model wants a writable cache is its own decision. Qwen Image 2.1 keeps its host prefix cache when its patcher is not dynamic; the generic sampler loop no longer skips non-dynamic patchers. --- comfy/model_base.py | 2 +- comfy/model_patcher.py | 2 -- tests-unit/comfy_test/qwen21_cache_admission_test.py | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/comfy/model_base.py b/comfy/model_base.py index 42ccd5dbfbc..856200a41c8 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -2800,7 +2800,7 @@ def get_writable_model(self, conds, noise_shape, model_options, device): transformer_options = model_options.get("transformer_options", {}) options = transformer_options.get("qwen_image21_cache", {}) model = self.diffusion_model - if options.get("device", "auto") != "auto" or not model.can_cache_prefix(transformer_options): + if options.get("device", "auto") != "auto" or not self.current_patcher.is_dynamic() or not model.can_cache_prefix(transformer_options): return None heads = model.transformer_blocks[0].attn.heads diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index 2ba12272ecb..b6c95f5165c 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -1409,8 +1409,6 @@ def set_additional_models(self, key: str, models: list['ModelPatcher']): def writable_models(self, conds, noise_shape, model_options): with contextlib.ExitStack() as stack: for patcher in [self] + self.get_additional_models_with_key("multigpu"): - if not patcher.is_dynamic(): - continue model = patcher.model.get_writable_model(conds, noise_shape, model_options, patcher.load_device) if model is not None: with comfy.model_management.cuda_device_context(patcher.load_device): diff --git a/tests-unit/comfy_test/qwen21_cache_admission_test.py b/tests-unit/comfy_test/qwen21_cache_admission_test.py index d8f3f020504..0b27ce6c41b 100644 --- a/tests-unit/comfy_test/qwen21_cache_admission_test.py +++ b/tests-unit/comfy_test/qwen21_cache_admission_test.py @@ -43,6 +43,7 @@ def test_cache_budget_admits_all_conditions_together(monkeypatch, vram_margin, r monkeypatch.setattr(system_memory, "virtual_memory_available", lambda: 3 * 1024 ** 3 + cache_bytes + ram_margin) monkeypatch.setattr(model_management, "free_pins", lambda size, **kwargs: min(size, reclaimable)) monkeypatch.setattr(patcher, "get_free_memory", lambda device: 4 * cache_bytes + vram_margin) + monkeypatch.setattr(patcher, "is_dynamic", lambda: True) cache = model.get_writable_model(conditions, (1, 4, 3, 5), {"transformer_options": transformer_options}, torch.device("cpu")) assert (cache is not None) == admitted From 8a226063d2cb68f0b914932c735d279dd3885739 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sat, 24 Oct 2026 20:36:33 +1000 Subject: [PATCH 04/24] Release writable models after the node Writable models now stay loaded until node cleanup instead of being torn down inside the sampler. The caller registers the (patcher, model) pair as ephemeral; cleanup drops the patcher first so unpinning happens before the model frees its VBAR and anonymous backing file, and discards dirty pins so nothing is flushed into a file about to die. The sampler loop over patchers no longer needs a context stack. --- comfy/model_management.py | 13 ++++++++++ comfy/model_patcher.py | 51 ++++++++++++--------------------------- comfy/samplers.py | 8 ++++-- execution.py | 1 + 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/comfy/model_management.py b/comfy/model_management.py index e33147e74b9..4434d082184 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -634,6 +634,8 @@ def get_torch_device_name(device): current_loaded_models: list[LoadedModel] = [] +EPHEMERAL_MODELS = [] + DIRTY_MMAPS = set() PIN_PRESSURE_HYSTERESIS = 256 * 1024 * 1024 @@ -1080,6 +1082,17 @@ def archive_model_dtypes(model): setattr(module, f"{buf_name}_comfy_model_dtype", buf.dtype) +def register_ephemeral_model(patcher, model): + """Keep a writable model loaded until the node ends, then drop its patcher before its weights.""" + EPHEMERAL_MODELS.append((patcher, model)) + +def cleanup_ephemeral_models(): + while EPHEMERAL_MODELS: + patcher, model = EPHEMERAL_MODELS.pop() + patcher.discard_writes() + del patcher + del model + def cleanup_models(): to_delete = [] for i in range(len(current_loaded_models)): diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index b6c95f5165c..ce9bcd4e279 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -19,7 +19,6 @@ from __future__ import annotations import collections -import contextlib import inspect import logging import math @@ -1405,16 +1404,6 @@ def get_injections(self, key: str): def set_additional_models(self, key: str, models: list['ModelPatcher']): self.additional_models[key] = models - @contextlib.contextmanager - def writable_models(self, conds, noise_shape, model_options): - with contextlib.ExitStack() as stack: - for patcher in [self] + self.get_additional_models_with_key("multigpu"): - model = patcher.model.get_writable_model(conds, noise_shape, model_options, patcher.load_device) - if model is not None: - with comfy.model_management.cuda_device_context(patcher.load_device): - stack.enter_context(writable_model(model, patcher.load_device)) - yield - def remove_additional_models(self, key: str): if key in self.additional_models: self.additional_models.pop(key) @@ -1833,6 +1822,13 @@ def register_load_device(self, device): def is_dynamic(self): return True + def discard_writes(self): + """Forget pinned writes so unpinning does not flush them to the backing file.""" + pin_state = self.model.dynamic_pins[self.load_device] + for subset in ("weights", "weights-loaded", "weights-fast"): + for module, _ in pin_state[subset][1]: + module._pins[subset]["dirty"] = False + def set_in_use_by_current_prompt(self, in_use): self.model.dynamic_pins[self.load_device]["current_prompt"] = in_use @@ -2244,29 +2240,12 @@ def get_non_dynamic_delegate(self): CoreModelPatcher = ModelPatcher -@contextlib.contextmanager def writable_model(model, device): - """Load a writable model over its own anonymous backing file for the duration of the block.""" - state, file = comfy.utils.create_safetensors(model.state_dict()) - patcher = None - try: - model.load_state_dict(state, assign=True) - patcher = ModelPatcherDynamic(model, device, torch.device("cpu"), fast_disk=comfy.storage.anonymous_fast_disk()) - comfy.model_management.load_models_gpu([patcher]) - patcher.set_in_use_by_current_prompt(True) - yield patcher - finally: - with comfy.model_management.cuda_device_context(device): - # The final forward may still be using the allocation graph's pages. - comfy.model_management.synchronize() - comfy.model_prefetch.cleanup_prefetch_queues() - comfy.model_prefetch.cleanup_malloc_graph() - if patcher is not None: - vbar = model.dynamic_vbars.pop(device, None) - if vbar is not None: - vbar.close() - for module in model.modules(): - for pin in getattr(module, "_pins", {}).values(): - pin["dirty"] = False - comfy.model_management.unload_model_and_clones(patcher, unload_additional_models=False) - file.close() + """Load a writable model over an anonymous backing file for the rest of the node.""" + state, _ = comfy.utils.create_safetensors(model.state_dict()) + model.load_state_dict(state, assign=True) + patcher = ModelPatcherDynamic(model, device, torch.device("cpu"), fast_disk=comfy.storage.anonymous_fast_disk()) + comfy.model_management.load_models_gpu([patcher]) + patcher.set_in_use_by_current_prompt(True) + comfy.model_management.register_ephemeral_model(patcher, model) + return patcher diff --git a/comfy/samplers.py b/comfy/samplers.py index a7cc3419e32..f66cbc264a9 100755 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -1234,8 +1234,12 @@ def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mas sampler, comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.SAMPLER_SAMPLE, extra_args["model_options"], is_model_options=True) ) - with self.model_patcher.writable_models(self.conds, noise.shape, self.model_options): - samples = executor.execute(self, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar) + for patcher in [self.model_patcher] + self.model_patcher.get_additional_models_with_key("multigpu"): + model = patcher.model.get_writable_model(self.conds, noise.shape, self.model_options, patcher.load_device) + if model is not None: + with comfy.model_management.cuda_device_context(patcher.load_device): + comfy.model_patcher.writable_model(model, patcher.load_device) + samples = executor.execute(self, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar) return self.inner_model.process_latent_out(samples.to(torch.float32)) def outer_sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None, latent_shapes=None): diff --git a/execution.py b/execution.py index a8918ada17a..f56bd0925cb 100644 --- a/execution.py +++ b/execution.py @@ -561,6 +561,7 @@ def pre_execute_cb(call_index): comfy.model_prefetch.cleanup_prefetch_queues() comfy.model_management.reset_cast_buffers() comfy_aimdo.model_vbar.vbars_reset_watermark_limits() + comfy.model_management.cleanup_ephemeral_models() if has_pending_tasks: pending_async_nodes[unique_id] = output_data From a7426576becabc16835afaa62ea67db8d37e01f8 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Nov 2026 21:08:42 +1000 Subject: [PATCH 05/24] Add model states A model state is a writable model's values as CPU tensors with its generation metadata, passed between nodes by value. Loading copies the values into the model's own backing: an anonymous file that dynamic VRAM pages from and writes back to, or plain CPU tensors. Flushing reads the loaded values back and drops the model; releasing drops it without them. Nothing writes a state's tensors in place, so a copy shares them. The sampler loads its writable model into a state and registers the state for release after the node, replacing writable_model. --- comfy/model_management.py | 11 ++-- comfy/model_patcher.py | 21 ++++---- comfy/model_state.py | 103 ++++++++++++++++++++++++++++++++++++++ comfy/samplers.py | 5 +- 4 files changed, 121 insertions(+), 19 deletions(-) create mode 100644 comfy/model_state.py diff --git a/comfy/model_management.py b/comfy/model_management.py index 4434d082184..38588d5df0e 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -1082,16 +1082,13 @@ def archive_model_dtypes(model): setattr(module, f"{buf_name}_comfy_model_dtype", buf.dtype) -def register_ephemeral_model(patcher, model): - """Keep a writable model loaded until the node ends, then drop its patcher before its weights.""" - EPHEMERAL_MODELS.append((patcher, model)) +def register_ephemeral_model(state): + """Keep a model state loaded until the node ends, then release whatever was not flushed.""" + EPHEMERAL_MODELS.append(state) def cleanup_ephemeral_models(): while EPHEMERAL_MODELS: - patcher, model = EPHEMERAL_MODELS.pop() - patcher.discard_writes() - del patcher - del model + EPHEMERAL_MODELS.pop().release() def cleanup_models(): to_delete = [] diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index ce9bcd4e279..ecc5c52f85d 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -1829,6 +1829,16 @@ def discard_writes(self): for module, _ in pin_state[subset][1]: module._pins[subset]["dirty"] = False + def flush_writes(self): + """Write every value of a writable model back to its backing file without evicting it.""" + for m in self.model.modules(): + if hasattr(m, "_v") and m._v_signature is not None: + m.writeback(m._v_weight) + pin_state = self.model.dynamic_pins[self.load_device] + for subset in ("weights", "weights-loaded", "weights-fast"): + for module, _ in pin_state[subset][1]: + comfy.pinned_memory.flush_pin(module, module._pins[subset]) + def set_in_use_by_current_prompt(self, in_use): self.model.dynamic_pins[self.load_device]["current_prompt"] = in_use @@ -2238,14 +2248,3 @@ def get_non_dynamic_delegate(self): CoreModelPatcher = ModelPatcher - - -def writable_model(model, device): - """Load a writable model over an anonymous backing file for the rest of the node.""" - state, _ = comfy.utils.create_safetensors(model.state_dict()) - model.load_state_dict(state, assign=True) - patcher = ModelPatcherDynamic(model, device, torch.device("cpu"), fast_disk=comfy.storage.anonymous_fast_disk()) - comfy.model_management.load_models_gpu([patcher]) - patcher.set_in_use_by_current_prompt(True) - comfy.model_management.register_ephemeral_model(patcher, model) - return patcher diff --git a/comfy/model_state.py b/comfy/model_state.py new file mode 100644 index 00000000000..8b9416257c3 --- /dev/null +++ b/comfy/model_state.py @@ -0,0 +1,103 @@ +import copy +import json +import os + +import safetensors +import safetensors.torch +import torch + +import comfy.memory_management +import comfy.model_management +import comfy.model_patcher +import comfy.model_prefetch +import comfy.storage +import comfy.utils + +METADATA_KEY = "comfy_model_state" + + +class ModelState: + """A writable model's values as CPU tensors with its generation metadata, passed between nodes by value. + + Loading copies the values into the model's own backing, so a state's tensors + are never written in place and copies share them. Flushing reads the loaded + model's values back and drops it; a node that loads a state flushes or + releases it before the state reaches another node. + """ + + def __init__(self, tensors=None, metadata=None): + self.tensors = {} if tensors is None else tensors + self.metadata = {} if metadata is None else metadata + self.patcher = None + + @classmethod + def open(cls, path, create="if_missing"): + """Read the state file at path; a missing file is an empty state unless create is never, and always starts empty.""" + if create != "always" and os.path.exists(path): + with safetensors.safe_open(path, "pt") as file: + metadata = file.metadata() or {} + if METADATA_KEY not in metadata: + raise ValueError(f"Not a model state file: {path}") + return cls({name: file.get_tensor(name).clone() for name in file.keys()}, json.loads(metadata[METADATA_KEY])) + if create == "never": + raise FileNotFoundError(f"Model state file does not exist: {path}") + return cls() + + def save(self, path): + if self.tensors is None: + raise RuntimeError("The model state was released without being written back; rerun the node that produced it") + safetensors.torch.save_file(self.tensors, path, {METADATA_KEY: json.dumps(self.metadata)}) + + def copy(self): + if self.tensors is None: + raise RuntimeError("The model state was released without being written back; rerun the node that produced it") + return ModelState(dict(self.tensors), copy.deepcopy(self.metadata)) + + def load(self, model, device): + """Load model with this state's values, or zeros, and return it ready to run.""" + if comfy.memory_management.aimdo_enabled and not comfy.model_management.is_device_cpu(device): + # Dynamic VRAM pages the model from an anonymous file and writes it back there. + tensors, _ = comfy.utils.create_safetensors(model.state_dict()) + for name, value in self.tensors.items(): + comfy.memory_management.write_tensor_file_slice(tensors[name], value) + model.load_state_dict(tensors, assign=True) + self.patcher = comfy.model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu"), fast_disk=comfy.storage.anonymous_fast_disk()) + else: + model.to_empty(device="cpu") + for name, value in model.state_dict().items(): + if self.tensors: + value.copy_(self.tensors[name]) + else: + value.zero_() + self.patcher = comfy.model_patcher.ModelPatcher(model, device, torch.device("cpu")) + comfy.model_management.load_models_gpu([self.patcher], force_full_load=not self.patcher.is_dynamic()) + if self.patcher.is_dynamic(): + self.patcher.set_in_use_by_current_prompt(True) + return model + + def flush(self): + """Take the loaded model's values and drop it.""" + if self.patcher is None: + return + # The last forward may still be using the allocation graph's pages. + comfy.model_management.synchronize() + comfy.model_prefetch.cleanup_prefetch_queues() + state_dict = self.patcher.model.state_dict() + if self.patcher.is_dynamic(): + self.patcher.flush_writes() + self.tensors = {name: torch.empty(value.shape, dtype=value.dtype) for name, value in state_dict.items()} + for name, value in state_dict.items(): + if not comfy.memory_management.read_tensor_file_slice_into(value, self.tensors[name]): + raise RuntimeError(f"Could not read model state tensor {name}") + else: + self.tensors = {name: value.cpu() for name, value in state_dict.items()} + self.patcher = None + + def release(self): + """Drop a still loaded model, discarding its values; the metadata may already describe them, so the state is unusable.""" + if self.patcher is None: + return + if self.patcher.is_dynamic(): + self.patcher.discard_writes() + self.patcher = None + self.tensors = None diff --git a/comfy/samplers.py b/comfy/samplers.py index f66cbc264a9..b1d265bc8cd 100755 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -15,6 +15,7 @@ import logging import comfy.sampler_helpers import comfy.model_patcher +import comfy.model_state import comfy.patcher_extension import comfy.hooks import comfy.context_windows @@ -1238,7 +1239,9 @@ def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mas model = patcher.model.get_writable_model(self.conds, noise.shape, self.model_options, patcher.load_device) if model is not None: with comfy.model_management.cuda_device_context(patcher.load_device): - comfy.model_patcher.writable_model(model, patcher.load_device) + state = comfy.model_state.ModelState() + state.load(model, patcher.load_device) + comfy.model_management.register_ephemeral_model(state) samples = executor.execute(self, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar) return self.inner_model.process_latent_out(samples.to(torch.float32)) From 021eb964f1b4f6cadf38724f5c342699f4d40db8 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sat, 24 Oct 2026 13:21:28 +1000 Subject: [PATCH 06/24] Use model states for Qwen autoregressive caches The KV cache is a module of writable weights that CLIP.generate loads into an anonymous model state for the node, so it is paged like any other model instead of being allocated in VRAM up front. An off-chain MTP accept moves rows within the attention caches, so the step pins those storages in one pop while the GPU is still verifying; only the moves themselves run after the accept sync. Linear caches only record the path for the next forward and are not pinned. --- comfy/sd.py | 4 + comfy/sd1_clip.py | 3 + comfy/text_encoders/llama.py | 68 ++++++++++-- comfy/text_encoders/qwen35.py | 109 ++++++++++++-------- tests-unit/comfy_test/qwen35_decode_test.py | 19 +++- 5 files changed, 143 insertions(+), 60 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index 5069b463dee..5c11904c1ef 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -97,6 +97,7 @@ import comfy.text_encoders.fibo2 import comfy.model_patcher +import comfy.model_state import comfy.lora import comfy.lora_convert import comfy.hooks @@ -484,7 +485,10 @@ def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_ self.cond_stage_model.set_clip_options({"layer": None}) self.cond_stage_model.set_clip_options({"execution_device": device}) + state = comfy.model_state.ModelState() + model_management.register_ephemeral_model(state) with model_management.cuda_device_context(device), comfy.ops.use_quantized_matmul(self.cond_stage_model, device): + self.cond_stage_model.set_clip_options({"model_state": state}) return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty, mtp=mtp) def decode(self, token_ids, skip_special_tokens=True): diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index 39280bb5e5a..f842de0b947 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -136,6 +136,7 @@ def __init__(self, device="cpu", max_length=77, self.return_projected_pooled = return_projected_pooled self.return_attention_masks = return_attention_masks self.execution_device = None + self.model_state = None if layer == "hidden": assert layer_idx is not None @@ -153,6 +154,7 @@ def set_clip_options(self, options): layer_idx = options.get("layer", self.layer_idx) self.return_projected_pooled = options.get("projected_pooled", self.return_projected_pooled) self.execution_device = options.get("execution_device", self.execution_device) + self.model_state = options.get("model_state", self.model_state) if isinstance(self.layer, list) or self.layer == "all": pass elif isinstance(layer_idx, list): @@ -168,6 +170,7 @@ def reset_clip_options(self): self.layer_idx = self.options_default[1] self.return_projected_pooled = self.options_default[2] self.execution_device = None + self.model_state = None def process_tokens(self, tokens, device): end_token = self.special_tokens.get("end", None) diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index d84d7b7cd11..67c3109e7cf 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -1,7 +1,8 @@ import torch import torch.nn as nn -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, Any, Tuple +import contextlib import math from tqdm import tqdm import comfy.utils @@ -28,6 +29,27 @@ class FixedKV: index: int position: torch.Tensor seqlen: torch.Tensor + storage: Any = field(default=None, init=False) + + def state_tensors(self): + return {"key": (self, "key"), "value": (self, "value")} + + @contextlib.contextmanager + def cast(self, device): + if self.storage is None: + yield + return + fields = self.state_tensors() + originals = {key: getattr(owner, name) for key, (owner, name) in fields.items()} + with contextlib.ExitStack() as stack: + try: + for key, (owner, name) in fields.items(): + value, _ = stack.enter_context(self.storage[key].access(device)) + setattr(owner, name, value) + yield + finally: + for key, (owner, name) in fields.items(): + setattr(owner, name, originals[key]) def prepare(self, num_tokens): self.position.copy_(self.seqlen) @@ -112,6 +134,9 @@ def mask(self, seq): class Int8FixedKVCache(FixedKVState): # INT8 prefix with a bounded BF16 window for page updates and speculative commits. + def state_tensors(self): + return {**super().state_tensors(), **{"packed_" + name: (self.packed, name) for name in ("key", "value", "key_scale", "value_scale")}} + @classmethod def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared): packed = Int8DecodeCache(batch, kv_heads, capacity, device) @@ -120,7 +145,7 @@ def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared): key = torch.zeros((batch, kv_heads, window, head_dim), device=device, dtype=dtype) cache = cls(key, torch.zeros_like(key), 0, *shared) cache.packed = packed - cache.chain_mask = torch.tensor([(1 << (r + 1)) - 1 for r in range(SPEC_ROWS)], dtype=torch.int32, device=device) + cache.chain_mask = torch.tensor([(1 << (r + 1)) - 1 for r in range(SPEC_ROWS)], dtype=torch.int32, device=cache.position.device) return cache def append(self, xk, xv): @@ -148,6 +173,23 @@ def attend(self, xq, xk, xv): mask = tree.mask if tree is not None else self.chain_mask[:seq] return comfy_kitchen.flash_attention_decode_tree_merge(out, lse, xq, xk, xv, mask, torch.empty_like(out)) +class KVCacheModel(nn.Module): + comfy_writable = True + + def __init__(self, caches, extra_caches=()): + super().__init__() + self.caches = caches + self.extra_caches = list(extra_caches) + self.layers = nn.ModuleList() + for cache in self.caches + self.extra_caches: + layer = nn.ModuleDict({key: comfy.ops.WritableWeight(getattr(owner, name).shape, getattr(owner, name).dtype) + for key, (owner, name) in cache.state_tensors().items()}) + self.layers.append(layer) + cache.storage = layer + + def get_dynamic_vram__units(self): + return self.layers, [] + @dataclass class Llama2Config: vocab_size: int = 128320 @@ -1048,7 +1090,8 @@ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermed elif intermediate_output < 0: intermediate_output = len(self.layers) + intermediate_output - prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), x.device, {"prefetch_dynamic_vbars": self.prefetch_dynamic_vbars and past_key_values is not None}) + prefetch_modules = [(layer, kv.storage) if kv.storage is not None else layer for layer, kv in zip(self.layers, past_key_values)] if fixed_kv else list(self.layers) + prefetch_queue = comfy.model_prefetch.make_prefetch_queue(prefetch_modules, x.device, {"prefetch_dynamic_vbars": self.prefetch_dynamic_vbars and past_key_values is not None}) next_key_values = list(past_key_values) if past_key_values is not None else [] for i, layer in enumerate(self.layers): if all_intermediate is not None: @@ -1067,13 +1110,14 @@ def core(): # Plain FixedKV shares GPU counters; prepare once, eager or graph-replayed. if i == 0 and type(past_kv) is FixedKV: past_kv.prepare(seq_len) - output, current_kv = layer( - x=x, - attention_mask=mask, - freqs_cis=freqs_cis, - optimized_attention=optimized_attention, - past_key_value=past_kv, - ) + with past_kv.cast(x.device) if fixed_kv else contextlib.nullcontext(): + output, current_kv = layer( + x=x, + attention_mask=mask, + freqs_cis=freqs_cis, + optimized_attention=optimized_attention, + past_key_value=past_kv, + ) if enable_graph: x.copy_(output) else: @@ -1191,7 +1235,7 @@ def logits(self, x): def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): return self.model.init_kv_cache(batch, max_cache_len, device, execution_dtype) - def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None, position_ids=None, deepstack_embeds=None, visual_pos_masks=None, embeds_info=None): + def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None, position_ids=None, deepstack_embeds=None, visual_pos_masks=None, embeds_info=None, model_state=None): device = embeds.device if stop_tokens is None: @@ -1209,6 +1253,8 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, max_cache_len = embeds.shape[1] + max_length past_key_values = self.init_kv_cache(embeds.shape[0], max_cache_len, device, execution_dtype) + if isinstance(past_key_values, KVCacheModel): + past_key_values = model_state.load(past_key_values, device).caches generator = torch.Generator(device=device).manual_seed(seed) if do_sample else None diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 4719c52dd2d..6989e87dea8 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -17,7 +17,7 @@ from comfy import sd1_clip import comfy.text_encoders.qwen_vl -from .llama import SPEC_ROWS, BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix +from .llama import SPEC_ROWS, BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, KVCacheModel, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix @dataclass @@ -37,6 +37,9 @@ class LinearKV(FixedKV): ctl_tracker: dict = None uncommitted: int = 0 + def state_tensors(self): + return {**super().state_tensors(), **{name: (self, name) for name in ("qkv_buf", "proj_buf", "gates_buf", "sumsq_buf")}} + @staticmethod def deferred_tracker(device): # ctl = {pending, parity, slot[sm], parent[sm], prog[sm]}; without MTP, only row 0 runs. @@ -481,11 +484,11 @@ def kv_cache_type(self, device, dtype): return Int8FixedKVCache return FixedKVCache - def init_kv_cache(self, batch, capacity, device, dtype, shared): + def init_kv_cache(self, batch, capacity, device, dtype, shared, storage_device=None): cache_type = self.kv_cache_type(device, dtype) if cache_type not in shared: shared[cache_type] = cache_type.shared(batch, device) - return cache_type.zeros(batch, self.num_kv_heads, capacity, self.head_dim, device, dtype, shared[cache_type]) + return cache_type.zeros(batch, self.num_kv_heads, capacity, self.head_dim, device if storage_device is None else storage_device, dtype, shared[cache_type]) def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=None, past_key_value=None): batch_size, seq_length, _ = x.shape @@ -898,9 +901,10 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, "repetition_penalty": repetition_penalty, "presence_penalty": kwargs.get("presence_penalty", 0.0) or 0.0, "seed": seed if seed is not None else 42} - return self._generate_mtp(embeds, max_length, stop_tokens, mtp, sampling=sampling, position_ids=kwargs.get("position_ids")) + return self._generate_mtp(embeds, max_length, stop_tokens, mtp, sampling=sampling, position_ids=kwargs.get("position_ids"), + model_state=kwargs.get("model_state")) - def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, position_ids=None): + def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, position_ids=None, model_state=None): device = embeds.device cfg = self.model.config if stop_tokens is None: @@ -910,9 +914,10 @@ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, pos if embeds.ndim == 2: embeds = embeds.unsqueeze(0) cap = embeds.shape[1] + max_length + 7 - pkv = self.init_kv_cache(embeds.shape[0], cap, device, dt) + cache = model_state.load(self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True), device) + pkv = cache.caches # repair window: drafting ahead plus a near-full rollback - mtp_kv = self.mtp.layers[0].self_attn.init_kv_cache(embeds.shape[0], cap, device, dt, {}) + mtp_kv = cache.extra_caches[0] head = self.model.lm_head if hasattr(self.model, "lm_head") else self.model.embed_tokens def verify_logits(x): @@ -979,6 +984,8 @@ def update_progress(n): kv.ctl_tracker["tree"] = tree else: kv.tracker["tree"] = tree + # only attention caches move rows on commit; linear caches record the path for the next forward + attention_kv = [kv for kv in pkv if not isinstance(kv, LinearKV)] verify_tokens = torch.zeros((embeds.shape[0], tree.rows), device=device, dtype=torch.long) verify_buffers = (torch.empty((embeds.shape[0], tree.rows, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, tree.rows).clone()) draft_top = torch.empty((embeds.shape[0], 7 if adaptive else tree.fanout), device=device, dtype=torch.long) @@ -998,14 +1005,15 @@ def draft(token, hidden, p): draft_freqs.copy_(freqs_at(p)) host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed) queue = comfy.model_prefetch.make_prefetch_queue( - [None if host_rows else embed, (head, self.mtp)], device, + [None if host_rows else embed, (head, self.mtp, mtp_kv.storage)], device, {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars}) def embed_core(): draft_emb.copy_(embed.host_rows(draft_token, out_dtype=dt) if host_rows else embed(draft_token).to(dt)) def core(): - n1, r1 = self.mtp(draft_emb, draft_hidden, draft_freqs, mtp_kv) + with mtp_kv.cast(device): + n1, r1 = self.mtp(draft_emb, draft_hidden, draft_freqs, mtp_kv) draft_hidden.copy_(r1) logits = self.logits(n1)[:, -1] if adaptive: @@ -1080,36 +1088,45 @@ def step(): packet = [toks[0], verify_tokens[0, 1:]] if adaptive: packet.append(draft_ranks.reshape(-1)) - vals = torch.cat(packet).tolist() - t, dr = vals[:tree.rows], [None] + vals[tree.rows:2 * tree.rows - 1] - # Continue only along a child containing the main model's chosen token. - path = [0] - while True: - node = next((c for c in tree.children[path[-1]] if dr[c] == t[path[-1]]), None) - if node is None: - break - path.append(node) - ranks = [] - if adaptive: - for node in path: - if node >= depth: + # the accepted rows may move within the attention caches: pin and cast them while the GPU + # is still verifying, so only the moves themselves run after the accept sync + storages = tuple(kv.storage for kv in attention_kv) + queue = comfy.model_prefetch.make_prefetch_queue([storages], device, + {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars}) + with contextlib.ExitStack() as casts: + comfy.model_prefetch.prefetch_queue_pop(queue, device, storages, + core=lambda: [casts.enter_context(kv.cast(device)) for kv in attention_kv]) + vals = torch.cat(packet).tolist() + t, dr = vals[:tree.rows], [None] + vals[tree.rows:2 * tree.rows - 1] + # Continue only along a child containing the main model's chosen token. + path = [0] + while True: + node = next((c for c in tree.children[path[-1]] if dr[c] == t[path[-1]]), None) + if node is None: break - candidates = vals[15 + 7 * node:22 + 7 * node] - ranks.append(next((r for r, token in enumerate(candidates) if token == t[node]), 7)) - if timers: - clocks[0], clocks[1], clocks[2] = 0.0, timers[0].elapsed_time(timers[1]), timers[0].elapsed_time(timers[2]) - else: - clocks[:] = [t * 1000 for t in clocks] - commit = tuple(dr[r] for r in path[1:]) + (t[path[-1]],) - row = path[-1] - next_tok = toks[:, row:row + 1] - if len(path) < tree.rows: - for kv in pkv: - kv.rollback(tree.rows - len(path)) - on_chain = row == len(path) - 1 - if not on_chain: - for kv in pkv: - kv.commit_path(pos, path) + path.append(node) + ranks = [] + if adaptive: + for node in path: + if node >= depth: + break + candidates = vals[15 + 7 * node:22 + 7 * node] + ranks.append(next((r for r, token in enumerate(candidates) if token == t[node]), 7)) + if timers: + clocks[0], clocks[1], clocks[2] = 0.0, timers[0].elapsed_time(timers[1]), timers[0].elapsed_time(timers[2]) + else: + clocks[:] = [t * 1000 for t in clocks] + commit = tuple(dr[r] for r in path[1:]) + (t[path[-1]],) + row = path[-1] + next_tok = toks[:, row:row + 1] + if len(path) < tree.rows: + for kv in pkv: + kv.rollback(tree.rows - len(path)) + on_chain = row == len(path) - 1 + if not on_chain: + for kv in pkv: + kv.commit_path(pos, path) + comfy.model_prefetch.prefetch_queue_pop(queue, device, None) # mtp entries fed by a rejected or off-chain draft token kept = min(len(path) if on_chain else len(path) - 1, depth) if kept < depth: @@ -1156,7 +1173,7 @@ def step(): comfy.model_prefetch.cleanup_prefetch_queues() return ids - def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): + def init_kv_cache(self, batch, max_cache_len, device, execution_dtype, mtp=False): model_config = self.model.config past_key_values = [] shared = {} @@ -1165,21 +1182,22 @@ def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): if model_config.layer_types[i] == "linear_attention": recurrent_state = torch.zeros( [batch, model_config.linear_num_value_heads, model_config.linear_key_head_dim, model_config.linear_value_head_dim], - device=device, dtype=torch.float32 + device="meta", dtype=torch.float32 ) conv_dim = model_config.linear_num_key_heads * model_config.linear_key_head_dim * 2 + model_config.linear_num_value_heads * model_config.linear_value_head_dim conv_state = torch.zeros( [batch, conv_dim, model_config.conv_kernel_size - 1], - device=device, dtype=execution_dtype + device="meta", dtype=execution_dtype ) kv = LinearKV(conv_state, recurrent_state, 0, None, None) kv.qkv_buf, kv.proj_buf, kv.gates_buf, kv.sumsq_buf = comfy_kitchen.gated_delta_deferred_buffers( - batch, conv_dim, model_config.linear_num_value_heads, model_config.linear_num_key_heads, execution_dtype, device) + batch, conv_dim, model_config.linear_num_value_heads, model_config.linear_num_key_heads, execution_dtype, "meta") kv.ctl, kv.ctl_tracker = tracker["ctl"], tracker past_key_values.append(kv) else: - past_key_values.append(self.model.layers[i].self_attn.init_kv_cache(batch, max_cache_len, device, execution_dtype, shared)) - return past_key_values + past_key_values.append(self.model.layers[i].self_attn.init_kv_cache(batch, max_cache_len, device, execution_dtype, shared, storage_device="meta")) + extra = [self.mtp.layers[0].self_attn.init_kv_cache(batch, max_cache_len, device, execution_dtype, {}, storage_device="meta")] if mtp else [] + return KVCacheModel(past_key_values, extra) # Tokenizer and Text Encoder Wrappers @@ -1258,7 +1276,8 @@ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min embeds, _, _, embeds_info = self.process_tokens(tokens_only, self.execution_device) position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device) return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, - presence_penalty=presence_penalty, position_ids=position_ids, mtp=mtp) + presence_penalty=presence_penalty, position_ids=position_ids, mtp=mtp, + model_state=self.model_state) class Qwen35TEModel(sd1_clip.SD1ClipModel): diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py index efd322865cf..b25454df817 100644 --- a/tests-unit/comfy_test/qwen35_decode_test.py +++ b/tests-unit/comfy_test/qwen35_decode_test.py @@ -9,7 +9,10 @@ if not torch.cuda.is_available(): cli_args.cpu = True +import comfy.model_management # noqa: E402 import comfy.ops # noqa: E402 +import comfy.model_state # noqa: E402 +from comfy.model_patcher import ModelPatcher # noqa: E402 from comfy.text_encoders.llama import MLP, RMSNorm # noqa: E402 @@ -337,6 +340,17 @@ class Tiny(qwen35.Qwen35): for name, p in model.named_parameters(): p.data = torch.randn_like(p) * (0.05 if "norm" in name else 0.3) model.model.lm_head.weight.data[:8] *= 4 # a few likely tokens, so some drafts survive + + # Generation loads its KV cache into a model state, as CLIP.generate does; each test ends its node. + monkeypatch.setattr(comfy.model_management, "EPHEMERAL_MODELS", []) + generate = model.generate + + def generate_with_cache(*args, model_state=None, **kwargs): + if model_state is None: + model_state = comfy.model_state.ModelState() + comfy.model_management.register_ephemeral_model(model_state) + return generate(*args, model_state=model_state, **kwargs) + model.generate = generate_with_cache return model @@ -508,8 +522,6 @@ def _attention_config(config): def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch): import comfy.ldm.modules.attention as attention import comfy.lora - from comfy.model_patcher import ModelPatcher - import comfy.text_encoders.llama as llama import comfy.text_encoders.qwen35 as qwen35 from comfy.text_encoders.llama import FixedKVCache @@ -517,7 +529,6 @@ def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch): monkeypatch.setattr(qwen35.comfy_kitchen, "int8_attention_is_available", lambda device: True) monkeypatch.setattr(qwen35.comfy_kitchen, "sol_attn_is_available", lambda device: False) monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: True) - monkeypatch.setattr(llama, "Int8DecodeCache", _FakePacked) monkeypatch.setattr(cli_args, "use_ck_attention", False) model = _tiny_model(monkeypatch, layers=8, head_dim=256) key = "model.layers.3.self_attn.comfy_attention.config" @@ -532,7 +543,7 @@ def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch): def cache_types(): kv = model.init_kv_cache(1, 32, device, torch.bfloat16) draft = model.mtp.layers[0].self_attn.init_kv_cache(1, 32, device, torch.bfloat16, {}) - return type(kv[3]), type(kv[7]), type(draft) + return type(kv.caches[3]), type(kv.caches[7]), type(draft) assert cache_types() == (qwen35.Int8FixedKVCache, FixedKVCache, qwen35.Int8FixedKVCache) monkeypatch.setattr(qwen35.comfy_kitchen, "sol_attn_is_available", lambda device: True) From 88ae8bf885224bc2b52bc99823b4c3abd0cd9a04 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sat, 24 Oct 2026 13:48:38 +1000 Subject: [PATCH 07/24] Support resuming Qwen generation from saved cache state Generation always records its cursor into the model state's metadata: cache length and capacity, the sampled-but-unfed token, the next RoPE position, each cache's commit cursor, the penalty history and RNG. A later generate call with that metadata restores the caches and feeds the pending token with the new prompt from the saved position. Resumed DeltaNet layers prefill through the decode path so their recurrent state survives; a tail cut by max_length or a stop token is rolled back so the cache ends exactly at the returned tokens. --- comfy/text_encoders/llama.py | 57 +++++++++++++++++--- comfy/text_encoders/qwen35.py | 98 ++++++++++++++++++++++++++++------- 2 files changed, 131 insertions(+), 24 deletions(-) diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index 67c3109e7cf..1701d954944 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -34,6 +34,12 @@ class FixedKV: def state_tensors(self): return {"key": (self, "key"), "value": (self, "value")} + def checkpoint_state(self): + return {"index": self.index} + + def restore_state(self, state): + self.index = state["index"] + @contextlib.contextmanager def cast(self, device): if self.storage is None: @@ -66,6 +72,10 @@ def rollback(self, discard=1): class FixedKVState(FixedKV): tracker: dict = None + def restore_state(self, state): + super().restore_state(state) + self.tracker["step"] = None + def prepare(self, num_tokens): tree = self.tracker["tree"] if self.tracker["step"] == (self.index, num_tokens, tree): @@ -190,6 +200,13 @@ def __init__(self, caches, extra_caches=()): def get_dynamic_vram__units(self): return self.layers, [] + def checkpoint_state(self): + return [cache.checkpoint_state() for cache in self.caches + self.extra_caches] + + def restore_state(self, state): + for cache, saved in zip(self.caches + self.extra_caches, state, strict=True): + cache.restore_state(saved) + @dataclass class Llama2Config: vocab_size: int = 128320 @@ -1235,6 +1252,26 @@ def logits(self, x): def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): return self.model.init_kv_cache(batch, max_cache_len, device, execution_dtype) + def prefill(self, embeds, past_key_values, position_ids=None, **kwargs): + return self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, position_ids=position_ids, **kwargs) + + def cache_capacity(self, needed, state): + """The cache length for a generation that needs this many tokens on top of the state's.""" + return needed + + def restore_generation_state(self, state, cache, generator): + if "caches" in state: + cache.restore_state(state["caches"]) + if generator is not None and "rng" in state: + generator.set_state(torch.tensor(state["rng"], dtype=torch.uint8)) + + def save_generation_state(self, state, cache, ids, generator, capacity, position): + # pending is the last sampled token, fed at position on the next turn + state.update(length=cache.caches[0].index, capacity=capacity, pending=ids[-1:], position=position, caches=cache.checkpoint_state(), + penalty_tokens=sorted(set(state.get("penalty_tokens", []) + ids))) + if generator is not None: + state["rng"] = generator.get_state().tolist() + def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None, position_ids=None, deepstack_embeds=None, visual_pos_masks=None, embeds_info=None, model_state=None): device = embeds.device @@ -1251,12 +1288,15 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, if embeds.ndim == 2: embeds = embeds.unsqueeze(0) - max_cache_len = embeds.shape[1] + max_length - past_key_values = self.init_kv_cache(embeds.shape[0], max_cache_len, device, execution_dtype) - if isinstance(past_key_values, KVCacheModel): - past_key_values = model_state.load(past_key_values, device).caches - + state = {} if model_state is None else model_state.metadata + max_cache_len = self.cache_capacity(state.get("length", 0) + embeds.shape[1] + max_length, state) + initial_tokens = state.get("penalty_tokens", initial_tokens) generator = torch.Generator(device=device).manual_seed(seed) if do_sample else None + cache = past_key_values = self.init_kv_cache(embeds.shape[0], max_cache_len, device, execution_dtype) + if isinstance(cache, KVCacheModel): + cache = model_state.load(cache, device) + self.restore_generation_state(state, cache, generator) + past_key_values = cache.caches generated_token_ids = [] pbar = comfy.utils.ProgressBar(max_length) @@ -1302,7 +1342,10 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, if step == 0 and deepstack_embeds is not None: extra["deepstack_embeds"] = deepstack_embeds extra["visual_pos_masks"] = visual_pos_masks - x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids, position_ids=position_ids, **extra, embeds_info=(embeds_info if step == 0 else None)) + if step == 0: + x, _, past_key_values = self.prefill(embeds, past_key_values, position_ids=position_ids, input_ids=current_input_ids, **extra, embeds_info=embeds_info) + else: + x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids, position_ids=position_ids, **extra, embeds_info=None) logits = self.logits(x)[:, -1] if penalty_mask is None and do_sample and penalize: # allocated on the (unbracketed) first step; later steps only index_fill_ it @@ -1336,6 +1379,8 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, finally: comfy.model_prefetch.cleanup_prefetch_queues() + if isinstance(cache, KVCacheModel): + self.save_generation_state(state, cache, generated_token_ids, generator, max_cache_len, cache.caches[0].index if next_pos is None else next_pos) return generated_token_ids def processed_probs(self, logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, presence_penalty=0.0, penalty_mask=None): diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 6989e87dea8..f928295b9bf 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -40,6 +40,14 @@ class LinearKV(FixedKV): def state_tensors(self): return {**super().state_tensors(), **{name: (self, name) for name in ("qkv_buf", "proj_buf", "gates_buf", "sumsq_buf")}} + def checkpoint_state(self): + # the accepted verify rows the next step still has to commit + return {"index": self.index, "uncommitted": self.uncommitted, "parity": self.ctl_tracker["parity"], "path": self.ctl_tracker["path"]} + + def restore_state(self, state): + self.index, self.uncommitted = state["index"], state["uncommitted"] + self.ctl_tracker.update(parity=state["parity"], path=state["path"], step=None) + @staticmethod def deferred_tracker(device): # ctl = {pending, parity, slot[sm], parent[sm], prog[sm]}; without MTP, only row 0 runs. @@ -372,6 +380,12 @@ def forward(self, x, past_key_value=None, **kwargs): proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim] z = self.in_proj_z(x) + # A restored cache can start on a decode step, before any prefill ran here. + if past_key_value is not None and past_key_value.g_decay is None: + past_key_value.g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp() + past_key_value.dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32) + past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device) + if past_key_value is not None and past_key_value.decode_step(seq_len): # decode: conv step into the side buffer, then gates + delta rule + gated norm in one # kernel; both first commit the previous step's accepted tokens @@ -404,12 +418,8 @@ def forward(self, x, past_key_value=None, **kwargs): # Split QKV and compute beta/g mixed_qkv = mixed_qkv.transpose(1, 2) # [B, seq_len, conv_dim] query, key, value = mixed_qkv.split([self.key_dim, self.key_dim, self.value_dim], dim=-1) - g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp() - dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32) - if past_key_value is not None: - past_key_value.g_decay = g_decay - past_key_value.dt_bias = dt_bias - past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device) + g_decay = past_key_value.g_decay if past_key_value is not None else -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp() + dt_bias = past_key_value.dt_bias if past_key_value is not None else comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32) beta = b.sigmoid() g = g_decay * F.softplus(a.float() + dt_bias) @@ -889,6 +899,38 @@ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermed position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device) return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids, past_key_values=past_key_values) + def prefill(self, embeds, past_key_values, position_ids=None, **kwargs): + if past_key_values[0].index == 0: + return super().prefill(embeds, past_key_values, position_ids, **kwargs) + # Resumed DeltaNet must preserve its recurrent and convolution state, so the new + # tokens go through the deferred decode kernels in rows of 1, 4 or 8. A multi-row + # chunk is a verify tree that is one chain: each row attends and commits after the + # row before it. + kwargs.pop("decode_buffers", None) + trackers = [kv.ctl_tracker if isinstance(kv, LinearKV) else kv.tracker for kv in past_key_values] + start = 0 + while start < embeds.shape[1]: + size = next(s for s in (SPEC_ROWS, 4, 1) if start + s <= embeds.shape[1]) + spec = [] + for _ in range(size - 1): + spec = [spec] + tree = VerifyTree(spec, size, embeds.device) if size > 1 else None + for tracker in trackers: + tracker["tree"] = tree + positions = position_ids[..., start:start + size] if position_ids is not None else None + result = super().prefill(embeds[:, start:start + size], past_key_values, positions, **kwargs) + start += size + for tracker in trackers: + tracker["tree"] = None + return result + + def cache_capacity(self, needed, state): + # a saved state fixes its capacity when first created, rounded up so later turns fit + capacity = state.get("capacity", (needed + 4095) // 4096 * 4096) + if needed > capacity: + raise ValueError(f"The saved cache holds {capacity} tokens but this generation needs {needed}") + return capacity + def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, **kwargs): mtp = kwargs.pop("mtp", True) if self.mtp is None or not mtp or kwargs.get("initial_input_ids") is not None: @@ -913,8 +955,12 @@ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, pos embeds = embeds.to(dt) if embeds.ndim == 2: embeds = embeds.unsqueeze(0) - cap = embeds.shape[1] + max_length + 7 + state = model_state.metadata + past_length = state.get("length", 0) + cap = self.cache_capacity(past_length + embeds.shape[1] + max_length + 7, state) + generator = torch.Generator(device=device).manual_seed(sampling["seed"]) if sampling is not None else None cache = model_state.load(self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True), device) + self.restore_generation_state(state, cache, generator) pkv = cache.caches # repair window: drafting ahead plus a near-full rollback mtp_kv = cache.extra_caches[0] @@ -926,13 +972,10 @@ def verify_logits(x): with comfy.ops.CastBiasWeightContext(head, x, offloadable=True) as (w, _bias): return F.linear(x, w) - generator = None - if sampling is not None: - generator = torch.Generator(device=device).manual_seed(sampling["seed"]) penalized = sampling is not None and penalty_active(sampling["repetition_penalty"], sampling["presence_penalty"]) # after an image, text positions run behind the cache index (MRoPE); decode continues from the last prefill position - rope_offset = 0 if position_ids is None else int(position_ids[:, -1].max()) + 1 - embeds.shape[1] + rope_offset = 0 if position_ids is None else int(position_ids[:, -1].max()) + 1 - past_length - embeds.shape[1] # rope table once per generate, sliced per draft ftab = rope_matrix(self.model.compute_freqs_cis(torch.arange(rope_offset, cap + rope_offset, device=device, dtype=torch.float).unsqueeze(0), device)) @@ -943,14 +986,16 @@ def freqs_at(p, n=1): # cross-step state lives in static carriers: nothing allocated inside a step may outlive it nt_buf = torch.empty((embeds.shape[0], 1), device=device, dtype=torch.long) h_buf = torch.empty((embeds.shape[0], 1, cfg.hidden_size), device=device, dtype=dt) - x, _, _ = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=pkv, position_ids=position_ids) + x, _, _ = self.prefill(embeds, pkv, position_ids) lg0 = self.logits(x)[:, -1] + pen_mask = torch.zeros((lg0.shape[-1],), device=device, dtype=torch.bool) if penalized else None + if penalized and "penalty_tokens" in state: + pen_mask.index_fill_(0, torch.tensor(state["penalty_tokens"], device=device, dtype=torch.long), True) if sampling is None: nt_buf.copy_(lg0.argmax(dim=-1, keepdim=True)) else: nt_buf.copy_(self.sample_token(lg0, sampling["temperature"], sampling["top_k"], sampling["top_p"], sampling["min_p"], - sampling["repetition_penalty"], [], generator, presence_penalty=sampling["presence_penalty"])) - pen_mask = torch.zeros((lg0.shape[-1],), device=device, dtype=torch.bool) if penalized else None + sampling["repetition_penalty"], [], generator, presence_penalty=sampling["presence_penalty"], penalty_mask=pen_mask)) h_buf.copy_(x[:, -1:, :]) del x, lg0 if hasattr(self.model.embed_tokens, "_v"): @@ -960,7 +1005,7 @@ def freqs_at(p, n=1): ids = [nt_buf[0].item()] if penalized: pen_mask.index_fill_(0, nt_buf.reshape(-1), True) - pos = embeds.shape[1] + pos = pkv[0].index progress = comfy.utils.ProgressBar(max_length) console = tqdm(total=max_length, desc="Generating tokens", initial=1) @@ -1138,12 +1183,12 @@ def step(): pen_mask.index_fill_(0, verify_tokens[:, r], True) pen_mask.index_fill_(0, nt_buf.reshape(-1), True) pos += len(commit) - return commit, ranks + return commit, ranks, kept try: while len(ids) < max_length and ids[-1] not in stop_tokens: with (comfy.model_prefetch.malloc_graph_scope(device) if compile_allocations else contextlib.nullcontext()): - commit, ranks = step() + commit, ranks, kept = step() if adaptive: steps += 1 for row in hits: @@ -1162,15 +1207,24 @@ def step(): widths = best tree.update(tree_spec(widths)) depth = tree.chain + accepted = len(commit) commit = list(commit[:max_length - len(ids)]) stop = next((i for i, t in enumerate(commit) if t in stop_tokens), None) if stop is not None: del commit[stop + 1:] + if accepted > len(commit): + for kv in pkv: + kv.rollback(accepted - len(commit)) + mtp_kv.rollback(max(0, kept - len(commit))) ids.extend(commit) update_progress(len(commit)) finally: console.close() comfy.model_prefetch.cleanup_prefetch_queues() + # A resumed cache decodes as a chain until its next verify installs a tree. + for kv in pkv: + (kv.ctl_tracker if isinstance(kv, LinearKV) else kv.tracker)["tree"] = None + self.save_generation_state(state, cache, ids, generator, cap, pkv[0].index + rope_offset) return ids def init_kv_cache(self, batch, max_cache_len, device, execution_dtype, mtp=False): @@ -1222,7 +1276,7 @@ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None if image is not None and len(images) == 0: images = [image[i:i + 1] for i in range(image.shape[0])] - skip_template = False + skip_template = kwargs.get("skip_template", False) if text.startswith('<|im_start|>'): skip_template = True if prevent_empty_text and text == '': @@ -1273,8 +1327,16 @@ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min if isinstance(tokens, dict): tokens = next(iter(tokens.values())) tokens_only = [[t[0] for t in b] for b in tokens] + state = self.model_state.metadata + resume = "pending" in state + if resume: + if len(tokens_only) != 1 or any(not isinstance(t, int) for t in tokens_only[0]): + raise ValueError("Continuing a saved generation supports one text sequence") + tokens_only[0] = state["pending"] + tokens_only[0] embeds, _, _, embeds_info = self.process_tokens(tokens_only, self.execution_device) position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device) + if resume: + position_ids = torch.arange(state["position"], state["position"] + embeds.shape[1], device=embeds.device).unsqueeze(0) return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=presence_penalty, position_ids=position_ids, mtp=mtp, model_state=self.model_state) From f3235764adfbfc4362ff6132fca7bfeadc63151a Mon Sep 17 00:00:00 2001 From: Rattus Date: Sat, 24 Oct 2026 22:17:09 +1000 Subject: [PATCH 08/24] Support anonymous files on other POSIX platforms Writable models flush into anonymous files on every platform, so fall back to an unlinked temporary file where memfd is unavailable. --- comfy/anonymous_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/comfy/anonymous_file.py b/comfy/anonymous_file.py index 0aa187db8eb..8077a5f7242 100644 --- a/comfy/anonymous_file.py +++ b/comfy/anonymous_file.py @@ -25,7 +25,8 @@ def create_anonymous_file(size=0): """Return an unbuffered binary file owned by the caller, initially sized in bytes. - Uses Linux memfd or a Windows temporary-cache/delete-on-close file. + Uses Linux memfd, a Windows temporary-cache/delete-on-close file, or an + unlinked temporary file elsewhere. Windows may still write to disk; Linux memfd may be swapped. Initial sizing is sparse with zero-filled holes; Windows requires a filesystem supporting sparse files. Keep the file open through all I/O and close @@ -47,7 +48,7 @@ def create_anonymous_file(size=0): raise OSError(errno, os.strerror(errno)) file = open(fd, "w+b", buffering=0) else: - raise NotImplementedError("anonymous files require Linux or Windows") + file = tempfile.TemporaryFile(buffering=0) try: if os.name == "nt": From cf157b606ecc487e1d0b6fef97ba2b6c859d2744 Mon Sep 17 00:00:00 2001 From: Rattus Date: Mon, 26 Oct 2026 16:04:22 +1000 Subject: [PATCH 09/24] Add Open Model State node Model State is a new core type: a writable model's values over a file, such as an LLM's KV cache. Open Model State opens a state file in the output folder, or starts an empty state when the file is missing or the node is set to always start fresh, and hands the state to a sampler. The node runs again whenever the file changes. --- comfy_api/latest/_io.py | 8 +++++ comfy_extras/nodes_model_state.py | 56 +++++++++++++++++++++++++++++++ nodes.py | 1 + 3 files changed, 65 insertions(+) create mode 100644 comfy_extras/nodes_model_state.py diff --git a/comfy_api/latest/_io.py b/comfy_api/latest/_io.py index 42d2d1f374f..09d3349b30e 100644 --- a/comfy_api/latest/_io.py +++ b/comfy_api/latest/_io.py @@ -21,6 +21,7 @@ from comfy.controlnet import ControlNet from comfy.hooks import HookGroup, HookKeyframeGroup from comfy.model_patcher import ModelPatcher + from comfy.model_state import ModelState as ModelState_ from comfy.samplers import CFGGuider, Sampler from comfy.sd import CLIP, VAE from comfy.sd import StyleModel as StyleModel_ @@ -615,6 +616,12 @@ class Model(ComfyTypeIO): if TYPE_CHECKING: Type = ModelPatcher +@comfytype(io_type="MODEL_STATE") +class ModelState(ComfyTypeIO): + """A writable model's values over a file, such as an LLM's KV cache.""" + if TYPE_CHECKING: + Type = ModelState_ + @comfytype(io_type="BACKGROUND_REMOVAL") class BackgroundRemoval(ComfyTypeIO): if TYPE_CHECKING: @@ -2646,6 +2653,7 @@ def as_dict(self): "ControlNet", "Vae", "Model", + "ModelState", "ModelPatch", "ClipVision", "ClipVisionOutput", diff --git a/comfy_extras/nodes_model_state.py b/comfy_extras/nodes_model_state.py new file mode 100644 index 00000000000..3f725bdf6fb --- /dev/null +++ b/comfy_extras/nodes_model_state.py @@ -0,0 +1,56 @@ +import os + +from typing_extensions import override + +import comfy.model_state +import folder_paths +from comfy_api.latest import ComfyExtension, io + + +def state_path(filename): + output_dir = folder_paths.get_output_directory() + path = os.path.abspath(os.path.join(output_dir, filename)) + if not folder_paths.is_within_directory(output_dir, path): + raise ValueError(f"Model state files must be inside the output folder: {filename}") + os.makedirs(os.path.dirname(path), exist_ok=True) + return path + + +class OpenModelState(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="OpenModelState", + display_name="Open Model State", + category="model_state", + search_aliases=["KV cache", "conversation", "resume"], + description="Opens a model state file in the output folder, such as a text generation conversation, so a sampler can continue from it. The file is only written by Save Model State.", + inputs=[ + io.String.Input("filename", default="model_states/ComfyUI.safetensors"), + io.Combo.Input("create", options=["if_missing", "never", "always"], default="if_missing", tooltip="What to do when the file is missing: start from an empty state, or fail. always ignores the file and starts empty on every run."), + ], + outputs=[io.ModelState.Output()], + ) + + @classmethod + def fingerprint_inputs(cls, filename, create): + if create == "always": + return float("NaN") + path = state_path(filename) + return os.stat(path).st_mtime_ns if os.path.exists(path) else None + + @classmethod + def execute(cls, filename, create) -> io.NodeOutput: + return io.NodeOutput(comfy.model_state.ModelState.open(state_path(filename), create)) + + +class ModelStateExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + OpenModelState, + ] + + +async def comfy_entrypoint() -> ModelStateExtension: + return ModelStateExtension() diff --git a/nodes.py b/nodes.py index e279ae2fec4..89ca4196817 100644 --- a/nodes.py +++ b/nodes.py @@ -2527,6 +2527,7 @@ async def init_builtin_extra_nodes(): "nodes_glsl.py", "nodes_lora_debug.py", "nodes_textgen.py", + "nodes_model_state.py", "nodes_text_overlay.py", "nodes_color.py", "nodes_toolkit.py", From 4263238729bdfc46c807a393bfebd5ea561450db Mon Sep 17 00:00:00 2001 From: Rattus Date: Mon, 26 Oct 2026 16:04:33 +1000 Subject: [PATCH 10/24] Add Save Model State node Writes a model state's values and metadata to a file in the output folder, typically the one it was opened from, and passes the state on so a loop can end on it. --- comfy_extras/nodes_model_state.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/comfy_extras/nodes_model_state.py b/comfy_extras/nodes_model_state.py index 3f725bdf6fb..2786c9190c4 100644 --- a/comfy_extras/nodes_model_state.py +++ b/comfy_extras/nodes_model_state.py @@ -44,11 +44,34 @@ def execute(cls, filename, create) -> io.NodeOutput: return io.NodeOutput(comfy.model_state.ModelState.open(state_path(filename), create)) +class SaveModelState(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="SaveModelState", + display_name="Save Model State", + category="model_state", + description="Writes a model state's values to a file in the output folder.", + is_output_node=True, + inputs=[ + io.ModelState.Input("state"), + io.String.Input("filename", default="model_states/ComfyUI.safetensors"), + ], + outputs=[io.ModelState.Output(tooltip="The saved state, for chaining or ending a loop.")], + ) + + @classmethod + def execute(cls, state, filename) -> io.NodeOutput: + state.save(state_path(filename)) + return io.NodeOutput(state) + + class ModelStateExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[io.ComfyNode]]: return [ OpenModelState, + SaveModelState, ] From 3db50de4006cbda224578117ee92683541197add Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Nov 2026 21:11:26 +1000 Subject: [PATCH 11/24] Generate text into model states CLIP.generate takes an optional model state to continue from. The TextGenerate node copies its input state so the input is left as it was and two generations from it are independent, applies the chat template's system prompt only when the state holds no conversation yet, and flushes the output state back to CPU only when another node consumes it; otherwise the loaded cache is released after the node as before. TextGenerateLTX2Prompt does not expose the state. --- comfy/sd.py | 6 ++++-- comfy_extras/nodes_textgen.py | 27 +++++++++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/comfy/sd.py b/comfy/sd.py index 5c11904c1ef..d3ea95672f5 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -477,7 +477,8 @@ def load_model(self, tokens={}): def get_key_patches(self): return self.patcher.get_key_patches() - def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True): + def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True, state=None): + """Generate tokens into a model state, continuing the generation it holds. The state stays loaded until flushed, or released after the node.""" self.cond_stage_model.reset_clip_options() self.load_model(tokens) @@ -485,7 +486,8 @@ def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_ self.cond_stage_model.set_clip_options({"layer": None}) self.cond_stage_model.set_clip_options({"execution_device": device}) - state = comfy.model_state.ModelState() + if state is None: + state = comfy.model_state.ModelState() model_management.register_ephemeral_model(state) with model_management.cuda_device_context(device), comfy.ops.use_quantized_matmul(self.cond_stage_model, device): self.cond_stage_model.set_clip_options({"model_state": state}) diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py index 0e7d3a9af6c..dba50f8375d 100644 --- a/comfy_extras/nodes_textgen.py +++ b/comfy_extras/nodes_textgen.py @@ -1,6 +1,8 @@ from comfy_api.latest import ComfyExtension, io from typing_extensions import override +import comfy.model_state + class TextGenerate(io.ComfyNode): @classmethod def define_schema(cls): @@ -41,19 +43,26 @@ def define_schema(cls): io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True), io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth and breadth to observed token matches and execution cost; 2-5 pins a constant draft depth; tree modes pin both breadth and depth. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."), io.String.Input("system_prompt", force_input=True, optional=True, tooltip="Replaces the system prompt in the model's chat template. Ignored when the default template is not used."), + io.ModelState.Input("state", optional=True, tooltip="Continues the conversation held by this model state and extends it with this turn. The prompt should be a single text turn."), ], outputs=[ io.String.Output(display_name="generated_text"), io.String.Output(display_name="thinking"), + io.ModelState.Output(display_name="state", tooltip="A copy of the input state extended by this turn; the input is left as it was."), ], + hidden=[io.Hidden.unique_id, io.Hidden.dynprompt], ) @classmethod - def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="") -> io.NodeOutput: + def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="", state=None) -> io.NodeOutput: mtp = {"off": False, "auto": True}.get(mtp, mtp) - tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt if use_default_template else "") + # the system prompt opens a conversation; later turns continue it + if not use_default_template or (state is not None and state.metadata): + system_prompt = "" + state = comfy.model_state.ModelState() if state is None else state.copy() + tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt) # Get sampling parameters from dynamic combo do_sample = sampling_mode.get("sampling_mode") == "on" @@ -76,8 +85,13 @@ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=F repetition_penalty=repetition_penalty, presence_penalty=presence_penalty, seed=seed, - mtp=mtp + mtp=mtp, + state=state, ) + # the output is written back only when something consumes it; otherwise it is released after the node + dynprompt = cls.hidden.dynprompt + if any(value == [cls.hidden.unique_id, 2] for node_id in dynprompt.all_node_ids() for value in dynprompt.get_node(node_id)["inputs"].values()): + state.flush() generated_text = clip.decode(generated_ids) @@ -90,7 +104,7 @@ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=F if not prompt.rstrip().endswith("") and not clip.decode(prompt_ids[-4:]).rstrip().endswith(""): reasoning, text = "", generated_text - return io.NodeOutput(text.strip(), reasoning.replace("", "", 1).strip()) + return io.NodeOutput(text.strip(), reasoning.replace("", "", 1).strip(), state) LTX2_T2V_SYSTEM_PROMPT = """You are a Creative Assistant. Given a user's raw input prompt describing a scene or concept, expand it into a detailed video generation prompt with specific visuals and integrated audio to guide a text-to-video model. @@ -233,8 +247,9 @@ def define_schema(cls): node_id="TextGenerateLTX2Prompt", display_name="Generate LTX2 Prompt", category=parent_schema.category, - inputs=parent_schema.inputs, - outputs=parent_schema.outputs, + inputs=[i for i in parent_schema.inputs if i.id != "state"], + outputs=parent_schema.outputs[:2], + hidden=parent_schema.hidden, search_aliases=["prompt enhance", "LLM", "gemma"], ) From 855d358e3f2f2a793d2ffae5da5ea9044f7536aa Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Nov 2026 21:13:15 +1000 Subject: [PATCH 12/24] Add tests for model states Covers loading, flushing, saving, copying and reopening a model state, continuing a Qwen generation from a saved state, and a CUDA script that checks plain, resident, pinned and file-only states against each other. --- tests-unit/comfy_test/qwen35_decode_test.py | 23 +++ .../comfy_test/qwen_writable_cache_cuda.py | 170 ++++++++++++++++++ tests-unit/comfy_test/tensor_file_test.py | 101 ++++++++++- 3 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 tests-unit/comfy_test/qwen_writable_cache_cuda.py diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py index b25454df817..5eeb2f47c23 100644 --- a/tests-unit/comfy_test/qwen35_decode_test.py +++ b/tests-unit/comfy_test/qwen35_decode_test.py @@ -424,6 +424,29 @@ def best_widths(widths, hits, draft_ms, verify_ms): assert len(set(switches)) == 4, switches +@pytest.mark.parametrize("mtp", [False, "3", "tree2s2"]) +def test_continuation_from_model_state_matches_uninterrupted_generation(monkeypatch, tmp_path, mtp): + model = _tiny_model(monkeypatch) + embeds = torch.randn(1, 5, 64) + expected = model.generate(embeds.clone(), do_sample=False, max_length=20, mtp=False) + for cutoff in (2, 4, 7, 9): + path = str(tmp_path / f"{cutoff}.safetensors") + state = comfy.model_state.ModelState.open(path) + first = model.generate(embeds.clone(), do_sample=False, max_length=cutoff, mtp=mtp, model_state=state) + assert state.metadata["length"] == embeds.shape[1] + cutoff - 1 + assert state.metadata["pending"] == first[-1:] + state.flush() + state.save(path) + + # The next turn reopens the saved file, as a later prompt would. + saved = comfy.model_state.ModelState.open(path, create="never") + pending = model.model.embed_tokens(torch.tensor([saved.metadata["pending"]])) + rest = model.generate(pending, do_sample=False, max_length=20 - cutoff, mtp=mtp, model_state=saved, + position_ids=torch.tensor([[saved.metadata["position"]]])) + saved.release() + assert first + rest == expected + + class _FakePacked: # Int8DecodeCache boundary with 4-row pages: records the BF16 refreshes it is asked for def __init__(self, batch, heads, capacity, device): diff --git a/tests-unit/comfy_test/qwen_writable_cache_cuda.py b/tests-unit/comfy_test/qwen_writable_cache_cuda.py new file mode 100644 index 00000000000..783f6a1085c --- /dev/null +++ b/tests-unit/comfy_test/qwen_writable_cache_cuda.py @@ -0,0 +1,170 @@ +"""Bounded hybrid Qwen generation against plain, resident, pinned and file-only model states. Run on a disposable CUDA host.""" +import json +import os +from pathlib import Path +import sys +import tempfile +import weakref + +os.environ.setdefault("PYTORCH_ALLOC_CONF", "backend:cudaMallocAsync") +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import safetensors.torch +import torch +import comfy_aimdo.control as aimdo + +assert aimdo.init("cuda") and aimdo.init_device(0) +from comfy.cli_args import args +args.use_ck_attention = True +import comfy.ldm.wan.model_animate2 # noqa: F401 -- initialize the model registry before the patcher +from comfy import memory_management, model_management as mm, model_patcher, model_prefetch as prefetch, model_state, ops, storage +from comfy.text_encoders import qwen35 as qwen +from comfy_kitchen.int8_decode import is_available + +memory_management.aimdo_enabled = True +device = torch.device("cuda", 0) +qwen.QWEN35_MODELS["probe"] = dict(vision=dict(hidden_size=32, num_heads=2, intermediate_size=64, depth=1)) + +class Probe(qwen.Qwen35): + model_type = "probe" + +config = dict(vocab_size=97, hidden_size=64, intermediate_size=128, num_hidden_layers=4, + layer_types=qwen._qwen35_layer_types(4), num_attention_heads=2, num_key_value_heads=1, + head_dim=256, linear_num_key_heads=2, linear_num_value_heads=2, + linear_key_head_dim=128, linear_value_head_dim=128, mtp=True, lm_head=True, stop_tokens=[]) + +with torch.inference_mode(), tempfile.TemporaryDirectory() as tmp: + torch.manual_seed(37) + memory_management.aimdo_enabled = False + model = Probe(config, torch.bfloat16, "cpu", ops.manual_cast) + memory_management.aimdo_enabled = True + for name, p in model.named_parameters(): + p.copy_(torch.randn_like(p) * (0.05 if "norm" in name else 0.08)) + model.model.lm_head.weight[:8].mul_(4) + patcher = model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu")) + mm.load_models_gpu([patcher]) + patcher.set_in_use_by_current_prompt(True) + original_forward = model.model.forward + outputs = [] + captured = set() + states = [] + mode = "hard" + + class _ModeState(model_state.ModelState): + # The hard mode loads without dynamic VRAM; the others toggle it back on for the model under test. + def __init__(self, mode): + super().__init__() + self.mode = mode + + def load(self, model, device): + states.append(self) + memory_management.aimdo_enabled = self.mode != "hard" + loaded = super().load(model, device) + memory_management.aimdo_enabled = True + return loaded + + def forward(*args, **kwargs): + for s in states: + if mode in ("ram", "disk") and not outputs or mode == "evict" and len(outputs) == 4: + s.patcher._vbar_get().set_watermark(0) + if mode == "evict" and len(outputs) == 8: + mm.DISABLE_PINNED_MEMORY = True + s.patcher.partially_unload_ram(1e32) + result = original_forward(*args, **kwargs) + for s in states: + if mode in ("ram", "disk") or mode == "evict" and len(outputs) >= 4: + assert s.patcher.loaded_size() == 0 + if mode == "disk" or mode == "evict" and len(outputs) >= 8: + assert s.patcher.loaded_ram_size() == 0 + outputs.append(result[0].cpu().clone()) + captured.update(i for i, layer in enumerate(model.model.layers) if hasattr(layer, "_comfy_graph")) + return result + + model.model.forward = forward + for attention, length in (("bf16", 19), ("int8", 19), ("int8", 1533)): + mm.args.use_ck_attention = attention == "int8" + embeds = torch.randn(1, length, 64, device=device, dtype=torch.bfloat16) + continuation = torch.randn(1, 1, 64, device=device, dtype=torch.bfloat16) + mm.args.disable_fast_disk = length <= 512 + storage.anonymous_fast_disk.cache_clear() + if attention == "int8": + assert is_available(device) + for layer in [model.model.layers[3], model.mtp.layers[0]]: + layer.self_attn.comfy_attention.method = "comfy_kitchen_int8" if attention == "int8" else "sdpa" + for graphs, queue in ((False, True), (True, True)): + mm.args.disable_cuda_graphs = not graphs + model.model.prefetch_dynamic_vbars = queue + for mtp in (False, "3"): + baseline = None + for mode in ("hard", "resident", "ram", "disk", "evict"): + mm.DISABLE_PINNED_MEMORY = mode == "disk" + outputs.clear() + captured.clear() + # The resident run is saved to a named file; the others stay anonymous. + path = os.path.join(tmp, f"{attention}-{length}-{graphs}-{mtp}.safetensors") if mode == "resident" else None + state = _ModeState(mode) + mm.register_ephemeral_model(state) + ids = model.generate(embeds.clone(), do_sample=True, seed=71, max_length=24, mtp=mtp, model_state=state) + # A consumer flushes the values back to CPU tensors, which drops the loaded model. + dead = [weakref.ref(s.patcher) for s in states] + [weakref.ref(s.patcher.model) for s in states] + states.clear() + state.flush() + assert state.patcher is None + saved = state.tensors + if mode == "hard": + # A plain model state restores its tensors into device memory. + reopened = state.load(model.init_kv_cache(1, state.metadata["capacity"], device, torch.bfloat16, mtp=bool(mtp)), device) + assert all(torch.equal(v.cpu(), saved[k]) for k, v in reopened.state_dict().items()) + del reopened + state.release() + if path is not None: + state.save(path) + assert all(torch.equal(v, saved[k]) for k, v in safetensors.torch.load_file(path).items()) + # Node end: the flushed patcher and its model must both be dead with nothing else holding them. + mm.reset_cast_buffers() + mm.cleanup_ephemeral_models() + assert not mm.EPHEMERAL_MODELS and all(ref() is None for ref in dead), (mode, dead) + assert any(cache.get("uncommitted") for cache in state.metadata["caches"]) # The last DeltaNet update has not been committed. + # The next turn copies the state, reopened by name or still in memory, onto a fresh cache. + source = state if path is None else model_state.ModelState.open(path, create="never") + assert source.metadata == json.loads(json.dumps(state.metadata)) + turn = source.copy() + assert turn.metadata == source.metadata and turn.metadata is not source.metadata + assert all(torch.equal(v, saved[k]) for k, v in turn.tensors.items()) + restored = turn.load(model.init_kv_cache(1, turn.metadata["capacity"], device, torch.bfloat16, mtp=bool(mtp)), device) + restored.restore_state(turn.metadata["caches"]) + resumed = original_forward(None, embeds=continuation, attention_mask=None, past_key_values=restored.caches)[0].clone() + prefetch.cleanup_prefetch_queues() + turn.release() + del restored + if baseline is None: + baseline = ids, list(outputs), resumed + else: + assert ids == baseline[0], (attention, graphs, queue, mtp, mode, ids, baseline[0]) + assert len(outputs) == len(baseline[1]) + error = max((a.float() - b.float()).abs().max().item() for a, b in zip(outputs, baseline[1])) + assert error == 0, (attention, graphs, queue, mtp, mode, error) + assert torch.equal(resumed, baseline[2]), ("resume", attention, graphs, queue, mtp, mode) + if graphs and mode in ("hard", "resident", "evict"): + assert captured + if mode in ("ram", "disk"): + assert not captured + print("QWEN_WRITABLE_PASS", attention, length, graphs, queue, mtp, mode, len(ids), len(captured), flush=True) # noqa: T201 + pending = torch.nn.Sequential(ops.WritableWeight((1,), torch.float32)) + pending.comfy_writable = True + output = torch.empty(2 ** 20, device=device) + state = model_state.ModelState() + mm.register_ephemeral_model(state) + state.load(pending, device) + with prefetch.malloc_graph_scope(device): + temporary = torch.full_like(output, 1.25) + torch.cuda._sleep(50_000_000) + output.copy_(temporary) + del temporary + del pending + prefetch.cleanup_prefetch_queues() + mm.reset_cast_buffers() + mm.cleanup_ephemeral_models() + assert torch.all(output.cpu() == 1.25), "Writable teardown freed pages before pending work finished" + mm.unload_model_and_clones(patcher) +print("QWEN_WRITABLE_ALL_PASS", flush=True) # noqa: T201 diff --git a/tests-unit/comfy_test/tensor_file_test.py b/tests-unit/comfy_test/tensor_file_test.py index abe30152cd8..6c343ca6d4c 100644 --- a/tests-unit/comfy_test/tensor_file_test.py +++ b/tests-unit/comfy_test/tensor_file_test.py @@ -5,8 +5,14 @@ import safetensors.torch import torch -from comfy.memory_management import read_tensor_file_slice_into, write_tensor_file_slice -from comfy.utils import create_safetensors +from comfy.cli_args import args as cli_args + +if not torch.cuda.is_available(): + cli_args.cpu = True + +import comfy.model_state # noqa: E402 +from comfy.memory_management import read_tensor_file_slice_into, write_tensor_file_slice # noqa: E402 +from comfy.utils import create_safetensors # noqa: E402 pytestmark = pytest.mark.skipif( os.name != "nt" and sys.platform != "linux", @@ -65,3 +71,94 @@ def test_anonymous_safetensors(): assert torch.equal(safetensors.torch.load(file.read())["kv"], expected) finally: file.close() + + +class _KV(torch.nn.Module): + def __init__(self, rows=4): + super().__init__() + self.register_buffer("k", torch.empty(rows, 8)) + self.register_buffer("v", torch.empty(rows, 8, dtype=torch.float16)) + + +def test_model_state_persists_values_and_metadata(tmp_path): + cpu = torch.device("cpu") + path = str(tmp_path / "state.safetensors") + state = comfy.model_state.ModelState.open(path) + assert state.tensors == {} and state.metadata == {} + model = state.load(_KV(), cpu) + assert not model.k.count_nonzero() and not model.v.count_nonzero() + model.k.fill_(3.0) + model.v.fill_(5.0) + state.metadata["position"] = 7 + state.flush() + assert state.patcher is None + state.save(path) + + saved = safetensors.torch.load_file(path) + assert torch.equal(saved["k"], torch.full((4, 8), 3.0)) and torch.equal(saved["v"], torch.full((4, 8), 5.0, dtype=torch.float16)) + + reopened = comfy.model_state.ModelState.open(path, create="never") + assert reopened.metadata == {"position": 7} + + # The next turn continues on a copy; the opened state only reaches the file when saved. + turn = reopened.copy() + model = turn.load(_KV(), cpu) + assert torch.equal(model.k, saved["k"]) and torch.equal(model.v, saved["v"]) + model.k[0, 0] = 9.0 + turn.metadata["position"] = 8 + turn.flush() + reopened.save(path) + assert safetensors.torch.load_file(path)["k"][0, 0] == 3.0 + turn.save(path) + assert safetensors.torch.load_file(path)["k"][0, 0] == 9.0 + assert comfy.model_state.ModelState.open(path).metadata == {"position": 8} + + with pytest.raises(FileNotFoundError): + comfy.model_state.ModelState.open(str(tmp_path / "missing.safetensors"), create="never") + with pytest.raises(RuntimeError): + comfy.model_state.ModelState.open(path).load(_KV(rows=1), cpu) + + recreated = comfy.model_state.ModelState.open(path, create="always") + assert recreated.metadata == {} and recreated.tensors == {} + assert not recreated.load(_KV(), cpu).k.count_nonzero() + assert safetensors.torch.load_file(path)["k"][0, 0] == 9.0 + + +def test_model_state_rejects_plain_safetensors(tmp_path): + path = str(tmp_path / "plain.safetensors") + safetensors.torch.save_file({"k": torch.zeros(4, 8)}, path) + with pytest.raises(ValueError): + comfy.model_state.ModelState.open(path) + + +def test_model_state_copies_are_independent(tmp_path): + cpu = torch.device("cpu") + first = comfy.model_state.ModelState() + model = first.load(_KV(), cpu) + model.k.fill_(2.0) + first.metadata["position"] = 3 + first.flush() + + second = first.copy() + assert second.metadata == {"position": 3} and second.metadata is not first.metadata + model = second.load(_KV(), cpu) + model.k[0, 0] = 1.0 + second.metadata["position"] = 4 + second.flush() + + third = second.copy() + assert third.metadata == {"position": 4} + assert torch.equal(third.load(_KV(), cpu).k, model.k) and model.k[0, 0] == 1.0 + assert torch.equal(first.load(_KV(), cpu).k, torch.full((4, 8), 2.0)) and first.metadata == {"position": 3} + + # Releasing a loaded state drops values its metadata may already describe, so it cannot reach another node. + first.patcher.model.k.fill_(5.0) + first.release() + with pytest.raises(RuntimeError): + first.copy() + with pytest.raises(RuntimeError): + first.save(str(tmp_path / "released.safetensors")) + # A written back state stays usable. + third.flush() + third.release() + assert torch.equal(third.copy().load(_KV(), cpu).k, model.k) From 02020d859d99f4800da69400551d364c9f5024e6 Mon Sep 17 00:00:00 2001 From: Rattus Date: Mon, 2 Nov 2026 19:55:43 +1000 Subject: [PATCH 13/24] Apply review suggestions to the model state nodes Name both nodes' state output "state", move Save Model State to model/merging next to the other save nodes, and say where it writes. --- comfy_extras/nodes_model_state.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/comfy_extras/nodes_model_state.py b/comfy_extras/nodes_model_state.py index 2786c9190c4..5463b420484 100644 --- a/comfy_extras/nodes_model_state.py +++ b/comfy_extras/nodes_model_state.py @@ -29,7 +29,7 @@ def define_schema(cls) -> io.Schema: io.String.Input("filename", default="model_states/ComfyUI.safetensors"), io.Combo.Input("create", options=["if_missing", "never", "always"], default="if_missing", tooltip="What to do when the file is missing: start from an empty state, or fail. always ignores the file and starts empty on every run."), ], - outputs=[io.ModelState.Output()], + outputs=[io.ModelState.Output(display_name="state")], ) @classmethod @@ -50,14 +50,14 @@ def define_schema(cls) -> io.Schema: return io.Schema( node_id="SaveModelState", display_name="Save Model State", - category="model_state", - description="Writes a model state's values to a file in the output folder.", + category="model/merging", + description="Writes a model state's values to a file in the folder output/model_states.", is_output_node=True, inputs=[ io.ModelState.Input("state"), io.String.Input("filename", default="model_states/ComfyUI.safetensors"), ], - outputs=[io.ModelState.Output(tooltip="The saved state, for chaining or ending a loop.")], + outputs=[io.ModelState.Output(display_name="state", tooltip="The saved state, for chaining or ending a loop.")], ) @classmethod From aa9fcaa18b1c3f82fbd3a5a60071eb843f7ade83 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 4 Nov 2026 13:52:17 -0800 Subject: [PATCH 14/24] Continue a saved Qwen3.5 state with one batched prefill A continued turn with more than SPEC_ROWS new tokens prefilled 8 rows at a time through the decode kernels. It now prefills in one batch: GatedDeltaNet starts its conv window and delta rule from the cached state, and the causal mask covers the cached prefix plus the new rows. The int8 cache's new rows attend the quantized prefix through the decode kernel and each other causally, merged by log-sum-exp, then the quantized pages grow one page per update through the bf16 window. A saved state used to end with the last step's accepted rows still uncommitted in the DeltaNet state. It now ends before them and stores them in pending, so the next turn re-feeds them. --- comfy/text_encoders/llama.py | 45 ++++++- comfy/text_encoders/qwen35.py | 51 +++++-- tests-unit/comfy_test/qwen35_decode_test.py | 139 +++++++++++++++++++- 3 files changed, 222 insertions(+), 13 deletions(-) diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index 1701d954944..23e43737271 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -183,6 +183,48 @@ def attend(self, xq, xk, xv): mask = tree.mask if tree is not None else self.chain_mask[:seq] return comfy_kitchen.flash_attention_decode_tree_merge(out, lse, xq, xk, xv, mask, torch.empty_like(out)) + def extend(self, xq, xk, xv): + # A prefill continuing a resumed prefix: the new rows attend the quantized prefix through the + # decode kernel and each other causally, then the quantized pages grow one page per update. + start, seq = self.index, xk.shape[2] + length = torch.full_like(self.position[:1], start) + self.packed.update(self.key, self.value, length) + step = max(1, 64 // (xq.shape[1] // xk.shape[1])) # query rows per call, no more than a verify step uses + parts = [self.packed.attend(xq[:, :, r:r + step].contiguous(), length) for r in range(0, seq, step)] + out = causal_merge(torch.cat([o for o, _ in parts], dim=1), torch.cat([lse for _, lse in parts], dim=2), xq, xk, xv) + window, page = self.key.shape[2], self.packed.key.shape[3] + pos = start + while pos < start + seq: + end = min(start + seq, (pos // page + 1) * page) + slots = torch.arange(pos, end, device=xk.device).remainder(window) + self.key.index_copy_(2, slots, xk[:, :, pos - start:end - start]) + self.value.index_copy_(2, slots, xv[:, :, pos - start:end - start]) + self.packed.update(self.key, self.value, torch.full_like(length, end)) + pos = end + return out + + +def causal_merge(out, lse, q, k, v, chunk=512): + """Fold rows' causal attention over each other into their attention over a prefix. + + out [B, S, H*D] and lse [B, H, S] are every row's attention over the prefix; q [B, H, S, D] and + k/v [B, Hk, S, D] the rows themselves. Row i also attends rows <= i. Returns [B, S, H*D].""" + batch, heads, rows, dim = q.shape + kf = k.float().repeat_interleave(heads // k.shape[1], dim=1) + vf = v.float().repeat_interleave(heads // k.shape[1], dim=1) + merged = torch.empty_like(out) + for r in range(0, rows, chunk): + n = min(chunk, rows - r) + scores = torch.matmul(q[:, :, r:r + n].float(), kf[:, :, :r + n].transpose(-1, -2)) * dim ** -0.5 + scores.masked_fill_(torch.ones(n, r + n, dtype=torch.bool, device=q.device).triu_(r + 1), float("-inf")) + m = torch.maximum(lse[:, :, r:r + n], scores.amax(-1)) + w_prefix = torch.exp(lse[:, :, r:r + n] - m) + w = torch.exp(scores - m.unsqueeze(-1)) + acc = out[:, r:r + n].view(batch, n, heads, dim).transpose(1, 2).float() * w_prefix.unsqueeze(-1) + torch.matmul(w, vf[:, :, :r + n]) + merged[:, r:r + n] = (acc / (w_prefix + w.sum(-1)).unsqueeze(-1)).transpose(1, 2).reshape(batch, n, heads * dim).to(out.dtype) + return merged + + class KVCacheModel(nn.Module): comfy_writable = True @@ -1075,7 +1117,8 @@ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermed spec_decode = fixed_kv and any(isinstance(kv, FixedKVState) for kv in past_key_values) and 2 <= seq_len <= SPEC_ROWS and past_len > 0 and attention_mask is None if seq_len > 1 and not spec_decode: # spec verify: the staircase decode bias is causal - causal_mask = torch.empty(past_len + seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(1) + # rows are this call's tokens, columns the cached prefix plus them (a prefill continuing a prefix) + causal_mask = torch.empty(seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(past_len + 1) if mask is not None: mask += causal_mask else: diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index f928295b9bf..db5a5e0e246 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -409,11 +409,19 @@ def forward(self, x, past_key_value=None, **kwargs): a = self.in_proj_a(x) # Conv1d - if past_key_value is not None: - conv_state = past_key_value.conv_state - conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0)) - conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:]) - mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len]) + continuing = past_key_value is not None and past_key_value.index > 0 + if continuing: + # a prefill continuing a resumed state: the conv window starts from the cached one + window = torch.cat([past_key_value.conv_state, mixed_qkv], dim=-1) + past_key_value.conv_state.copy_(window[:, :, -past_key_value.conv_state.shape[-1]:]) + with comfy.ops.CastBiasWeightContext(self.conv1d, window, offloadable=True) as (conv_weight, conv_bias): + mixed_qkv = F.silu(F.conv1d(window, conv_weight, conv_bias, groups=self.conv1d.groups)) + else: + if past_key_value is not None: + conv_state = past_key_value.conv_state + conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0)) + conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:]) + mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len]) # Split QKV and compute beta/g mixed_qkv = mixed_qkv.transpose(1, 2) # [B, seq_len, conv_dim] @@ -434,7 +442,7 @@ def forward(self, x, past_key_value=None, **kwargs): # Delta rule core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule( query, key, value, g=g, beta=beta, - initial_state=None, + initial_state=past_key_value.recurrent_state if continuing else None, output_final_state=past_key_value is not None, ) @@ -529,6 +537,8 @@ def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=No if decode and (isinstance(past_key_value, Int8FixedKVCache) or self.comfy_attention.method is None and not comfy.model_management.comfy_kitchen_attention_enabled()): output = past_key_value.attend(xq, xk, xv) + elif not decode and isinstance(past_key_value, Int8FixedKVCache) and past_key_value.index > 0: + output = past_key_value.extend(xq, xk, xv) # a prefill continuing a resumed state else: if decode: xk, xv = past_key_value.key, past_key_value.value @@ -900,10 +910,10 @@ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermed return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids, past_key_values=past_key_values) def prefill(self, embeds, past_key_values, position_ids=None, **kwargs): - if past_key_values[0].index == 0: + if past_key_values[0].index == 0 or self.batched_continue(embeds.shape[1], past_key_values): return super().prefill(embeds, past_key_values, position_ids, **kwargs) - # Resumed DeltaNet must preserve its recurrent and convolution state, so the new - # tokens go through the deferred decode kernels in rows of 1, 4 or 8. A multi-row + # Otherwise the new tokens go through the deferred decode kernels in rows of 1, 4 or 8, + # which also commit any rows the saved state left uncommitted. A multi-row # chunk is a verify tree that is one chain: each row attends and commits after the # row before it. kwargs.pop("decode_buffers", None) @@ -924,6 +934,29 @@ def prefill(self, embeds, past_key_values, position_ids=None, **kwargs): tracker["tree"] = None return result + @staticmethod + def batched_continue(num_tokens, past_key_values): + # A resumed state continues as one batched prefill when no DeltaNet rows wait for a commit; + # up to SPEC_ROWS tokens would take the decode path inside the layers. + return num_tokens > SPEC_ROWS and all(kv.uncommitted == 0 for kv in past_key_values if isinstance(kv, LinearKV)) + + def save_generation_state(self, state, cache, ids, generator, capacity, position): + # The last step's accepted tokens sit in the attention caches but not yet in the DeltaNet state + # (deferred commit). Save the state before them and re-feed them with the pending token, so the + # next turn can continue as one batched prefill. + pkv = cache.caches + replay = next((kv.uncommitted for kv in pkv if isinstance(kv, LinearKV)), 0) + if 0 < replay < len(ids): + for kv in pkv: + kv.index -= replay + if isinstance(kv, LinearKV): + kv.uncommitted = 0 + kv.ctl_tracker["path"] = None + super().save_generation_state(state, cache, ids, generator, capacity, position - replay) + state["pending"] = ids[-1 - replay:] + return + super().save_generation_state(state, cache, ids, generator, capacity, position) + def cache_capacity(self, needed, state): # a saved state fixes its capacity when first created, rounded up so later turns fit capacity = state.get("capacity", (needed + 4095) // 4096 * 4096) diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py index 5eeb2f47c23..748cb73bf8b 100644 --- a/tests-unit/comfy_test/qwen35_decode_test.py +++ b/tests-unit/comfy_test/qwen35_decode_test.py @@ -433,20 +433,153 @@ def test_continuation_from_model_state_matches_uninterrupted_generation(monkeypa path = str(tmp_path / f"{cutoff}.safetensors") state = comfy.model_state.ModelState.open(path) first = model.generate(embeds.clone(), do_sample=False, max_length=cutoff, mtp=mtp, model_state=state) - assert state.metadata["length"] == embeds.shape[1] + cutoff - 1 - assert state.metadata["pending"] == first[-1:] + # pending is the last sampled token, after the last step's tokens the DeltaNet state hadn't committed + pending_ids = state.metadata["pending"] + assert pending_ids == first[-len(pending_ids):] + assert state.metadata["length"] == embeds.shape[1] + cutoff - len(pending_ids) state.flush() state.save(path) # The next turn reopens the saved file, as a later prompt would. saved = comfy.model_state.ModelState.open(path, create="never") pending = model.model.embed_tokens(torch.tensor([saved.metadata["pending"]])) + position = saved.metadata["position"] rest = model.generate(pending, do_sample=False, max_length=20 - cutoff, mtp=mtp, model_state=saved, - position_ids=torch.tensor([[saved.metadata["position"]]])) + position_ids=torch.arange(position, position + len(pending_ids)).unsqueeze(0)) saved.release() assert first + rest == expected +def test_batched_continue_matches_one_prefill(monkeypatch): + # a prefill continuing a cached prefix (DeltaNet conv window and recurrent state, attention prefix) + # gives the same hidden states as prefilling the whole sequence at once + model = _tiny_model(monkeypatch) + embeds = torch.randn(1, 30, 64) + + def prefill(*parts): + state = comfy.model_state.ModelState() + comfy.model_management.register_ephemeral_model(state) + cpu = torch.device("cpu") + cache = state.load(model.init_kv_cache(1, 32, cpu, torch.float32, mtp=True), cpu) + out = [model.prefill(embeds[:, part].clone(), cache.caches)[0] for part in parts] # the forward reuses its input buffer + state.release() + return out + + whole, = prefill(slice(0, 30)) + _, rest = prefill(slice(0, 11), slice(11, 30)) + assert torch.allclose(rest, whole[:, 11:], atol=1e-4) + +@pytest.mark.parametrize("mtp", [False, "3", "tree2s2"]) +def test_long_continuation_prefills_in_one_batch(monkeypatch, tmp_path, mtp): + # a continued turn with more than SPEC_ROWS new tokens prefills as one batch, from the saved + # DeltaNet conv/recurrent state and the attention prefix, and matches reading the whole prompt at once + import comfy.text_encoders.qwen35 as qwen35 + model = _tiny_model(monkeypatch) + batched = [] + check = qwen35.Qwen35.batched_continue + monkeypatch.setattr(qwen35.Qwen35, "batched_continue", staticmethod(lambda n, kvs: batched.append(check(n, kvs)) or batched[-1])) + turn1, turn2 = torch.randn(1, 5, 64), torch.randn(1, 13, 64) + path = str(tmp_path / "state.safetensors") + state = comfy.model_state.ModelState.open(path) + first = model.generate(turn1.clone(), do_sample=False, max_length=6, mtp=mtp, model_state=state) + state.flush() + state.save(path) + + saved = comfy.model_state.ModelState.open(path, create="never") + pending_ids, position = saved.metadata["pending"], saved.metadata["position"] + embeds = torch.cat([model.model.embed_tokens(torch.tensor([pending_ids])), turn2], dim=1) + batched.clear() + rest = model.generate(embeds, do_sample=False, max_length=12, mtp=mtp, model_state=saved, + position_ids=torch.arange(position, position + embeds.shape[1]).unsqueeze(0)) + saved.release() + assert batched[0] is True + + whole = torch.cat([turn1, model.model.embed_tokens(torch.tensor([first])), turn2], dim=1) + assert rest == model.generate(whole, do_sample=False, max_length=12, mtp=False) + + + +class _ExactPacked: + # Int8DecodeCache stand-in that pages exact rows, 4 per page, so a continued int8 prefill can be checked + # against bf16. A refresh reads only what the kernel may: the last two pages, from the circular window. + dim = 64 + + def __init__(self, batch, heads, capacity, device): + shape = (batch, (capacity + 3) // 4, heads, 4, self.dim) + self.key, self.value = torch.zeros(shape, device=device), torch.zeros(shape, device=device) + self.key_scale, self.value_scale = torch.zeros(1, device=device), torch.zeros(1, device=device) + + def update(self, key, value, length, initialize=False): + n = int(length.item()) + first = 0 if initialize else max(0, (n - 1) // 4 * 4 - 4) + for i in range(first, n): + src = i if initialize else i % key.shape[2] + self.key[:, i // 4, :, i % 4] = key[:, :, src] + self.value[:, i // 4, :, i % 4] = value[:, :, src] + + def attend(self, query, length): + n = int(length.item()) + batch, heads, seq, dim = query.shape + k, v = (t.permute(0, 2, 1, 3, 4).reshape(batch, t.shape[2], -1, dim)[:, :, :n] for t in (self.key, self.value)) + k, v = (t.repeat_interleave(heads // t.shape[1], dim=1) for t in (k, v)) + scores = torch.matmul(query.float(), k.float().transpose(-1, -2)) * dim ** -0.5 + out = torch.matmul(scores.softmax(-1), v.float()) + return out.transpose(1, 2).reshape(batch, seq, heads * dim).to(query.dtype), scores.logsumexp(-1) + + +def test_int8_extend_matches_full_attention(monkeypatch): + # a continued prefill over an int8 prefix: each new row sees the prefix and the new rows up to itself, + # and the pages afterwards hold every row, across page and window boundaries + import comfy.text_encoders.llama as llama + monkeypatch.setattr(llama, "Int8DecodeCache", _ExactPacked, raising=False) + kv = llama.Int8FixedKVCache.zeros(1, 2, 64, 64, "cpu", torch.float32, llama.FixedKVCache.shared(1, "cpu")) + q, k, v = torch.randn(1, 4, 30, 64), torch.randn(1, 2, 30, 64), torch.randn(1, 2, 30, 64) + + def full(rows, length): + scores = torch.matmul(q[:, :, rows], k.repeat_interleave(2, dim=1)[:, :, :length].transpose(-1, -2)) * 64 ** -0.5 + scores.masked_fill_(torch.arange(length) > torch.arange(length)[rows].view(-1, 1), float("-inf")) + return torch.matmul(scores.softmax(-1), v.repeat_interleave(2, dim=1)[:, :, :length]).transpose(1, 2).reshape(1, -1, 256) + + kv.prepare(11) + kv.append(k[:, :, :11], v[:, :, :11]) + kv.advance(11) + kv.prepare(19) + assert torch.allclose(kv.extend(q[:, :, 11:], k[:, :, 11:], v[:, :, 11:]), full(slice(11, 30), 30), atol=1e-5) + out, _ = kv.packed.attend(q[:, :, 29:], torch.tensor([30])) + assert torch.allclose(out, full(slice(29, 30), 30), atol=1e-5) + +@pytest.mark.parametrize("mtp", [False, "3", "tree2s2"]) +def test_int8_long_continuation_matches_bf16(monkeypatch, tmp_path, mtp): + # the int8 cache continues a long turn through extend(): new rows attend the paged prefix and each + # other, then the pages grow through the window; with exact pages it must match the bf16 cache + import comfy.text_encoders.llama as llama + import comfy.text_encoders.qwen35 as qwen35 + model = _tiny_model(monkeypatch) + turn1, turn2 = torch.randn(1, 5, 64), torch.randn(1, 13, 64) + + def two_turns(name): + path = str(tmp_path / f"{name}.safetensors") + state = comfy.model_state.ModelState.open(path) + first = model.generate(turn1.clone(), do_sample=False, max_length=6, mtp=mtp, model_state=state) + state.flush() + state.save(path) + saved = comfy.model_state.ModelState.open(path, create="never") + pending_ids, position = saved.metadata["pending"], saved.metadata["position"] + embeds = torch.cat([model.model.embed_tokens(torch.tensor([pending_ids])), turn2], dim=1) + rest = model.generate(embeds, do_sample=False, max_length=12, mtp=mtp, model_state=saved, + position_ids=torch.arange(position, position + embeds.shape[1]).unsqueeze(0)) + saved.release() + return first, rest + + expected = two_turns("bf16") + monkeypatch.setattr(llama, "Int8DecodeCache", _ExactPacked, raising=False) + monkeypatch.setattr(qwen35.GatedAttention, "kv_cache_type", lambda self, device, dtype: qwen35.Int8FixedKVCache) + extended = [] + extend = llama.Int8FixedKVCache.extend + monkeypatch.setattr(llama.Int8FixedKVCache, "extend", lambda self, q, k, v: extended.append(k.shape[2]) or extend(self, q, k, v)) + assert two_turns("int8") == expected + assert extended and min(extended) > 8 + class _FakePacked: # Int8DecodeCache boundary with 4-row pages: records the BF16 refreshes it is asked for def __init__(self, batch, heads, capacity, device): From 7885cebad5598d74c7c236f3eb5b241bd390cf3a Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 3 Nov 2026 23:39:14 -0800 Subject: [PATCH 15/24] Add a prompt state cache for Qwen3.5 TextGenerate Agent-style chat sends a long shared prompt with a short new part each turn. With prefix_cache on (or COMFY_LLM_PREFIX_CACHE=on), TextGenerate keeps one model state per conversation, takes the state with the longest matching token prefix, resumes from the DeltaNet checkpoint at the last message boundary inside it, and prefills only the rest in batched segments. Image positions are keyed by a hash of the image, so image prompts match too. States stay loaded between calls. Past a size or count budget the least recently used one is saved to a file and loaded again on its next match. The shared start (up to the end of the system message) is saved once per system prompt, and a new conversation starts from that file. The mode applies per call; env follows COMFY_LLM_PREFIX_CACHE (off by default). Attention uses the bf16 KV cache while the cache is on, since the int8 cache can't continue a prefill from a prefix. --- comfy/text_encoders/llm_state_cache.py | 484 +++++++++++++++++++++++++ comfy/text_encoders/qwen35.py | 32 +- comfy_extras/nodes_textgen.py | 12 +- 3 files changed, 520 insertions(+), 8 deletions(-) create mode 100644 comfy/text_encoders/llm_state_cache.py diff --git a/comfy/text_encoders/llm_state_cache.py b/comfy/text_encoders/llm_state_cache.py new file mode 100644 index 00000000000..4324abcf6be --- /dev/null +++ b/comfy/text_encoders/llm_state_cache.py @@ -0,0 +1,484 @@ +"""Reuse a hybrid (attention + DeltaNet) LLM's prompt state across TextGenerate calls, kept in model +states (comfy.model_state) so the KV lives in dynamic VRAM, spills to RAM when another model needs the card, and +streams back per layer. + +The cache keeps a few resident states, one per conversation. A new prompt takes the state with the longest common +token prefix, resumes from the latest DeltaNet checkpoint inside it (a recurrent state can't be truncated; the +attention KV is truncated by moving its index back), and prefills the rest in batched segments that end on message +boundaries, where it takes the next checkpoints. Image positions get ids from a hash of the image, so image prompts +match too. + +Memory: states stay loaded between calls (nothing is written per call). Past COMFY_LLM_STATE_CACHE_GB of states, or +COMFY_LLM_STATE_CACHE_ENTRIES of them, the least recently used is saved to a file in COMFY_LLM_STATE_CACHE_DIR and +dropped; its next match loads the file. The shared start (the prompt up to the end of the system message) is saved +once to a file named by the hash of its tokens, if no such file exists, and a new conversation with no resident match +starts from it. + +COMFY_LLM_PREFIX_CACHE: off (default) | on. +Attention uses the bf16 FixedKVCache while the cache is on: Int8FixedKVCache can't continue a prefill from a prefix. +""" +import contextlib +import hashlib +import json +import logging +import os +import time + +import safetensors +import safetensors.torch +import torch + +MODE = os.environ.get("COMFY_LLM_PREFIX_CACHE", "off") +CACHE_DIR = os.environ.get("COMFY_LLM_STATE_CACHE_DIR", "/tmp/llm_state_cache") +BUDGET = int(float(os.environ.get("COMFY_LLM_STATE_CACHE_GB", "16")) * 2**30) +MAX_ENTRIES = int(os.environ.get("COMFY_LLM_STATE_CACHE_ENTRIES", "4")) +HEADROOM = int(os.environ.get("COMFY_LLM_PREFIX_CACHE_HEADROOM", "8192")) # capacity beyond this call, for later turns +CKPT_RECENT = int(os.environ.get("COMFY_LLM_PREFIX_CACHE_CKPTS", "6")) # message-boundary checkpoints kept +MSG_END = (248046, 198) # "<|im_end|>\n" in the Qwen3.5 vocab: a checkpoint may sit after it +MIN_SUFFIX = 9 # suffixes of llama.SPEC_ROWS (8) or fewer tokens would take the decode/verify paths +ROUND = 4096 + +stats = {} +_entries = [] # resident and spilled states, most recently used last +_tick = [0] + + +_call = [None] # a per-call mode set by the TextGenerate node, else MODE + + +def enabled(): + return (_call[0] or MODE) == "on" + + +@contextlib.contextmanager +def call_mode(mode): + _call[0] = mode + try: + yield + finally: + _call[0] = None + + +class _Entry: + """One conversation's state: a loaded ModelState (or a file it was spilled to), the tokens its KV holds, and DeltaNet + checkpoints {position: {layer: (conv, recurrent)}} on the CPU.""" + + def __init__(self, key, state, capacity): + self.key = key + self.state = state # comfy.model_state.ModelState, loaded; None once spilled + self.capacity = capacity + self.ids = [] + self.checkpoints = {} + self.file = None + self.used = 0 + + def nbytes(self): + n = sum(c.nbytes + r.nbytes for st in self.checkpoints.values() for c, r in st.values()) + if self.state is not None and self.state.patcher is not None: + n += sum(t.nbytes for t in self.state.patcher.model.state_dict().values()) + return n + + +def _model_key(model, dtype): + return f"{getattr(model, 'model_type', type(model).__name__)}|{dtype}" + + +def _lcp(a, b): + n = 0 + for x, y in zip(a, b): + if x != y: + break + n += 1 + return n + + +def _best(key, ids): + """(entry, checkpoint position) with the most reusable tokens, or (None, 0).""" + best, best_pos = None, 0 + limit_all = len(ids) - MIN_SUFFIX + for e in _entries: + if e.key != key: + continue + lcp = _lcp(ids, e.ids) + usable = [p for p in e.checkpoints if p <= min(lcp, limit_all)] + if usable and max(usable) > best_pos: + best, best_pos = e, max(usable) + return best, best_pos + + +@contextlib.contextmanager +def _cast(model, kvs, device): + """Device views of these caches' tensors outside a forward (their storage streams through a prefetch queue).""" + import comfy.model_prefetch + kvs = [kv for kv in kvs if kv.storage is not None] + if not kvs: + yield + return + storages = tuple(kv.storage for kv in kvs) + queue = comfy.model_prefetch.make_prefetch_queue([storages], device, {"prefetch_dynamic_vbars": model.model.prefetch_dynamic_vbars}) + with contextlib.ExitStack() as casts: + comfy.model_prefetch.prefetch_queue_pop(queue, device, storages, core=lambda: [casts.enter_context(kv.cast(device)) for kv in kvs]) + yield + comfy.model_prefetch.prefetch_queue_pop(queue, device, None) + + +def _linear(cache): + from .qwen35 import LinearKV + return {i: kv for i, kv in enumerate(cache.caches) if isinstance(kv, LinearKV)} + + +def _snapshot(model, cache, device): + lin = _linear(cache) + with _cast(model, lin.values(), device): + out = {i: (kv.conv_state.to("cpu", copy=True), kv.recurrent_state.to("cpu", copy=True)) for i, kv in lin.items()} + return out + + +def _restore_linear(model, cache, states, device): + lin = _linear(cache) + with _cast(model, lin.values(), device): + for i, (c, r) in states.items(): + lin[i].conv_state.copy_(c) + lin[i].recurrent_state.copy_(r) + + +def _set_index(cache, n): + """Every layer at index n with no step in flight: slots from n on are masked by the next prepare().""" + from .llama import FixedKVState + from .qwen35 import LinearKV + for kv in cache.caches: + kv.index = n + if isinstance(kv, FixedKVState): + kv.tracker["step"] = -1 + kv.tracker["tree"] = None + elif isinstance(kv, LinearKV): + kv.uncommitted = 0 + kv.ctl_tracker.update(step=None, path=None, tree=None) + for kv in cache.extra_caches: # the MTP layer's cache starts empty on every call, as on a fresh generate + kv.index = 0 + if isinstance(kv, FixedKVState): + kv.tracker["step"] = -1 + kv.tracker["tree"] = None + + +def _capacity(need): + return (need + HEADROOM + ROUND - 1) // ROUND * ROUND + + +def _padded(tensors, model_sd): + """CPU tensors shaped like model_sd: equal shapes pass through, attention caches (capacity on dim 2) are copied into + zeros of the new capacity, cut to it if longer.""" + out = {} + for name, like in model_sd.items(): + t = tensors.get(name) + if t is None: + out[name] = torch.zeros(like.shape, dtype=like.dtype) + elif tuple(t.shape) == tuple(like.shape): + out[name] = t + else: + z = torch.zeros(like.shape, dtype=like.dtype) + n = min(t.shape[2], like.shape[2]) + z[:, :, :n] = t[:, :, :n] + out[name] = z + return out + + +def _load(model, capacity, device, dtype, tensors=None): + """A loaded ModelState holding a fresh KV cache model of this capacity, filled from tensors (CPU, any capacity).""" + import comfy.model_state + kvm = model.init_kv_cache(1, capacity, device, dtype, mtp=True) + state = comfy.model_state.ModelState() + if tensors: + state.tensors = _padded(tensors, kvm.state_dict()) + state.load(kvm, device) + state.patcher.set_in_use_by_current_prompt(True) + return state, kvm + + +def _activate(entry, device): + import comfy.model_management + comfy.model_management.load_models_gpu([entry.state.patcher]) + entry.state.patcher.set_in_use_by_current_prompt(True) + return entry.state.patcher.model + + +def _attention_names(kvm): + """State-dict names of the attention caches (capacity on dim 2), the MTP layer's included.""" + from .qwen35 import LinearKV + out = set() + for i, kv in enumerate(kvm.caches + kvm.extra_caches): + if not isinstance(kv, LinearKV): + out.update((f"layers.{i}.key.weight", f"layers.{i}.value.weight")) + return out + + +def _trimmed_tensors(state_tensors, n, attention): + """A state's CPU tensors with the attention caches cut to the n tokens used (the rest is masked anyway).""" + return {name: t[:, :, :n].contiguous() if name in attention else t for name, t in state_tensors.items()} + + +def _file_tensors(entry_tensors, checkpoints): + t = dict(entry_tensors) + for pos, st in checkpoints.items(): + for i, (c, r) in st.items(): + t[f"ckpt.{pos}.{i}.conv"] = c.contiguous() + t[f"ckpt.{pos}.{i}.rec"] = r.contiguous() + return t + + +def _write(path, tensors, meta): + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + f".tmp{os.getpid()}" + safetensors.torch.save_file(tensors, tmp, {"llm_state_cache": json.dumps(meta)}) + os.replace(tmp, path) + + +def _read(path): + with safetensors.safe_open(path, "pt") as f: + meta = json.loads(f.metadata()["llm_state_cache"]) + tensors, cps = {}, {} + for name in f.keys(): + if name.startswith("ckpt."): + _, pos, i, kind = name.split(".") + cps.setdefault(int(pos), {}).setdefault(int(i), [None, None])[0 if kind == "conv" else 1] = f.get_tensor(name) + else: + tensors[name] = f.get_tensor(name) + return tensors, {p: {i: tuple(v) for i, v in st.items()} for p, st in cps.items()}, meta + + +def _ids_json(ids): + return [list(t) if isinstance(t, tuple) else t for t in ids] + + +def _prefix_path(key, ids): + h = hashlib.blake2b(json.dumps([key, _ids_json(ids)]).encode(), digest_size=16).hexdigest() + return os.path.join(CACHE_DIR, f"prefix-{h}.safetensors") + + +def _spill(entry): + """Save a resident entry to a file and free its state; its next match loads the file.""" + t = time.perf_counter() + state = entry.state + attention = _attention_names(state.patcher.model) + state.flush() + n = len(entry.ids) + path = os.path.join(CACHE_DIR, f"spill-{os.getpid()}-{id(entry)}.safetensors") + _write(path, _file_tensors(_trimmed_tensors(state.tensors, n, attention), entry.checkpoints), + {"key": entry.key, "ids": _ids_json(entry.ids)}) + entry.state = None + entry.file = path + entry.checkpoints = {p: {} for p in entry.checkpoints} # positions kept for matching; the states are in the file + logging.info("llm state cache: spilled %d tokens to disk in %.2f s", n, time.perf_counter() - t) + + +def _drop(entry): + if entry.state is not None: + entry.state.release() + if entry.file is not None: + try: + os.remove(entry.file) + except OSError: + pass + _entries.remove(entry) + + +def _enforce_budget(keep): + resident = [e for e in _entries if e.state is not None] + resident.sort(key=lambda e: e.used) + total = sum(e.nbytes() for e in resident) + for e in resident: + if e is keep: + continue + if total <= BUDGET and len(resident) <= MAX_ENTRIES: + break + total -= e.nbytes() + _spill(e) + resident = [r for r in resident if r is not e] + spilled = [e for e in _entries if e.state is None] + spilled.sort(key=lambda e: e.used) + while len(spilled) > MAX_ENTRIES: # spilled files are bounded too + _drop(spilled.pop(0)) + + +def _take(model, key, ids, need, device, dtype): + """(entry, cache model, prefix length) for this prompt, allocating or loading as needed.""" + entry, n = _best(key, ids) + start = _shared_start(ids) + path = _prefix_path(key, ids[:start]) if start else None + if entry is not None and n <= start < len(entry.ids) and path is not None and os.path.exists(path): + entry, n = None, 0 # another conversation: fork from the shared-start file, keep its state whole + stats["fork"] = True + if entry is not None: + _entries.remove(entry) + if entry.state is None: + tensors, cps, _ = _read(entry.file) + os.remove(entry.file) + entry.file = None + cap = max(_capacity(need), entry.capacity) + entry.state, kvm = _load(model, cap, device, dtype, tensors) + entry.capacity = cap + entry.checkpoints = {p: cps[p] for p in cps if p in entry.checkpoints} + stats["restore"] = "from-disk" + elif entry.capacity < need: + attention = _attention_names(entry.state.patcher.model) + entry.state.flush() + cap = _capacity(need) + tensors = _trimmed_tensors(entry.state.tensors, n, attention) + entry.state, kvm = _load(model, cap, device, dtype, tensors) + entry.capacity = cap + stats["restore"] = "grow" + else: + kvm = _activate(entry, device) + stats["restore"] = "resident" + _restore_linear(model, kvm, entry.checkpoints[n], device) + _set_index(kvm, n) + entry.checkpoints = {p: st for p, st in entry.checkpoints.items() if p <= n} + entry.ids = list(ids[:n]) + return entry, kvm, n + cap = _capacity(need) + if path is not None and os.path.exists(path): + tensors, cps, _ = _read(path) + state, kvm = _load(model, cap, device, dtype, tensors) + entry = _Entry(key, state, cap) + entry.checkpoints = {start: cps[start]} + _restore_linear(model, kvm, cps[start], device) + _set_index(kvm, start) + entry.ids = list(ids[:start]) + stats["restore"] = "shared-start-file" + return entry, kvm, start + state, kvm = _load(model, cap, device, dtype) + return _Entry(key, state, cap), kvm, 0 + + +def _boundaries(ids): + return [j for j in range(2, len(ids) + 1) if ids[j - 2] == MSG_END[0] and ids[j - 1] == MSG_END[1]] + + +def _shared_start(ids): + """The end of the system message (the first message boundary), where a conversation's shared start ends.""" + ends = _boundaries(ids) + if not ends or ends[0] < MIN_SUFFIX or len(ids) - ends[0] < MIN_SUFFIX: + return 0 + return ends[0] + + +def _checkpoint_positions(ids, start): + ends = _boundaries(ids) + want = sorted(set(ends[:1] + ends[-CKPT_RECENT:])) if CKPT_RECENT > 0 else [] + out, prev = [], start + for pos in want: + if pos - prev >= MIN_SUFFIX and len(ids) - pos >= MIN_SUFFIX: + out.append(pos) + prev = pos + return out + + +def prompt_ids(tokens, embeds_info, seq_len): + """Cache ids for one prompt, one per embed position: the token ids, and for each image span a key per position + made from a hash of the image. None when the prompt holds anything else (video, audio, raw embeddings) or the + spans don't line up.""" + spans = iter(embeds_info) + out = [] + for t in tokens: + if isinstance(t, int): + out.append(t) + continue + if not isinstance(t, dict) or t.get("type") != "image" or not torch.is_tensor(t.get("data")): + return None + info = next(spans, None) + if info is None or info.get("type") != "image" or info.get("index") != len(out): + return None + digest = _image_digest(t["data"]) + out.extend((digest, k) for k in range(info["size"])) + if next(spans, None) is not None or len(out) != seq_len: + return None + return out + + +def _image_digest(image): + data = image.detach().to("cpu").contiguous() + h = hashlib.blake2b(digest_size=16) + h.update(f"{tuple(data.shape)}|{data.dtype}".encode()) + h.update(data.view(torch.uint8).numpy().tobytes() if data.numel() else b"") + return "img:" + h.hexdigest() + + +_current = {} + + +def prefill(model, embeds, ids, need, forward, position_ids=None): + """Prefill the whole prompt `embeds`, reusing a cached prefix of `ids`. Returns (cache model, x of the last position). + `forward(embeds, caches, position_ids)` runs one segment.""" + if _current: + abort() # the previous generate failed after its prefill + stats.clear() + device, dtype = embeds.device, embeds.dtype + key = _model_key(model, dtype) + entry, kvm, n = _take(model, key, ids, need, device, dtype) + stats.update(prompt_len=len(ids), prefix_len=n) + start, x = n, None + shared = _shared_start(ids) + for pos in _checkpoint_positions(ids, n) + [len(ids)]: + x = forward(embeds[:, start:pos], kvm.caches, None if position_ids is None else position_ids[..., start:pos]) + entry.checkpoints[pos] = _snapshot(model, kvm, device) + if pos == shared and n < shared: + _save_shared_start(model, kvm, key, ids, shared, entry.checkpoints[pos], device) + start = pos + if len(entry.checkpoints) > CKPT_RECENT + 3: + keep = sorted(entry.checkpoints) + keep = keep[:1] + keep[-(CKPT_RECENT + 2):] + entry.checkpoints = {p: entry.checkpoints[p] for p in keep} + entry.ids = list(ids) + _current.update(entry=entry, kvm=kvm, device=device, model=model) + return kvm, x + + +def _save_shared_start(model, kvm, key, ids, pos, linear_states, device): + """Write the shared start (KV up to pos + the DeltaNet states at pos) once, if no file has it.""" + path = _prefix_path(key, ids[:pos]) + if os.path.exists(path): + return + linear = _linear(kvm) + attn = {i: kv for i, kv in enumerate(kvm.caches) if i not in linear} + tensors = {} + with _cast(model, attn.values(), device): + for i, kv in attn.items(): + tensors[f"layers.{i}.key.weight"] = kv.key[:, :, :pos].to("cpu", copy=True) + tensors[f"layers.{i}.value.weight"] = kv.value[:, :, :pos].to("cpu", copy=True) + _write(path, _file_tensors(tensors, {pos: linear_states}), {"key": key, "ids": _ids_json(ids[:pos])}) + stats["shared_start_saved"] = True + + +def finish(ids, generated): + """Checkpoint the prompt plus the generated tokens the DeltaNet state has committed, keep the state resident, and + spill others past the budget.""" + entry, kvm, device, model = (_current.get(k) for k in ("entry", "kvm", "device", "model")) + _current.clear() + if entry is None: + return + stats["gen_tokens"] = len(generated) + full = list(ids) + list(generated) + lin = next(iter(_linear(kvm).values())) + # deferred commit: a decode step leaves the DeltaNet state untouched until the next step, so the state lags the + # index by the last step's still-uncommitted tokens + idx = lin.index - lin.uncommitted + if len(ids) < idx <= len(full): + entry.checkpoints[idx] = _snapshot(model, kvm, device) + entry.ids = full[:idx] + entry.state.patcher.set_in_use_by_current_prompt(False) + _tick[0] += 1 + entry.used = _tick[0] + _entries.append(entry) + _enforce_budget(entry) + stats["entries"] = len(_entries) + stats["resident"] = sum(1 for e in _entries if e.state is not None) + stats["cache_gb"] = round(sum(e.nbytes() for e in _entries if e.state is not None) / 1e9, 3) + logging.info("LLM_PREFIX_CACHE %s", json.dumps(stats)) + + +def abort(): + """A generate failed after prefill: drop the half-written state.""" + entry = _current.get("entry") + _current.clear() + if entry is not None and entry.state is not None: + entry.state.release() diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index db5a5e0e246..ead311d3631 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -16,6 +16,7 @@ from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_for_device from comfy import sd1_clip import comfy.text_encoders.qwen_vl +from . import llm_state_cache from .llama import SPEC_ROWS, BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, KVCacheModel, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix @@ -496,6 +497,8 @@ def __init__(self, config, device=None, dtype=None, ops=None): self.comfy_attention = ComfyAttention() def kv_cache_type(self, device, dtype): + if llm_state_cache.enabled(): + return FixedKVCache # the prompt state cache continues prefills from a cached prefix; int8 can't method = self.comfy_attention.method if method == "comfy_kitchen_int8" or method is None and comfy.model_management.comfy_kitchen_attention_enabled(): if dtype == torch.bfloat16 and self.head_dim == 256 and int8_decode_is_available(device): @@ -977,9 +980,9 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, "presence_penalty": kwargs.get("presence_penalty", 0.0) or 0.0, "seed": seed if seed is not None else 42} return self._generate_mtp(embeds, max_length, stop_tokens, mtp, sampling=sampling, position_ids=kwargs.get("position_ids"), - model_state=kwargs.get("model_state")) + model_state=kwargs.get("model_state"), cache_ids=kwargs.get("cache_ids")) - def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, position_ids=None, model_state=None): + def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, position_ids=None, model_state=None, cache_ids=None): device = embeds.device cfg = self.model.config if stop_tokens is None: @@ -992,8 +995,15 @@ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, pos past_length = state.get("length", 0) cap = self.cache_capacity(past_length + embeds.shape[1] + max_length + 7, state) generator = torch.Generator(device=device).manual_seed(sampling["seed"]) if sampling is not None else None - cache = model_state.load(self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True), device) - self.restore_generation_state(state, cache, generator) + if cache_ids is not None: + # the prompt state cache: a resident state with the longest matching prefix, prefilled to the end of the prompt + cache, x = llm_state_cache.prefill(self, embeds, cache_ids, cap, + lambda e, kv, p: self.model.forward(None, embeds=e, attention_mask=None, past_key_values=kv, position_ids=p)[0], + position_ids=position_ids) + cap = cache.caches[self.model.config.layer_types.index("full_attention")].key.shape[2] + else: + cache = model_state.load(self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True), device) + self.restore_generation_state(state, cache, generator) pkv = cache.caches # repair window: drafting ahead plus a near-full rollback mtp_kv = cache.extra_caches[0] @@ -1019,7 +1029,8 @@ def freqs_at(p, n=1): # cross-step state lives in static carriers: nothing allocated inside a step may outlive it nt_buf = torch.empty((embeds.shape[0], 1), device=device, dtype=torch.long) h_buf = torch.empty((embeds.shape[0], 1, cfg.hidden_size), device=device, dtype=dt) - x, _, _ = self.prefill(embeds, pkv, position_ids) + if cache_ids is None: + x, _, _ = self.prefill(embeds, pkv, position_ids) lg0 = self.logits(x)[:, -1] pen_mask = torch.zeros((lg0.shape[-1],), device=device, dtype=torch.bool) if penalized else None if penalized and "penalty_tokens" in state: @@ -1257,6 +1268,9 @@ def step(): # A resumed cache decodes as a chain until its next verify installs a tree. for kv in pkv: (kv.ctl_tracker if isinstance(kv, LinearKV) else kv.tracker)["tree"] = None + if cache_ids is not None: + llm_state_cache.finish(cache_ids, ids) + return ids self.save_generation_state(state, cache, ids, generator, cap, pkv[0].index + rope_offset) return ids @@ -1370,9 +1384,15 @@ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device) if resume: position_ids = torch.arange(state["position"], state["position"] + embeds.shape[1], device=embeds.device).unsqueeze(0) + extra = {} + if llm_state_cache.enabled() and not state and mtp and len(tokens_only) == 1: + # no explicit state: the prompt state cache finds a matching prefix itself (image positions are hashed) + cache_ids = llm_state_cache.prompt_ids(tokens_only[0], embeds_info, embeds.shape[1]) + if cache_ids is not None: + extra["cache_ids"] = cache_ids return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, presence_penalty=presence_penalty, position_ids=position_ids, mtp=mtp, - model_state=self.model_state) + model_state=self.model_state, **extra) class Qwen35TEModel(sd1_clip.SD1ClipModel): diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py index dba50f8375d..2000ed15b17 100644 --- a/comfy_extras/nodes_textgen.py +++ b/comfy_extras/nodes_textgen.py @@ -2,6 +2,8 @@ from typing_extensions import override import comfy.model_state +from comfy.text_encoders import llm_state_cache + class TextGenerate(io.ComfyNode): @classmethod @@ -43,6 +45,7 @@ def define_schema(cls): io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True), io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth and breadth to observed token matches and execution cost; 2-5 pins a constant draft depth; tree modes pin both breadth and depth. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."), io.String.Input("system_prompt", force_input=True, optional=True, tooltip="Replaces the system prompt in the model's chat template. Ignored when the default template is not used."), + io.Combo.Input("prefix_cache", options=["env", "off", "on"], default="env", optional=True, advanced=True, tooltip="Reuse the prompt state of an earlier call that shares this prompt's start (one kept per conversation). env follows COMFY_LLM_PREFIX_CACHE."), io.ModelState.Input("state", optional=True, tooltip="Continues the conversation held by this model state and extends it with this turn. The prompt should be a single text turn."), ], outputs=[ @@ -54,8 +57,13 @@ def define_schema(cls): ) @classmethod - def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="", state=None) -> io.NodeOutput: + def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="", state=None, prefix_cache="env") -> io.NodeOutput: + # the mode applies to this call only; env follows COMFY_LLM_PREFIX_CACHE + with llm_state_cache.call_mode(None if prefix_cache == "env" else prefix_cache): + return cls._generate(clip, prompt, max_length, sampling_mode, image, thinking, use_default_template, video, audio, mtp, system_prompt, state) + @classmethod + def _generate(cls, clip, prompt, max_length, sampling_mode, image, thinking, use_default_template, video, audio, mtp, system_prompt, state) -> io.NodeOutput: mtp = {"off": False, "auto": True}.get(mtp, mtp) # the system prompt opens a conversation; later turns continue it @@ -247,7 +255,7 @@ def define_schema(cls): node_id="TextGenerateLTX2Prompt", display_name="Generate LTX2 Prompt", category=parent_schema.category, - inputs=[i for i in parent_schema.inputs if i.id != "state"], + inputs=[i for i in parent_schema.inputs if i.id not in ("state", "prefix_cache")], outputs=parent_schema.outputs[:2], hidden=parent_schema.hidden, search_aliases=["prompt enhance", "LLM", "gemma"], From eeab416ec3e24d0304e0b02e6bf5cdc15973f3a3 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Tue, 3 Nov 2026 23:39:14 -0800 Subject: [PATCH 16/24] Accept older prefix_cache mode names and add LLMPrefixCacheStats vram, gpu, cpu and disk map to on and stats to off. LLMPrefixCacheStats returns the last generate's cache stats as JSON. --- comfy_extras/nodes_textgen.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py index 2000ed15b17..ed8d66699a9 100644 --- a/comfy_extras/nodes_textgen.py +++ b/comfy_extras/nodes_textgen.py @@ -1,9 +1,15 @@ from comfy_api.latest import ComfyExtension, io from typing_extensions import override +import json +import time + import comfy.model_state from comfy.text_encoders import llm_state_cache +# older clients send these mode names and read LLMPrefixCacheStats +LEGACY_CACHE_MODES = {"vram": "on", "gpu": "on", "cpu": "on", "disk": "on", "stats": "off"} + class TextGenerate(io.ComfyNode): @classmethod @@ -45,7 +51,7 @@ def define_schema(cls): io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True), io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth and breadth to observed token matches and execution cost; 2-5 pins a constant draft depth; tree modes pin both breadth and depth. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."), io.String.Input("system_prompt", force_input=True, optional=True, tooltip="Replaces the system prompt in the model's chat template. Ignored when the default template is not used."), - io.Combo.Input("prefix_cache", options=["env", "off", "on"], default="env", optional=True, advanced=True, tooltip="Reuse the prompt state of an earlier call that shares this prompt's start (one kept per conversation). env follows COMFY_LLM_PREFIX_CACHE."), + io.Combo.Input("prefix_cache", options=["env", "off", "on"] + list(LEGACY_CACHE_MODES), default="env", optional=True, advanced=True, tooltip="Reuse the prompt state of an earlier call that shares this prompt's start (one kept per conversation). env follows COMFY_LLM_PREFIX_CACHE. vram, gpu, disk and stats are older clients' names."), io.ModelState.Input("state", optional=True, tooltip="Continues the conversation held by this model state and extends it with this turn. The prompt should be a single text turn."), ], outputs=[ @@ -59,7 +65,7 @@ def define_schema(cls): @classmethod def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="", state=None, prefix_cache="env") -> io.NodeOutput: # the mode applies to this call only; env follows COMFY_LLM_PREFIX_CACHE - with llm_state_cache.call_mode(None if prefix_cache == "env" else prefix_cache): + with llm_state_cache.call_mode(None if prefix_cache == "env" else LEGACY_CACHE_MODES.get(prefix_cache, prefix_cache)): return cls._generate(clip, prompt, max_length, sampling_mode, image, thinking, use_default_template, video, audio, mtp, system_prompt, state) @classmethod @@ -297,12 +303,32 @@ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=F return io.NodeOutput(out.args[0] or prompt, out.args[1]) +class LLMPrefixCacheStats(io.ComfyNode): + """The last generate's prompt state cache line, as JSON (for older clients); wire the generated text in so it runs after.""" + @classmethod + def define_schema(cls): + return io.Schema(node_id="LLMPrefixCacheStats", category="text", + inputs=[io.String.Input("text", force_input=True)], + outputs=[io.String.Output(display_name="stats")]) + + @classmethod + def fingerprint_inputs(cls, **kwargs): + return time.time() + + @classmethod + def execute(cls, text) -> io.NodeOutput: + st = dict(llm_state_cache.stats) + st.setdefault("hit", st.get("prefix_len", 0) > 0) + return io.NodeOutput(json.dumps(st)) + + class TextgenExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[io.ComfyNode]]: return [ TextGenerate, TextGenerateLTX2Prompt, + LLMPrefixCacheStats, ] async def comfy_entrypoint() -> TextgenExtension: From 876878f4bcbb0d4330edc08b4172ebed98b5cd07 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 4 Nov 2026 14:28:53 -0800 Subject: [PATCH 17/24] Let the prompt state cache use the int8 KV cache Drop the forced bf16 KV cache: prefills now continue an int8 prefix. Cache keys include the KV cache type, the attention tensors are trimmed and padded along an int8 cache's page dimension too, and the shared-start file keeps its pages. An int8 state resumes only where its bf16 window still holds the two pages before the resume point. The rope capacity comes from the cache entry (an int8 cache's key tensor is its window). The in-use flag is only set on a dynamic-VRAM patcher, so the cache also runs on a CPU device. Add unit tests for the cache: a next turn, a spilled conversation and a new conversation from the shared-start file, each on bf16 and int8. --- comfy/text_encoders/llm_state_cache.py | 86 +++++++++---- comfy/text_encoders/qwen35.py | 9 +- tests-unit/comfy_test/llm_state_cache_test.py | 120 ++++++++++++++++++ tests-unit/comfy_test/qwen35_decode_test.py | 16 +-- 4 files changed, 193 insertions(+), 38 deletions(-) create mode 100644 tests-unit/comfy_test/llm_state_cache_test.py diff --git a/comfy/text_encoders/llm_state_cache.py b/comfy/text_encoders/llm_state_cache.py index 4324abcf6be..77ac79cb918 100644 --- a/comfy/text_encoders/llm_state_cache.py +++ b/comfy/text_encoders/llm_state_cache.py @@ -15,7 +15,7 @@ starts from it. COMFY_LLM_PREFIX_CACHE: off (default) | on. -Attention uses the bf16 FixedKVCache while the cache is on: Int8FixedKVCache can't continue a prefill from a prefix. +With the int8 KV cache, a state resumes only where its bf16 window still holds the two pages before the resume point. """ import contextlib import hashlib @@ -71,6 +71,8 @@ def __init__(self, key, state, capacity): self.checkpoints = {} self.file = None self.used = 0 + self.window = _int8_window(state.patcher.model) if state is not None else None # (rows, page) for an int8 cache + self.written = 0 # rows written past the committed ones (draft rows included) bound what the window still holds def nbytes(self): n = sum(c.nbytes + r.nbytes for st in self.checkpoints.values() for c, r in st.values()) @@ -79,8 +81,24 @@ def nbytes(self): return n -def _model_key(model, dtype): - return f"{getattr(model, 'model_type', type(model).__name__)}|{dtype}" +def _model_key(model, dtype, device): + from .qwen35 import GatedAttention + attn = next(m for m in model.modules() if isinstance(m, GatedAttention)) + return f"{getattr(model, 'model_type', type(model).__name__)}|{dtype}|{attn.kv_cache_type(device, dtype).__name__}" + + +def _int8_window(kvm): + from .llama import Int8FixedKVCache + kv = next((kv for kv in kvm.caches if isinstance(kv, Int8FixedKVCache)), None) + return None if kv is None else (kv.key.shape[2], kv.packed.key.shape[3]) + + +def _resumable(entry, pos): + """An int8 cache refreshes its pages from the bf16 window: the two pages before pos must still be in it.""" + if entry.window is None: + return True + rows, page = entry.window + return entry.written - rows <= max(0, (pos - 1) // page * page - page) def _lcp(a, b): @@ -100,7 +118,7 @@ def _best(key, ids): if e.key != key: continue lcp = _lcp(ids, e.ids) - usable = [p for p in e.checkpoints if p <= min(lcp, limit_all)] + usable = [p for p in e.checkpoints if p <= min(lcp, limit_all) and _resumable(e, p)] if usable and max(usable) > best_pos: best, best_pos = e, max(usable) return best, best_pos @@ -166,8 +184,8 @@ def _capacity(need): def _padded(tensors, model_sd): - """CPU tensors shaped like model_sd: equal shapes pass through, attention caches (capacity on dim 2) are copied into - zeros of the new capacity, cut to it if longer.""" + """CPU tensors shaped like model_sd: equal shapes pass through, attention caches (capacity on dim 2, int8 pages on + dim 1) are copied into zeros of the new capacity, cut to it if longer.""" out = {} for name, like in model_sd.items(): t = tensors.get(name) @@ -177,12 +195,18 @@ def _padded(tensors, model_sd): out[name] = t else: z = torch.zeros(like.shape, dtype=like.dtype) - n = min(t.shape[2], like.shape[2]) - z[:, :, :n] = t[:, :, :n] + common = tuple(slice(0, min(a, b)) for a, b in zip(t.shape, like.shape)) + z[common] = t[common] out[name] = z return out +def _in_use(patcher, flag): + # only a dynamic-VRAM patcher pins for the current prompt (none on a CPU device) + if patcher.is_dynamic(): + patcher.set_in_use_by_current_prompt(flag) + + def _load(model, capacity, device, dtype, tensors=None): """A loaded ModelState holding a fresh KV cache model of this capacity, filled from tensors (CPU, any capacity).""" import comfy.model_state @@ -191,30 +215,38 @@ def _load(model, capacity, device, dtype, tensors=None): if tensors: state.tensors = _padded(tensors, kvm.state_dict()) state.load(kvm, device) - state.patcher.set_in_use_by_current_prompt(True) + _in_use(state.patcher, True) return state, kvm def _activate(entry, device): import comfy.model_management comfy.model_management.load_models_gpu([entry.state.patcher]) - entry.state.patcher.set_in_use_by_current_prompt(True) + _in_use(entry.state.patcher, True) return entry.state.patcher.model -def _attention_names(kvm): - """State-dict names of the attention caches (capacity on dim 2), the MTP layer's included.""" +def _attention_layout(kvm): + """{state-dict name: (dim, rows per slot)} of the attention caches, the MTP layer's included: capacity on dim 2 (an + int8 cache's bf16 window too: row r sits in slot r % window), an int8 cache's pages on dim 1.""" from .qwen35 import LinearKV - out = set() + out = {} for i, kv in enumerate(kvm.caches + kvm.extra_caches): - if not isinstance(kv, LinearKV): - out.update((f"layers.{i}.key.weight", f"layers.{i}.value.weight")) + if isinstance(kv, LinearKV): + continue + for key in kv.state_tensors(): + out[f"layers.{i}.{key}.weight"] = (1, kv.packed.key.shape[3]) if key.startswith("packed_") else (2, 1) return out +def _trimmed(t, n, layout): + dim, unit = layout + return t.narrow(dim, 0, min(t.shape[dim], -(-n // unit))).contiguous() + + def _trimmed_tensors(state_tensors, n, attention): """A state's CPU tensors with the attention caches cut to the n tokens used (the rest is masked anyway).""" - return {name: t[:, :, :n].contiguous() if name in attention else t for name, t in state_tensors.items()} + return {name: _trimmed(t, n, attention[name]) if name in attention else t for name, t in state_tensors.items()} def _file_tensors(entry_tensors, checkpoints): @@ -259,7 +291,7 @@ def _spill(entry): """Save a resident entry to a file and free its state; its next match loads the file.""" t = time.perf_counter() state = entry.state - attention = _attention_names(state.patcher.model) + attention = _attention_layout(state.patcher.model) state.flush() n = len(entry.ids) path = os.path.join(CACHE_DIR, f"spill-{os.getpid()}-{id(entry)}.safetensors") @@ -320,7 +352,7 @@ def _take(model, key, ids, need, device, dtype): entry.checkpoints = {p: cps[p] for p in cps if p in entry.checkpoints} stats["restore"] = "from-disk" elif entry.capacity < need: - attention = _attention_names(entry.state.patcher.model) + attention = _attention_layout(entry.state.patcher.model) entry.state.flush() cap = _capacity(need) tensors = _trimmed_tensors(entry.state.tensors, n, attention) @@ -340,6 +372,7 @@ def _take(model, key, ids, need, device, dtype): tensors, cps, _ = _read(path) state, kvm = _load(model, cap, device, dtype, tensors) entry = _Entry(key, state, cap) + entry.written = start entry.checkpoints = {start: cps[start]} _restore_linear(model, kvm, cps[start], device) _set_index(kvm, start) @@ -407,13 +440,13 @@ def _image_digest(image): def prefill(model, embeds, ids, need, forward, position_ids=None): - """Prefill the whole prompt `embeds`, reusing a cached prefix of `ids`. Returns (cache model, x of the last position). + """Prefill the whole prompt `embeds`, reusing a cached prefix of `ids`. Returns (cache model, x of the last position, capacity). `forward(embeds, caches, position_ids)` runs one segment.""" if _current: abort() # the previous generate failed after its prefill stats.clear() device, dtype = embeds.device, embeds.dtype - key = _model_key(model, dtype) + key = _model_key(model, dtype, device) entry, kvm, n = _take(model, key, ids, need, device, dtype) stats.update(prompt_len=len(ids), prefix_len=n) start, x = n, None @@ -429,8 +462,9 @@ def prefill(model, embeds, ids, need, forward, position_ids=None): keep = keep[:1] + keep[-(CKPT_RECENT + 2):] entry.checkpoints = {p: entry.checkpoints[p] for p in keep} entry.ids = list(ids) + entry.written = len(ids) _current.update(entry=entry, kvm=kvm, device=device, model=model) - return kvm, x + return kvm, x, entry.capacity def _save_shared_start(model, kvm, key, ids, pos, linear_states, device): @@ -440,11 +474,13 @@ def _save_shared_start(model, kvm, key, ids, pos, linear_states, device): return linear = _linear(kvm) attn = {i: kv for i, kv in enumerate(kvm.caches) if i not in linear} + layout = _attention_layout(kvm) tensors = {} with _cast(model, attn.values(), device): for i, kv in attn.items(): - tensors[f"layers.{i}.key.weight"] = kv.key[:, :, :pos].to("cpu", copy=True) - tensors[f"layers.{i}.value.weight"] = kv.value[:, :, :pos].to("cpu", copy=True) + for key, (owner, field) in kv.state_tensors().items(): + name = f"layers.{i}.{key}.weight" + tensors[name] = _trimmed(getattr(owner, field), pos, layout[name]).to("cpu", copy=True) _write(path, _file_tensors(tensors, {pos: linear_states}), {"key": key, "ids": _ids_json(ids[:pos])}) stats["shared_start_saved"] = True @@ -452,6 +488,7 @@ def _save_shared_start(model, kvm, key, ids, pos, linear_states, device): def finish(ids, generated): """Checkpoint the prompt plus the generated tokens the DeltaNet state has committed, keep the state resident, and spill others past the budget.""" + from . import llama entry, kvm, device, model = (_current.get(k) for k in ("entry", "kvm", "device", "model")) _current.clear() if entry is None: @@ -465,7 +502,8 @@ def finish(ids, generated): if len(ids) < idx <= len(full): entry.checkpoints[idx] = _snapshot(model, kvm, device) entry.ids = full[:idx] - entry.state.patcher.set_in_use_by_current_prompt(False) + entry.written = max(kv.index for kv in kvm.caches) + llama.SPEC_ROWS # a verify step writes up to SPEC_ROWS rows ahead + _in_use(entry.state.patcher, False) _tick[0] += 1 entry.used = _tick[0] _entries.append(entry) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index ead311d3631..f228a54faa5 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -497,8 +497,6 @@ def __init__(self, config, device=None, dtype=None, ops=None): self.comfy_attention = ComfyAttention() def kv_cache_type(self, device, dtype): - if llm_state_cache.enabled(): - return FixedKVCache # the prompt state cache continues prefills from a cached prefix; int8 can't method = self.comfy_attention.method if method == "comfy_kitchen_int8" or method is None and comfy.model_management.comfy_kitchen_attention_enabled(): if dtype == torch.bfloat16 and self.head_dim == 256 and int8_decode_is_available(device): @@ -997,10 +995,9 @@ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, pos generator = torch.Generator(device=device).manual_seed(sampling["seed"]) if sampling is not None else None if cache_ids is not None: # the prompt state cache: a resident state with the longest matching prefix, prefilled to the end of the prompt - cache, x = llm_state_cache.prefill(self, embeds, cache_ids, cap, - lambda e, kv, p: self.model.forward(None, embeds=e, attention_mask=None, past_key_values=kv, position_ids=p)[0], - position_ids=position_ids) - cap = cache.caches[self.model.config.layer_types.index("full_attention")].key.shape[2] + cache, x, cap = llm_state_cache.prefill(self, embeds, cache_ids, cap, + lambda e, kv, p: self.model.forward(None, embeds=e, attention_mask=None, past_key_values=kv, position_ids=p)[0], + position_ids=position_ids) else: cache = model_state.load(self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True), device) self.restore_generation_state(state, cache, generator) diff --git a/tests-unit/comfy_test/llm_state_cache_test.py b/tests-unit/comfy_test/llm_state_cache_test.py new file mode 100644 index 00000000000..1bf528451ad --- /dev/null +++ b/tests-unit/comfy_test/llm_state_cache_test.py @@ -0,0 +1,120 @@ +import types + +import pytest +import torch + +from qwen35_decode_test import _ExactPacked, _tiny_model + +import comfy.text_encoders.llm_state_cache as cache # noqa: E402 + +END = list(cache.MSG_END) + + +class _Pages16(_ExactPacked): + page = 16 # a 48-row window: room to resume a turn after the previous reply's draft rows + + +@pytest.fixture(params=["bf16", "int8"]) +def model(request, monkeypatch, tmp_path): + import comfy.text_encoders.llama as llama + import comfy.text_encoders.qwen35 as qwen35 + model = _tiny_model(monkeypatch) + if request.param == "int8": + monkeypatch.setattr(llama, "Int8DecodeCache", _Pages16, raising=False) + monkeypatch.setattr(qwen35.GatedAttention, "kv_cache_type", lambda self, device, dtype: qwen35.Int8FixedKVCache) + monkeypatch.setattr(cache, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(cache, "ROUND", 64) + monkeypatch.setattr(cache, "HEADROOM", 0) # a later turn outgrows the first one's capacity + cache._entries.clear() + cache._current.clear() + yield model + for entry in list(cache._entries): + cache._drop(entry) + + +_table = {} + + +def _embeds(model, ids): + # generated tokens embed as the model embeds them; other ids (message ends, prompt text) get fixed vectors + rows = [] + for i in ids: + if i < 97: + rows.append(model.model.embed_tokens(torch.tensor([i]))[0]) + else: + rows.append(_table.setdefault(i, torch.randn(64, generator=torch.Generator().manual_seed(i)))) + return torch.stack(rows).unsqueeze(0) + + +def _text(seed, n): + return [1000 + seed * 100 + k for k in range(n)] + + +def _generate(model, ids, max_length, mode): + """(generated ids, first logits after the prompt) with the cache on or off.""" + first = [] + logits = model.logits + + def record(x): + out = logits(x) + if not first: + first.append(out.detach().clone()) + return out + model.logits = record + try: + with cache.call_mode(mode): + kwargs = {"cache_ids": list(ids)} if mode == "on" else {} + out = model.generate(_embeds(model, ids), do_sample=False, max_length=max_length, mtp=True, **kwargs) + finally: + model.logits = logits + return out, first[0] + + +def _check(model, ids, max_length=8): + got, got_logits = _generate(model, ids, max_length, "on") + stats = dict(cache.stats) + want, want_logits = _generate(model, ids, max_length, "off") + assert got == want + assert torch.allclose(got_logits, want_logits, atol=1e-4), stats + return stats + + +def test_next_turn_resumes_the_conversation(model): + system = _text(0, 12) + END + turn1 = system + _text(1, 20) + END + stats = _check(model, turn1) + assert stats["prefix_len"] == 0 and stats.get("shared_start_saved") + reply = cache._entries[-1].ids[len(turn1):] # the reply tokens the state holds + turn2 = turn1 + reply + END + _text(2, 30) + END + stats = _check(model, turn2) + assert stats["prefix_len"] >= len(turn1) and stats["restore"] in ("resident", "grow") + + +def test_spilled_conversation_reloads_from_disk(model, monkeypatch): + monkeypatch.setattr(cache, "MAX_ENTRIES", 1) + # longer than the int8 window, so the pages must survive the spill + a = _text(0, 12) + END + _text(1, 70) + END + b = _text(5, 12) + END + _text(6, 20) + END # another system prompt: no shared start + _check(model, a) + reply = cache._entries[-1].ids[len(a):] + _check(model, b) + assert sum(e.state is None for e in cache._entries) == 1 # a was written to disk + stats = _check(model, a + reply + END + _text(2, 15) + END) + assert stats["restore"] == "from-disk" and stats["prefix_len"] >= len(a) + + +def test_new_conversation_starts_from_the_shared_start_file(model): + system = _text(0, 70) + END # longer than the int8 window, so the file must hold its pages + _check(model, system + _text(1, 20) + END) + for entry in list(cache._entries): # as after a restart: only the files remain + cache._drop(entry) + stats = _check(model, system + _text(3, 25) + END) + assert stats["restore"] == "shared-start-file" and stats["prefix_len"] == len(system) + + +def test_int8_resumes_only_inside_its_window(): + entry = types.SimpleNamespace(window=(48, 16), written=200) + # the refresh before pos reads the page holding pos - 1 and the one before it; the window holds rows 152..199 + assert not cache._resumable(entry, 170) # needs rows from 144 + assert cache._resumable(entry, 180) # needs rows from 160 + assert cache._resumable(types.SimpleNamespace(window=None, written=200), 10) diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py index 748cb73bf8b..8c22353278d 100644 --- a/tests-unit/comfy_test/qwen35_decode_test.py +++ b/tests-unit/comfy_test/qwen35_decode_test.py @@ -500,22 +500,22 @@ def test_long_continuation_prefills_in_one_batch(monkeypatch, tmp_path, mtp): class _ExactPacked: - # Int8DecodeCache stand-in that pages exact rows, 4 per page, so a continued int8 prefill can be checked + # Int8DecodeCache stand-in that pages exact rows, `page` per page, so a continued int8 prefill can be checked # against bf16. A refresh reads only what the kernel may: the last two pages, from the circular window. - dim = 64 + dim, page = 64, 4 def __init__(self, batch, heads, capacity, device): - shape = (batch, (capacity + 3) // 4, heads, 4, self.dim) + shape = (batch, -(-capacity // self.page), heads, self.page, self.dim) self.key, self.value = torch.zeros(shape, device=device), torch.zeros(shape, device=device) - self.key_scale, self.value_scale = torch.zeros(1, device=device), torch.zeros(1, device=device) + self.key_scale, self.value_scale = (torch.zeros(shape[:3] + (1,), device=device) for _ in range(2)) # per page, as the kernel's def update(self, key, value, length, initialize=False): - n = int(length.item()) - first = 0 if initialize else max(0, (n - 1) // 4 * 4 - 4) + n, page = int(length.item()), self.page + first = 0 if initialize else max(0, (n - 1) // page * page - page) for i in range(first, n): src = i if initialize else i % key.shape[2] - self.key[:, i // 4, :, i % 4] = key[:, :, src] - self.value[:, i // 4, :, i % 4] = value[:, :, src] + self.key[:, i // page, :, i % page] = key[:, :, src] + self.value[:, i // page, :, i % page] = value[:, :, src] def attend(self, query, length): n = int(length.item()) From aace0b7a92adc1e18019800c04347898deef0986 Mon Sep 17 00:00:00 2001 From: Christian Byrne Date: Sun, 1 Nov 2026 07:57:48 -0800 Subject: [PATCH 18/24] Fix MTPHead unpack crash in prefetch queue cleanup (CORE-644) (#17180) Cleanup now skips unreached raw module groups, resets PREFETCH_QUEUES before walking them, keeps cleaning after one failure and re-raises the first error, and always drops _prefetch markers. The Qwen3.5 MTP draft step passes its group as a list. Regression tests in tests-unit/comfy_test/model_prefetch_cleanup_test.py. --- comfy/model_prefetch.py | 58 ++++++-- comfy/text_encoders/qwen35.py | 2 +- .../comfy_test/model_prefetch_cleanup_test.py | 128 ++++++++++++++++++ 3 files changed, 173 insertions(+), 15 deletions(-) create mode 100644 tests-unit/comfy_test/model_prefetch_cleanup_test.py diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 022685125d2..f68ee59bb79 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -123,21 +123,35 @@ def pin_modules(comfy_modules, device, dtype=None): return offload_stream, fully_faulted def cleanup_prefetched_modules(module, comfy_modules): + # Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight + # skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error. + first_error = None for s in comfy_modules: prefetch = getattr(s, "_prefetch", None) if prefetch is None: continue - for param_key in ("weight", "bias"): - lowvram_fn = getattr(s, param_key + "_lowvram_function", None) - if lowvram_fn is not None: - lowvram_fn.clear_prepared() - if prefetch["signature"] is not None: - comfy_aimdo.model_vbar.vbar_unpin(s._v) - delattr(s, "_prefetch") + try: + for param_key in ("weight", "bias"): + lowvram_fn = getattr(s, param_key + "_lowvram_function", None) + if lowvram_fn is not None: + lowvram_fn.clear_prepared() + if prefetch["signature"] is not None: + comfy_aimdo.model_vbar.vbar_unpin(s._v) + except Exception as e: + if first_error is None: + first_error = e + finally: + delattr(s, "_prefetch") for block in getattr(module, "_v_blocks_faulted", ()): - comfy_aimdo.model_vbar.vbar_unpin(block) + try: + comfy_aimdo.model_vbar.vbar_unpin(block) + except Exception as e: + if first_error is None: + first_error = e if hasattr(module, "_v_blocks_faulted"): del module._v_blocks_faulted + if first_error is not None: + raise first_error def fault_graph_blocks(module, blocks): signatures = [] @@ -174,19 +188,35 @@ def cleanup_prefetch_queues(): global MALLOC_GRAPH_ROGUES global MALLOC_GRAPH_USED - cleanup_malloc_graph() - for queue in PREFETCH_QUEUES: - # Only the head has been prefetched; later entries are raw module groups. - if queue[0] is not None: - _, (prefetched_module, comfy_modules) = queue[0] - cleanup_prefetched_modules(prefetched_module, comfy_modules) + queues = PREFETCH_QUEUES + # Reset before walking so a failing entry can't leave stale queues behind for every later node. PREFETCH_QUEUES = [] + first_error = None + try: + cleanup_malloc_graph() + except Exception as e: + first_error = e + for queue in queues: + # Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later + # entries, and a head that was not reached yet, still hold the raw module or module group. + head = queue[0] if queue else None + if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple): + continue + prefetched_module, comfy_modules = head[1] + try: + cleanup_prefetched_modules(prefetched_module, comfy_modules) + except Exception as e: + # Keep unpinning the other queues; report the first failure once they are done. + if first_error is None: + first_error = e GRAPH_WARMED_MODULES.clear() if MALLOC_GRAPH_USED: logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES) MALLOC_GRAPH_BREAKS = 0 MALLOC_GRAPH_ROGUES = 0 MALLOC_GRAPH_USED = False + if first_error is not None: + raise first_error def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None): malloc_graph = MALLOC_GRAPHS.get(threading.get_ident()) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index f228a54faa5..978e045eb7b 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -1091,7 +1091,7 @@ def draft(token, hidden, p): draft_freqs.copy_(freqs_at(p)) host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed) queue = comfy.model_prefetch.make_prefetch_queue( - [None if host_rows else embed, (head, self.mtp, mtp_kv.storage)], device, + [None if host_rows else embed, [head, self.mtp, mtp_kv.storage]], device, {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars}) def embed_core(): diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py new file mode 100644 index 00000000000..4ccaa71474c --- /dev/null +++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py @@ -0,0 +1,128 @@ +import pytest +import torch + +from comfy.cli_args import args as cli_args + +if not torch.cuda.is_available(): + cli_args.cpu = True + +import comfy.model_prefetch as model_prefetch # noqa: E402 + + +class MTPHead(torch.nn.Module): + pass + + +@pytest.fixture(autouse=True) +def _reset_queues(): + model_prefetch.PREFETCH_QUEUES = [] + yield + model_prefetch.PREFETCH_QUEUES = [] + + +def _prefetched(module): + module._prefetch = {"signature": None} + return (None, (module, [module])) + + +@pytest.mark.parametrize("pending_unit", [tuple, list]) +def test_cleanup_skips_unconsumed_grouped_unit(pending_unit): + # Qwen3.5 MTP draft queue interrupted after the embed pop: the embed slot has + # been pinned, the grouped (head, mtp) slot has not been reached yet. + embed = torch.nn.Embedding(4, 4) + head = torch.nn.Linear(4, 4) + queue = [_prefetched(embed), pending_unit((head, MTPHead())), None] + model_prefetch.PREFETCH_QUEUES = [queue] + + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + assert not hasattr(embed, "_prefetch") + + +def test_cleanup_resets_queues_when_entry_cleanup_fails(monkeypatch): + def boom(module, comfy_modules): + raise RuntimeError("unpin failed") + + monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", boom) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(torch.nn.Linear(4, 4)), None]] + + with pytest.raises(RuntimeError): + model_prefetch.cleanup_prefetch_queues() + # A failed cleanup must not poison every later node on the worker. + assert model_prefetch.PREFETCH_QUEUES == [] + model_prefetch.cleanup_prefetch_queues() + + +@pytest.mark.parametrize("pending_unit", [tuple, list]) +def test_cleanup_skips_unreached_grouped_head(pending_unit): + # Interrupted between popping the consumed slot and pinning the next one: the + # head of the queue is still the raw (head, mtp) group, not a pinned slot. + queue = [pending_unit((torch.nn.Linear(4, 4), MTPHead())), None] + model_prefetch.PREFETCH_QUEUES = [queue] + + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + + +def test_cleanup_unpins_later_queues_when_an_earlier_one_fails(monkeypatch): + real_cleanup = model_prefetch.cleanup_prefetched_modules + failing = torch.nn.Linear(4, 4) + later = torch.nn.Linear(4, 4) + + def cleanup(module, comfy_modules): + if module is failing: + raise RuntimeError("unpin failed") + real_cleanup(module, comfy_modules) + + monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", cleanup) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(failing), None], [_prefetched(later), None]] + + with pytest.raises(RuntimeError, match="unpin failed"): + model_prefetch.cleanup_prefetch_queues() + # The first failure is re-raised, but only after every other queue was cleaned up. + assert not hasattr(later, "_prefetch") + assert model_prefetch.PREFETCH_QUEUES == [] + + +def test_failed_unpin_still_clears_prefetch_markers(monkeypatch): + # A module whose unpin raised must not keep _prefetch, or a later cast would skip + # the fresh cast and reuse the stale transfer state. The rest of the group is still cleaned. + calls = [] + + def vbar_unpin(v): + calls.append(v) + if v == "first": + raise RuntimeError("unpin failed") + + monkeypatch.setattr(model_prefetch.comfy_aimdo.model_vbar, "vbar_unpin", vbar_unpin) + first = torch.nn.Linear(4, 4) + second = torch.nn.Linear(4, 4) + for module, v in ((first, "first"), (second, "second")): + module._v = v + module._prefetch = {"signature": "sig"} + model_prefetch.PREFETCH_QUEUES = [[(None, (first, [first, second])), None]] + + with pytest.raises(RuntimeError, match="unpin failed"): + model_prefetch.cleanup_prefetch_queues() + assert calls == ["first", "second"] + assert not hasattr(first, "_prefetch") + assert not hasattr(second, "_prefetch") + + +def test_malloc_graph_cleanup_failure_still_cleans_queues(monkeypatch): + # After a GPU failure the graph abort can raise too. Queues and _prefetch markers must not + # survive into the next node, and the graph error still reaches the caller. + def boom(): + raise RuntimeError("graph abort failed") + + monkeypatch.setattr(model_prefetch, "cleanup_malloc_graph", boom) + module = torch.nn.Linear(4, 4) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(module), None]] + + with pytest.raises(RuntimeError, match="graph abort failed"): + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + assert not hasattr(module, "_prefetch") From af6b54eae143f10cb4927198234ac553b720fabb Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Wed, 4 Nov 2026 10:37:39 -0800 Subject: [PATCH 19/24] Register w3a8_int8 and w5a8_int8 grouped INT8 formats For comfy-kitchen#303's 3- and 5-bit widths of AsymW4A8Int8Layout. --- comfy/ops.py | 2 +- comfy/quant_ops.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/comfy/ops.py b/comfy/ops.py index 820e0a37d76..58a88f486c8 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -1130,7 +1130,7 @@ def linear_input_act_(x, weight, bias, input_act=None, act_weight=None, act_eps= # Grouped-scale integer weights that run through the shared INT8 GEMM (comfy_kitchen # AsymW4A8Int8Layout): format -> code width, which also fixes the packed weight width K*bits/8. -_GROUPED_INT8_FORMATS = {"asym_w4a8_int8": 4, "w6a8_int8": 6} +_GROUPED_INT8_FORMATS = {"w3a8_int8": 3, "asym_w4a8_int8": 4, "w5a8_int8": 5, "w6a8_int8": 6} class QuantLinearFunc(torch.autograd.Function): diff --git a/comfy/quant_ops.py b/comfy/quant_ops.py index 0b0f4f86369..6f5e58a878f 100644 --- a/comfy/quant_ops.py +++ b/comfy/quant_ops.py @@ -264,6 +264,11 @@ class TensorCoreFP8E5M2Layout(_TensorCoreFP8LayoutBase): "quantize_input": False, } +# 3-bit (8-entry codebook, [N, 3K/8]) and 5-bit (uniform, [N, 5K/8]) widths of the same layout; +# need comfy_kitchen with 3/5-bit support (comfy-kitchen#303). +for _fmt in ("w3a8_int8", "w5a8_int8"): + QUANT_ALGOS[_fmt] = dict(QUANT_ALGOS["w6a8_int8"]) + # ============================================================================== # Re-exports for backward compatibility From 512bb0fd1d6e50b6159680ca9c1a6e681006df71 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 5 Nov 2026 01:31:41 -0800 Subject: [PATCH 20/24] Pin comfy-kitchen to #303's head (34bc583) for the w3/w5 formats For our evals and demos only, until comfy-kitchen#303 merges and is released. --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index beddb6f33a0..0a5ab43f9cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,9 @@ SQLAlchemy>=2.0.0 filelock av>=17.0.0 comfy-aimdo==0.6.0 -comfy-kitchen==0.2.40 +# Pinned to comfy-kitchen#303's head (3- and 5-bit weight widths) for our evals and demos, until it merges and is released. +# Boxes with no GitHub access or no CUDA toolkit install the CUDA wheel #303's CI built from this same commit instead. +comfy-kitchen @ git+https://github.com/kijai/comfy-kitchen.git@34bc5830941b7d356343f2bceeb72712d4efade6 requests simpleeval>=1.0.0 blake3 From 21f6ba1bb9aba8d2e78f36f8a5290bed4bf796b4 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 5 Nov 2026 11:42:36 -0800 Subject: [PATCH 21/24] Merge a continued int8 prefill's rows per key/value head group The causal merge repeated the keys and values to every query head in float32 and scored 512 rows against the whole continue at once: several GB of temporaries for a continue of 30k rows. It now scores each key/value head's query heads in one matmul and sizes the row chunks to a fixed float32 budget. The prefix attention is written into one preallocated output instead of concatenated. --- comfy/text_encoders/llama.py | 38 +++++++++++++++------ tests-unit/comfy_test/qwen35_decode_test.py | 7 ++-- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index 23e43737271..e2e8ed016af 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -190,8 +190,15 @@ def extend(self, xq, xk, xv): length = torch.full_like(self.position[:1], start) self.packed.update(self.key, self.value, length) step = max(1, 64 // (xq.shape[1] // xk.shape[1])) # query rows per call, no more than a verify step uses - parts = [self.packed.attend(xq[:, :, r:r + step].contiguous(), length) for r in range(0, seq, step)] - out = causal_merge(torch.cat([o for o, _ in parts], dim=1), torch.cat([lse for _, lse in parts], dim=2), xq, xk, xv) + out, lse = None, None + for r in range(0, seq, step): + o, s = self.packed.attend(xq[:, :, r:r + step].contiguous(), length) + if out is None: + out = o.new_empty((o.shape[0], seq, o.shape[2])) + lse = s.new_empty((s.shape[0], s.shape[1], seq)) + out[:, r:r + step] = o + lse[:, :, r:r + step] = s + out = causal_merge(out, lse, xq, xk, xv) window, page = self.key.shape[2], self.packed.key.shape[3] pos = start while pos < start + seq: @@ -204,23 +211,32 @@ def extend(self, xq, xk, xv): return out -def causal_merge(out, lse, q, k, v, chunk=512): +CAUSAL_MERGE_ELEMENTS = 1 << 26 # float32 scores per chunk (256 MB) + + +def causal_merge(out, lse, q, k, v): """Fold rows' causal attention over each other into their attention over a prefix. out [B, S, H*D] and lse [B, H, S] are every row's attention over the prefix; q [B, H, S, D] and - k/v [B, Hk, S, D] the rows themselves. Row i also attends rows <= i. Returns [B, S, H*D].""" + k/v [B, Hk, S, D] the rows themselves. Row i also attends rows <= i. Returns [B, S, H*D]. + The query heads sharing a key/value head go through one matmul, so k and v are never repeated.""" batch, heads, rows, dim = q.shape - kf = k.float().repeat_interleave(heads // k.shape[1], dim=1) - vf = v.float().repeat_interleave(heads // k.shape[1], dim=1) + kv_heads = k.shape[1] + groups = heads // kv_heads + kf, vf = k.float(), v.float() merged = torch.empty_like(out) + chunk = max(1, min(rows, CAUSAL_MERGE_ELEMENTS // (heads * rows))) for r in range(0, rows, chunk): n = min(chunk, rows - r) - scores = torch.matmul(q[:, :, r:r + n].float(), kf[:, :, :r + n].transpose(-1, -2)) * dim ** -0.5 + qc = q[:, :, r:r + n].float().reshape(batch, kv_heads, groups * n, dim) + scores = torch.matmul(qc, kf[:, :, :r + n].transpose(-1, -2)).mul_(dim ** -0.5).view(batch, heads, n, r + n) scores.masked_fill_(torch.ones(n, r + n, dtype=torch.bool, device=q.device).triu_(r + 1), float("-inf")) - m = torch.maximum(lse[:, :, r:r + n], scores.amax(-1)) - w_prefix = torch.exp(lse[:, :, r:r + n] - m) - w = torch.exp(scores - m.unsqueeze(-1)) - acc = out[:, r:r + n].view(batch, n, heads, dim).transpose(1, 2).float() * w_prefix.unsqueeze(-1) + torch.matmul(w, vf[:, :, :r + n]) + prefix_lse = lse[:, :, r:r + n] + m = torch.maximum(prefix_lse, scores.amax(-1)) + w_prefix = torch.exp(prefix_lse - m) + w = scores.sub_(m.unsqueeze(-1)).exp_() + acc = torch.matmul(w.view(batch, kv_heads, groups * n, r + n), vf[:, :, :r + n]).view(batch, heads, n, dim) + acc.add_(out[:, r:r + n].view(batch, n, heads, dim).transpose(1, 2).float() * w_prefix.unsqueeze(-1)) merged[:, r:r + n] = (acc / (w_prefix + w.sum(-1)).unsqueeze(-1)).transpose(1, 2).reshape(batch, n, heads * dim).to(out.dtype) return merged diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py index 8c22353278d..402a9be6369 100644 --- a/tests-unit/comfy_test/qwen35_decode_test.py +++ b/tests-unit/comfy_test/qwen35_decode_test.py @@ -527,11 +527,14 @@ def attend(self, query, length): return out.transpose(1, 2).reshape(batch, seq, heads * dim).to(query.dtype), scores.logsumexp(-1) -def test_int8_extend_matches_full_attention(monkeypatch): +@pytest.mark.parametrize("merge_elements", [1 << 26, 4 * 19 * 5]) +def test_int8_extend_matches_full_attention(monkeypatch, merge_elements): # a continued prefill over an int8 prefix: each new row sees the prefix and the new rows up to itself, - # and the pages afterwards hold every row, across page and window boundaries + # and the pages afterwards hold every row, across page and window boundaries; the small merge budget + # takes the new rows in chunks of 5 import comfy.text_encoders.llama as llama monkeypatch.setattr(llama, "Int8DecodeCache", _ExactPacked, raising=False) + monkeypatch.setattr(llama, "CAUSAL_MERGE_ELEMENTS", merge_elements) kv = llama.Int8FixedKVCache.zeros(1, 2, 64, 64, "cpu", torch.float32, llama.FixedKVCache.shared(1, "cpu")) q, k, v = torch.randn(1, 4, 30, 64), torch.randn(1, 2, 30, 64), torch.randn(1, 2, 30, 64) From 8e66106a7fe7458605976667aff117e6fa327730 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 5 Nov 2026 12:04:14 -0800 Subject: [PATCH 22/24] Prefill a long prompt in pieces of at most 4096 rows A one-pass prefill of a 41k-row prompt needs about 18 GB of working memory, which pages every weight out of VRAM; the decode that follows reads part of them back from system memory each step. Each piece now continues the one before through the batched continue. --- comfy/text_encoders/qwen35.py | 16 +++++++++- tests-unit/comfy_test/qwen35_decode_test.py | 34 +++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 978e045eb7b..3482f4e4d29 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -866,6 +866,11 @@ def forward(self, embeds, hidden, freqs_cis, past_key_value): # Model Wrapper +# Rows per prefill pass: one pass over a long prompt needs enough working memory to page the +# weights out of VRAM, and the decode after it would read them back from system memory. +PREFILL_ROWS = 4096 + + class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module): model_type = "qwen35_2b" @@ -911,7 +916,16 @@ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermed return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids, past_key_values=past_key_values) def prefill(self, embeds, past_key_values, position_ids=None, **kwargs): - if past_key_values[0].index == 0 or self.batched_continue(embeds.shape[1], past_key_values): + rows = embeds.shape[1] + batched = past_key_values[0].index == 0 or self.batched_continue(rows, past_key_values) + if batched and rows > PREFILL_ROWS and not kwargs: + # a long prompt goes in equal pieces, each continuing the one before + size = -(-rows // -(-rows // PREFILL_ROWS)) + for start in range(0, rows, size): + positions = position_ids[..., start:start + size] if position_ids is not None else None + result = super().prefill(embeds[:, start:start + size], past_key_values, positions) + return result + if batched: return super().prefill(embeds, past_key_values, position_ids, **kwargs) # Otherwise the new tokens go through the deferred decode kernels in rows of 1, 4 or 8, # which also commit any rows the saved state left uncommitted. A multi-row diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py index 402a9be6369..0475bdedcb1 100644 --- a/tests-unit/comfy_test/qwen35_decode_test.py +++ b/tests-unit/comfy_test/qwen35_decode_test.py @@ -469,6 +469,40 @@ def prefill(*parts): _, rest = prefill(slice(0, 11), slice(11, 30)) assert torch.allclose(rest, whole[:, 11:], atol=1e-4) +@pytest.mark.parametrize("kv", ["bf16", "int8"]) +def test_long_prompt_prefills_in_pieces(monkeypatch, kv): + # a prompt over PREFILL_ROWS goes in equal pieces that continue each other, and its last hidden state + # and the cache it leaves match one pass over the whole prompt + import comfy.text_encoders.llama as llama + import comfy.text_encoders.qwen35 as qwen35 + model = _tiny_model(monkeypatch) + if kv == "int8": + monkeypatch.setattr(llama, "Int8DecodeCache", _ExactPacked, raising=False) + monkeypatch.setattr(qwen35.GatedAttention, "kv_cache_type", lambda self, device, dtype: qwen35.Int8FixedKVCache) + embeds, more = torch.randn(1, 30, 64), torch.randn(1, 12, 64) + passes = [] + forward = model.model.forward + monkeypatch.setattr(model.model, "forward", lambda *a, **kw: passes.append(kw["embeds"].shape[1]) or forward(*a, **kw)) + + def prefill(rows): + monkeypatch.setattr(qwen35, "PREFILL_ROWS", rows) + state = comfy.model_state.ModelState() + comfy.model_management.register_ephemeral_model(state) + cpu = torch.device("cpu") + cache = state.load(model.init_kv_cache(1, 48, cpu, torch.float32, mtp=True), cpu) + last = model.prefill(embeds.clone(), cache.caches)[0][:, -1] # the forward reuses its input buffer + monkeypatch.setattr(qwen35, "PREFILL_ROWS", 4096) + after = model.prefill(more.clone(), cache.caches)[0] + state.release() + return last, after + + whole = prefill(4096) + passes.clear() + pieces = prefill(11) + assert passes == [10, 10, 10, 12] + assert torch.allclose(pieces[0], whole[0], atol=1e-4) + assert torch.allclose(pieces[1], whole[1], atol=1e-4) + @pytest.mark.parametrize("mtp", [False, "3", "tree2s2"]) def test_long_continuation_prefills_in_one_batch(monkeypatch, tmp_path, mtp): # a continued turn with more than SPEC_ROWS new tokens prefills as one batch, from the saved From 8d94c644714553000d7b8c10d6f4d69a1f91229e Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 5 Nov 2026 12:04:27 -0800 Subject: [PATCH 23/24] Prefill the prompt state cache's segments through Qwen35.prefill So a long segment goes in pieces too, and a short one through the decode path. --- comfy/text_encoders/qwen35.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 3482f4e4d29..e3dbc523123 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -1010,7 +1010,7 @@ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, pos if cache_ids is not None: # the prompt state cache: a resident state with the longest matching prefix, prefilled to the end of the prompt cache, x, cap = llm_state_cache.prefill(self, embeds, cache_ids, cap, - lambda e, kv, p: self.model.forward(None, embeds=e, attention_mask=None, past_key_values=kv, position_ids=p)[0], + lambda e, kv, p: self.prefill(e, kv, p)[0], position_ids=position_ids) else: cache = model_state.load(self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True), device) From 998d7f1443e8d1323690ceb7aa02dca357434071 Mon Sep 17 00:00:00 2001 From: Simon Pinfold Date: Thu, 5 Nov 2026 14:02:40 -0800 Subject: [PATCH 24/24] Don't query device memory from /system_stats while a prompt runs From ComfyUI#17557. --- comfy/model_management.py | 3 ++ main.py | 3 +- server.py | 17 +++++++- tests-unit/app_test/test_prompt_worker.py | 31 ++++++++++++++ tests-unit/server_test/test_system_stats.py | 45 +++++++++++++++++++++ 5 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 tests-unit/server_test/test_system_stats.py diff --git a/comfy/model_management.py b/comfy/model_management.py index 38588d5df0e..5fd83deab69 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -313,6 +313,9 @@ def cuda_device_context(device): if prev is not None: torch.cuda.set_device(prev) +# Held while a prompt executes: a dynamic VRAM page-out can hold torch's allocator lock and hand the GIL to another thread, where torch.cuda.memory_stats then deadlocks. +model_execution_lock = threading.Lock() + def get_total_memory(dev=None, torch_total_too=False): global directml_enabled if dev is None: diff --git a/main.py b/main.py index dc484c7eb01..64988f13467 100644 --- a/main.py +++ b/main.py @@ -381,7 +381,8 @@ def prompt_worker(q, server_instance, asset_manager): asset_manager.pause_background_scan() background_scan_paused = True - e.execute(item[2], prompt_id, extra_data, item[4]) + with comfy.model_management.model_execution_lock: + e.execute(item[2], prompt_id, extra_data, item[4]) need_gc = True diff --git a/server.py b/server.py index 3eb75d925bb..409e597a4ab 100644 --- a/server.py +++ b/server.py @@ -273,6 +273,8 @@ def __init__(self, loop, asset_manager: "AssetManager"): self.prompt_queue = execution.PromptQueue(self) self.loop = loop self.messages = asyncio.Queue() + self.device_memory = {} + self.read_device_memory_unless_a_prompt_runs([comfy.model_management.get_torch_device()] + list(comfy.model_management.get_all_torch_devices())) self.client_session:Optional[aiohttp.ClientSession] = None self.number = 0 @@ -790,10 +792,11 @@ async def system_stats(request): else: torch_devices = [primary_device] + list(torch_devices) + self.read_device_memory_unless_a_prompt_runs(torch_devices) + device_entries = [] for d in torch_devices: - vram_total, torch_vram_total = comfy.model_management.get_total_memory(d, torch_total_too=True) - vram_free, torch_vram_free = comfy.model_management.get_free_memory(d, torch_free_too=True) + vram_total, torch_vram_total, vram_free, torch_vram_free = self.device_memory[str(d)] device_entries.append({ "name": comfy.model_management.get_torch_device_name(d), "type": d.type, @@ -1319,6 +1322,16 @@ async def setup(self): timeout = aiohttp.ClientTimeout(total=None) # no timeout self.client_session = aiohttp.ClientSession(timeout=timeout) + def read_device_memory_unless_a_prompt_runs(self, devices): + if not comfy.model_management.model_execution_lock.acquire(blocking=False): + return + try: + for d in devices: + self.device_memory[str(d)] = (comfy.model_management.get_total_memory(d, torch_total_too=True) + + comfy.model_management.get_free_memory(d, torch_free_too=True)) + finally: + comfy.model_management.model_execution_lock.release() + def add_routes(self): # The comfy-agent host mounts its own /api/agent and /api/workflows # routes (the proxy + trust boundary now live in the wheel). Do it before diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py index 16e7a11065b..be04be7f8b3 100644 --- a/tests-unit/app_test/test_prompt_worker.py +++ b/tests-unit/app_test/test_prompt_worker.py @@ -155,3 +155,34 @@ def test_prompt_worker_resumes_scan_once_the_queue_is_empty(monkeypatch) -> None def test_prompt_worker_keeps_scan_paused_while_prompts_are_queued(monkeypatch) -> None: assert _paused_when_the_worker_asks_for_the_next_prompt(monkeypatch, queued_after_first=1) == [True] + + +class LockCheckingExecutor(Executor): + held_during_execute = None + + def execute(self, *args, **kwargs) -> None: + LockCheckingExecutor.held_during_execute = main.comfy.model_management.model_execution_lock.locked() + + +def test_prompt_worker_holds_model_execution_lock_only_while_executing(monkeypatch) -> None: + monkeypatch.setattr(main.execution, "PromptExecutor", LockCheckingExecutor) + monkeypatch.setattr(main.time, "perf_counter", lambda: 1.0) + LockCheckingExecutor.held_during_execute = None + lock = main.comfy.model_management.model_execution_lock + held_while_waiting: list[bool] = [] + queue = Queue(on_second_get=lambda: held_while_waiting.append(lock.locked())) + + with pytest.raises(LoopEscape, match="^prompt worker requested a second item$"): + main.prompt_worker(queue, Server(), AssetManager()) + + assert LockCheckingExecutor.held_during_execute is True + assert held_while_waiting == [False] + + +def test_prompt_worker_releases_model_execution_lock_when_execute_raises(monkeypatch) -> None: + monkeypatch.setattr(main.execution, "PromptExecutor", ExecuteFailureExecutor) + + with pytest.raises(RuntimeError, match="^forced execute failure$"): + main.prompt_worker(Queue(), Server(), AssetManager()) + + assert not main.comfy.model_management.model_execution_lock.locked() diff --git a/tests-unit/server_test/test_system_stats.py b/tests-unit/server_test/test_system_stats.py new file mode 100644 index 00000000000..f82fec3b763 --- /dev/null +++ b/tests-unit/server_test/test_system_stats.py @@ -0,0 +1,45 @@ +from unittest.mock import MagicMock + +import pytest +from aiohttp.test_utils import TestClient, TestServer + +import comfy.model_management +import server + + +@pytest.mark.asyncio +async def test_system_stats_reports_last_values_while_a_prompt_runs(monkeypatch): + memory = {"total": (100, 6), "free": (14, 4)} + queried = [] + + def get_total_memory(dev=None, torch_total_too=False): + queried.append("total") + return memory["total"] if torch_total_too else memory["total"][0] + + def get_free_memory(dev=None, torch_free_too=False): + queried.append("free") + return memory["free"] if torch_free_too else memory["free"][0] + + monkeypatch.setattr(comfy.model_management, "get_total_memory", get_total_memory) + monkeypatch.setattr(comfy.model_management, "get_free_memory", get_free_memory) + prompt_server = server.PromptServer(None, MagicMock(enabled=False)) + handler = next(r.handler for r in prompt_server.routes if getattr(r, "path", None) == "/system_stats") + prompt_server.app.router.add_get("/system_stats", handler) + + def device(stats): + d = stats["devices"][0] + return d["vram_total"], d["torch_vram_total"], d["vram_free"], d["torch_vram_free"] + + async with TestClient(TestServer(prompt_server.app)) as client: + with comfy.model_management.model_execution_lock: + queried.clear() + assert device(await (await client.get("/system_stats")).json()) == (100, 6, 14, 4), \ + "a first request during a prompt gets the values read at startup" + assert queried == ["total", "free"], "only the cpu's RAM is read while a prompt runs" + + memory.update(total=(200, 7), free=(30, 5)) + assert device(await (await client.get("/system_stats")).json()) == (200, 7, 30, 5) + + memory.update(total=(300, 8), free=(40, 6)) + with comfy.model_management.model_execution_lock: + assert device(await (await client.get("/system_stats")).json()) == (200, 7, 30, 5)