diff --git a/comfy/anonymous_file.py b/comfy/anonymous_file.py index 0aa187db8eb..8077a5f7242 100644 --- a/comfy/anonymous_file.py +++ b/comfy/anonymous_file.py @@ -25,7 +25,8 @@ def create_anonymous_file(size=0): """Return an unbuffered binary file owned by the caller, initially sized in bytes. - Uses Linux memfd or a Windows temporary-cache/delete-on-close file. + Uses Linux memfd, a Windows temporary-cache/delete-on-close file, or an + unlinked temporary file elsewhere. Windows may still write to disk; Linux memfd may be swapped. Initial sizing is sparse with zero-filled holes; Windows requires a filesystem supporting sparse files. Keep the file open through all I/O and close @@ -47,7 +48,7 @@ def create_anonymous_file(size=0): raise OSError(errno, os.strerror(errno)) file = open(fd, "w+b", buffering=0) else: - raise NotImplementedError("anonymous files require Linux or Windows") + file = tempfile.TemporaryFile(buffering=0) try: if os.name == "nt": diff --git a/comfy/model_base.py b/comfy/model_base.py index 42ccd5dbfbc..856200a41c8 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -2800,7 +2800,7 @@ def get_writable_model(self, conds, noise_shape, model_options, device): transformer_options = model_options.get("transformer_options", {}) options = transformer_options.get("qwen_image21_cache", {}) model = self.diffusion_model - if options.get("device", "auto") != "auto" or not model.can_cache_prefix(transformer_options): + if options.get("device", "auto") != "auto" or not self.current_patcher.is_dynamic() or not model.can_cache_prefix(transformer_options): return None heads = model.transformer_blocks[0].attn.heads diff --git a/comfy/model_management.py b/comfy/model_management.py index e33147e74b9..5fd83deab69 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -313,6 +313,9 @@ def cuda_device_context(device): if prev is not None: torch.cuda.set_device(prev) +# Held while a prompt executes: a dynamic VRAM page-out can hold torch's allocator lock and hand the GIL to another thread, where torch.cuda.memory_stats then deadlocks. +model_execution_lock = threading.Lock() + def get_total_memory(dev=None, torch_total_too=False): global directml_enabled if dev is None: @@ -634,6 +637,8 @@ def get_torch_device_name(device): current_loaded_models: list[LoadedModel] = [] +EPHEMERAL_MODELS = [] + DIRTY_MMAPS = set() PIN_PRESSURE_HYSTERESIS = 256 * 1024 * 1024 @@ -1080,6 +1085,14 @@ def archive_model_dtypes(model): setattr(module, f"{buf_name}_comfy_model_dtype", buf.dtype) +def register_ephemeral_model(state): + """Keep a model state loaded until the node ends, then release whatever was not flushed.""" + EPHEMERAL_MODELS.append(state) + +def cleanup_ephemeral_models(): + while EPHEMERAL_MODELS: + EPHEMERAL_MODELS.pop().release() + def cleanup_models(): to_delete = [] for i in range(len(current_loaded_models)): diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index 83b92edb563..ecc5c52f85d 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -19,7 +19,6 @@ from __future__ import annotations import collections -import contextlib import inspect import logging import math @@ -38,6 +37,7 @@ import comfy.model_prefetch import comfy.ops import comfy.patcher_extension +import comfy.storage import comfy.utils import comfy_aimdo.host_buffer from comfy.comfy_types import UnetWrapperFunction @@ -1404,18 +1404,6 @@ def get_injections(self, key: str): def set_additional_models(self, key: str, models: list['ModelPatcher']): self.additional_models[key] = models - @contextlib.contextmanager - def writable_models(self, conds, noise_shape, model_options): - with contextlib.ExitStack() as stack: - for patcher in [self] + self.get_additional_models_with_key("multigpu"): - if not patcher.is_dynamic(): - continue - model = patcher.model.get_writable_model(conds, noise_shape, model_options, patcher.load_device) - if model is not None: - with comfy.model_management.cuda_device_context(patcher.load_device): - stack.enter_context(patcher.writable_model(model)) - yield - def remove_additional_models(self, key: str): if key in self.additional_models: self.additional_models.pop(key) @@ -1834,34 +1822,22 @@ def register_load_device(self, device): def is_dynamic(self): return True - @contextlib.contextmanager - def writable_model(self, model): - state, file = comfy.utils.create_safetensors(model.state_dict()) - patcher = None - try: - model.load_state_dict(state, assign=True) - patcher = ModelPatcherDynamic(model, self.load_device, torch.device("cpu"), fast_disk=self.fast_disk) - self.set_additional_models("writable", [patcher]) - # Loading reverses this list; the writable model gets priority over its owner. - comfy.model_management.load_models_gpu([patcher, self]) - patcher.set_in_use_by_current_prompt(True) - yield patcher - finally: - with comfy.model_management.cuda_device_context(self.load_device): - # The final forward may still be using the allocation graph's pages. - comfy.model_management.synchronize() - comfy.model_prefetch.cleanup_prefetch_queues() - comfy.model_prefetch.cleanup_malloc_graph() - if patcher is not None: - self.remove_additional_models("writable") - vbar = model.dynamic_vbars.pop(patcher.load_device, None) - if vbar is not None: - vbar.close() - for module in model.modules(): - for pin in getattr(module, "_pins", {}).values(): - pin["dirty"] = False - comfy.model_management.unload_model_and_clones(patcher, unload_additional_models=False) - file.close() + def discard_writes(self): + """Forget pinned writes so unpinning does not flush them to the backing file.""" + pin_state = self.model.dynamic_pins[self.load_device] + for subset in ("weights", "weights-loaded", "weights-fast"): + for module, _ in pin_state[subset][1]: + module._pins[subset]["dirty"] = False + + def flush_writes(self): + """Write every value of a writable model back to its backing file without evicting it.""" + for m in self.model.modules(): + if hasattr(m, "_v") and m._v_signature is not None: + m.writeback(m._v_weight) + pin_state = self.model.dynamic_pins[self.load_device] + for subset in ("weights", "weights-loaded", "weights-fast"): + for module, _ in pin_state[subset][1]: + comfy.pinned_memory.flush_pin(module, module._pins[subset]) def set_in_use_by_current_prompt(self, in_use): self.model.dynamic_pins[self.load_device]["current_prompt"] = in_use diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 6cfe9921f46..f68ee59bb79 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -123,20 +123,52 @@ def pin_modules(comfy_modules, device, dtype=None): return offload_stream, fully_faulted def cleanup_prefetched_modules(module, comfy_modules): + # Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight + # skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error. + first_error = None for s in comfy_modules: prefetch = getattr(s, "_prefetch", None) if prefetch is None: continue - for param_key in ("weight", "bias"): - lowvram_fn = getattr(s, param_key + "_lowvram_function", None) - if lowvram_fn is not None: - lowvram_fn.clear_prepared() - if prefetch["signature"] is not None: - comfy_aimdo.model_vbar.vbar_unpin(s._v) - delattr(s, "_prefetch") - if getattr(module, "_v_block_faulted", False): - comfy_aimdo.model_vbar.vbar_unpin(module._v_block) - del module._v_block_faulted + try: + for param_key in ("weight", "bias"): + lowvram_fn = getattr(s, param_key + "_lowvram_function", None) + if lowvram_fn is not None: + lowvram_fn.clear_prepared() + if prefetch["signature"] is not None: + comfy_aimdo.model_vbar.vbar_unpin(s._v) + except Exception as e: + if first_error is None: + first_error = e + finally: + delattr(s, "_prefetch") + for block in getattr(module, "_v_blocks_faulted", ()): + try: + comfy_aimdo.model_vbar.vbar_unpin(block) + except Exception as e: + if first_error is None: + first_error = e + if hasattr(module, "_v_blocks_faulted"): + del module._v_blocks_faulted + if first_error is not None: + raise first_error + +def fault_graph_blocks(module, blocks): + signatures = [] + pinned = [] + try: + for block in blocks: + signature = comfy_aimdo.model_vbar.vbar_fault(block) + if signature is None: + return None + pinned.append(block) + signatures.append(signature) + module._v_blocks_faulted = pinned + pinned = [] + return signatures + finally: + for block in pinned: + comfy_aimdo.model_vbar.vbar_unpin(block) def _drop_graph(module): graph = getattr(module, "_comfy_graph", None) @@ -156,19 +188,35 @@ def cleanup_prefetch_queues(): global MALLOC_GRAPH_ROGUES global MALLOC_GRAPH_USED - cleanup_malloc_graph() - for queue in PREFETCH_QUEUES: - # Only the head has been prefetched; later entries are raw module groups. - if queue[0] is not None: - _, (prefetched_module, comfy_modules) = queue[0] - cleanup_prefetched_modules(prefetched_module, comfy_modules) + queues = PREFETCH_QUEUES + # Reset before walking so a failing entry can't leave stale queues behind for every later node. PREFETCH_QUEUES = [] + first_error = None + try: + cleanup_malloc_graph() + except Exception as e: + first_error = e + for queue in queues: + # Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later + # entries, and a head that was not reached yet, still hold the raw module or module group. + head = queue[0] if queue else None + if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple): + continue + prefetched_module, comfy_modules = head[1] + try: + cleanup_prefetched_modules(prefetched_module, comfy_modules) + except Exception as e: + # Keep unpinning the other queues; report the first failure once they are done. + if first_error is None: + first_error = e GRAPH_WARMED_MODULES.clear() if MALLOC_GRAPH_USED: logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES) MALLOC_GRAPH_BREAKS = 0 MALLOC_GRAPH_ROGUES = 0 MALLOC_GRAPH_USED = False + if first_error is not None: + raise first_error def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None): malloc_graph = MALLOC_GRAPHS.get(threading.get_ident()) @@ -197,14 +245,16 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap malloc_graph.resume() GRAPH_CAPTURE_STREAMS[device] = capture_stream - signature = None + prefetch = queue[1] + prefetch_modules = prefetch if isinstance(prefetch, (list, tuple)) else (prefetch,) + blocks = [root._v_block for root in prefetch_modules if getattr(root, "_v_block", None) is not None] if enable_graph else [] + signatures = None graph_hit = False graph = getattr(module, "_comfy_graph", None) if enable_graph else None if graph is not None: - signature = comfy_aimdo.model_vbar.vbar_fault(module._v_block) - if signature is not None: - module._v_block_faulted = True - graph_hit = comfy_aimdo.model_vbar.vbar_signature_compare(signature, graph["signature"]) + signatures = fault_graph_blocks(module, blocks) + if signatures is not None: + graph_hit = all(comfy_aimdo.model_vbar.vbar_signature_compare(a, b) for a, b in zip(signatures, graph["signatures"])) if malloc_graph is not None and malloc_scope is not None: if malloc_graph.iterate(malloc_scope if module is not None and not graph_hit else None): @@ -225,10 +275,8 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap return fully_faulted = False - prefetch = queue[0] if prefetch is not None: comfy_modules = [] - prefetch_modules = prefetch if isinstance(prefetch, (list, tuple)) else (prefetch,) for root in prefetch_modules: for s in root.modules(): if hasattr(s, "_v"): @@ -241,11 +289,9 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap if core is not None: if enable_graph and fully_faulted and module in GRAPH_WARMED_MODULES: - if signature is None: - signature = comfy_aimdo.model_vbar.vbar_fault(module._v_block) - if signature is not None: - module._v_block_faulted = True - if signature is not None: + if signatures is None: + signatures = fault_graph_blocks(module, blocks) + if signatures is not None: _drop_graph(module) malloc_graph.pause() graph = torch.cuda.CUDAGraph() @@ -264,7 +310,7 @@ def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_grap malloc_graph.resume(sync=True) comfy.model_management.current_stream(device).wait_stream(capture_stream) graph.replay() - module._comfy_graph = {"graph": graph, "signature": signature} + module._comfy_graph = {"graph": graph, "signatures": signatures} malloc_graph._comfy_cuda_graph_modules.add(module) return if capture_stream is None: diff --git a/comfy/model_state.py b/comfy/model_state.py new file mode 100644 index 00000000000..8b9416257c3 --- /dev/null +++ b/comfy/model_state.py @@ -0,0 +1,103 @@ +import copy +import json +import os + +import safetensors +import safetensors.torch +import torch + +import comfy.memory_management +import comfy.model_management +import comfy.model_patcher +import comfy.model_prefetch +import comfy.storage +import comfy.utils + +METADATA_KEY = "comfy_model_state" + + +class ModelState: + """A writable model's values as CPU tensors with its generation metadata, passed between nodes by value. + + Loading copies the values into the model's own backing, so a state's tensors + are never written in place and copies share them. Flushing reads the loaded + model's values back and drops it; a node that loads a state flushes or + releases it before the state reaches another node. + """ + + def __init__(self, tensors=None, metadata=None): + self.tensors = {} if tensors is None else tensors + self.metadata = {} if metadata is None else metadata + self.patcher = None + + @classmethod + def open(cls, path, create="if_missing"): + """Read the state file at path; a missing file is an empty state unless create is never, and always starts empty.""" + if create != "always" and os.path.exists(path): + with safetensors.safe_open(path, "pt") as file: + metadata = file.metadata() or {} + if METADATA_KEY not in metadata: + raise ValueError(f"Not a model state file: {path}") + return cls({name: file.get_tensor(name).clone() for name in file.keys()}, json.loads(metadata[METADATA_KEY])) + if create == "never": + raise FileNotFoundError(f"Model state file does not exist: {path}") + return cls() + + def save(self, path): + if self.tensors is None: + raise RuntimeError("The model state was released without being written back; rerun the node that produced it") + safetensors.torch.save_file(self.tensors, path, {METADATA_KEY: json.dumps(self.metadata)}) + + def copy(self): + if self.tensors is None: + raise RuntimeError("The model state was released without being written back; rerun the node that produced it") + return ModelState(dict(self.tensors), copy.deepcopy(self.metadata)) + + def load(self, model, device): + """Load model with this state's values, or zeros, and return it ready to run.""" + if comfy.memory_management.aimdo_enabled and not comfy.model_management.is_device_cpu(device): + # Dynamic VRAM pages the model from an anonymous file and writes it back there. + tensors, _ = comfy.utils.create_safetensors(model.state_dict()) + for name, value in self.tensors.items(): + comfy.memory_management.write_tensor_file_slice(tensors[name], value) + model.load_state_dict(tensors, assign=True) + self.patcher = comfy.model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu"), fast_disk=comfy.storage.anonymous_fast_disk()) + else: + model.to_empty(device="cpu") + for name, value in model.state_dict().items(): + if self.tensors: + value.copy_(self.tensors[name]) + else: + value.zero_() + self.patcher = comfy.model_patcher.ModelPatcher(model, device, torch.device("cpu")) + comfy.model_management.load_models_gpu([self.patcher], force_full_load=not self.patcher.is_dynamic()) + if self.patcher.is_dynamic(): + self.patcher.set_in_use_by_current_prompt(True) + return model + + def flush(self): + """Take the loaded model's values and drop it.""" + if self.patcher is None: + return + # The last forward may still be using the allocation graph's pages. + comfy.model_management.synchronize() + comfy.model_prefetch.cleanup_prefetch_queues() + state_dict = self.patcher.model.state_dict() + if self.patcher.is_dynamic(): + self.patcher.flush_writes() + self.tensors = {name: torch.empty(value.shape, dtype=value.dtype) for name, value in state_dict.items()} + for name, value in state_dict.items(): + if not comfy.memory_management.read_tensor_file_slice_into(value, self.tensors[name]): + raise RuntimeError(f"Could not read model state tensor {name}") + else: + self.tensors = {name: value.cpu() for name, value in state_dict.items()} + self.patcher = None + + def release(self): + """Drop a still loaded model, discarding its values; the metadata may already describe them, so the state is unusable.""" + if self.patcher is None: + return + if self.patcher.is_dynamic(): + self.patcher.discard_writes() + self.patcher = None + self.tensors = None diff --git a/comfy/ops.py b/comfy/ops.py index 820e0a37d76..58a88f486c8 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -1130,7 +1130,7 @@ def linear_input_act_(x, weight, bias, input_act=None, act_weight=None, act_eps= # Grouped-scale integer weights that run through the shared INT8 GEMM (comfy_kitchen # AsymW4A8Int8Layout): format -> code width, which also fixes the packed weight width K*bits/8. -_GROUPED_INT8_FORMATS = {"asym_w4a8_int8": 4, "w6a8_int8": 6} +_GROUPED_INT8_FORMATS = {"w3a8_int8": 3, "asym_w4a8_int8": 4, "w5a8_int8": 5, "w6a8_int8": 6} class QuantLinearFunc(torch.autograd.Function): diff --git a/comfy/quant_ops.py b/comfy/quant_ops.py index 0b0f4f86369..6f5e58a878f 100644 --- a/comfy/quant_ops.py +++ b/comfy/quant_ops.py @@ -264,6 +264,11 @@ class TensorCoreFP8E5M2Layout(_TensorCoreFP8LayoutBase): "quantize_input": False, } +# 3-bit (8-entry codebook, [N, 3K/8]) and 5-bit (uniform, [N, 5K/8]) widths of the same layout; +# need comfy_kitchen with 3/5-bit support (comfy-kitchen#303). +for _fmt in ("w3a8_int8", "w5a8_int8"): + QUANT_ALGOS[_fmt] = dict(QUANT_ALGOS["w6a8_int8"]) + # ============================================================================== # Re-exports for backward compatibility diff --git a/comfy/samplers.py b/comfy/samplers.py index a7cc3419e32..b1d265bc8cd 100755 --- a/comfy/samplers.py +++ b/comfy/samplers.py @@ -15,6 +15,7 @@ import logging import comfy.sampler_helpers import comfy.model_patcher +import comfy.model_state import comfy.patcher_extension import comfy.hooks import comfy.context_windows @@ -1234,8 +1235,14 @@ def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mas sampler, comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.SAMPLER_SAMPLE, extra_args["model_options"], is_model_options=True) ) - with self.model_patcher.writable_models(self.conds, noise.shape, self.model_options): - samples = executor.execute(self, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar) + for patcher in [self.model_patcher] + self.model_patcher.get_additional_models_with_key("multigpu"): + model = patcher.model.get_writable_model(self.conds, noise.shape, self.model_options, patcher.load_device) + if model is not None: + with comfy.model_management.cuda_device_context(patcher.load_device): + state = comfy.model_state.ModelState() + state.load(model, patcher.load_device) + comfy.model_management.register_ephemeral_model(state) + samples = executor.execute(self, sigmas, extra_args, callback, noise, latent_image, denoise_mask, disable_pbar) return self.inner_model.process_latent_out(samples.to(torch.float32)) def outer_sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None, latent_shapes=None): diff --git a/comfy/sd.py b/comfy/sd.py index 5069b463dee..d3ea95672f5 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -97,6 +97,7 @@ import comfy.text_encoders.fibo2 import comfy.model_patcher +import comfy.model_state import comfy.lora import comfy.lora_convert import comfy.hooks @@ -476,7 +477,8 @@ def load_model(self, tokens={}): def get_key_patches(self): return self.patcher.get_key_patches() - def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True): + def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True, state=None): + """Generate tokens into a model state, continuing the generation it holds. The state stays loaded until flushed, or released after the node.""" self.cond_stage_model.reset_clip_options() self.load_model(tokens) @@ -484,7 +486,11 @@ def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_ self.cond_stage_model.set_clip_options({"layer": None}) self.cond_stage_model.set_clip_options({"execution_device": device}) + if state is None: + state = comfy.model_state.ModelState() + model_management.register_ephemeral_model(state) with model_management.cuda_device_context(device), comfy.ops.use_quantized_matmul(self.cond_stage_model, device): + self.cond_stage_model.set_clip_options({"model_state": state}) return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty, mtp=mtp) def decode(self, token_ids, skip_special_tokens=True): diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.py index 39280bb5e5a..f842de0b947 100644 --- a/comfy/sd1_clip.py +++ b/comfy/sd1_clip.py @@ -136,6 +136,7 @@ def __init__(self, device="cpu", max_length=77, self.return_projected_pooled = return_projected_pooled self.return_attention_masks = return_attention_masks self.execution_device = None + self.model_state = None if layer == "hidden": assert layer_idx is not None @@ -153,6 +154,7 @@ def set_clip_options(self, options): layer_idx = options.get("layer", self.layer_idx) self.return_projected_pooled = options.get("projected_pooled", self.return_projected_pooled) self.execution_device = options.get("execution_device", self.execution_device) + self.model_state = options.get("model_state", self.model_state) if isinstance(self.layer, list) or self.layer == "all": pass elif isinstance(layer_idx, list): @@ -168,6 +170,7 @@ def reset_clip_options(self): self.layer_idx = self.options_default[1] self.return_projected_pooled = self.options_default[2] self.execution_device = None + self.model_state = None def process_tokens(self, tokens, device): end_token = self.special_tokens.get("end", None) diff --git a/comfy/storage.py b/comfy/storage.py index 2f40250e5c4..cba18697e1a 100644 --- a/comfy/storage.py +++ b/comfy/storage.py @@ -3,6 +3,7 @@ import os import platform import re +import tempfile import comfy_aimdo.storage from comfy.cli_args import args @@ -80,6 +81,14 @@ def fast_storage(path): return None +@functools.lru_cache(maxsize=None) +def anonymous_fast_disk(): + # Linux anonymous files are memfds and already live in RAM. + if os.name != "nt": + return not args.disable_fast_disk + return model_fast_disk([tempfile.gettempdir()]) + + def annotate_state_dict(state_dict, path): path = os.path.realpath(path) for value in state_dict.values(): diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index d84d7b7cd11..e2e8ed016af 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -1,7 +1,8 @@ import torch import torch.nn as nn -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, Any, Tuple +import contextlib import math from tqdm import tqdm import comfy.utils @@ -28,6 +29,33 @@ class FixedKV: index: int position: torch.Tensor seqlen: torch.Tensor + storage: Any = field(default=None, init=False) + + def state_tensors(self): + return {"key": (self, "key"), "value": (self, "value")} + + def checkpoint_state(self): + return {"index": self.index} + + def restore_state(self, state): + self.index = state["index"] + + @contextlib.contextmanager + def cast(self, device): + if self.storage is None: + yield + return + fields = self.state_tensors() + originals = {key: getattr(owner, name) for key, (owner, name) in fields.items()} + with contextlib.ExitStack() as stack: + try: + for key, (owner, name) in fields.items(): + value, _ = stack.enter_context(self.storage[key].access(device)) + setattr(owner, name, value) + yield + finally: + for key, (owner, name) in fields.items(): + setattr(owner, name, originals[key]) def prepare(self, num_tokens): self.position.copy_(self.seqlen) @@ -44,6 +72,10 @@ def rollback(self, discard=1): class FixedKVState(FixedKV): tracker: dict = None + def restore_state(self, state): + super().restore_state(state) + self.tracker["step"] = None + def prepare(self, num_tokens): tree = self.tracker["tree"] if self.tracker["step"] == (self.index, num_tokens, tree): @@ -112,6 +144,9 @@ def mask(self, seq): class Int8FixedKVCache(FixedKVState): # INT8 prefix with a bounded BF16 window for page updates and speculative commits. + def state_tensors(self): + return {**super().state_tensors(), **{"packed_" + name: (self.packed, name) for name in ("key", "value", "key_scale", "value_scale")}} + @classmethod def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared): packed = Int8DecodeCache(batch, kv_heads, capacity, device) @@ -120,7 +155,7 @@ def zeros(cls, batch, kv_heads, capacity, head_dim, device, dtype, shared): key = torch.zeros((batch, kv_heads, window, head_dim), device=device, dtype=dtype) cache = cls(key, torch.zeros_like(key), 0, *shared) cache.packed = packed - cache.chain_mask = torch.tensor([(1 << (r + 1)) - 1 for r in range(SPEC_ROWS)], dtype=torch.int32, device=device) + cache.chain_mask = torch.tensor([(1 << (r + 1)) - 1 for r in range(SPEC_ROWS)], dtype=torch.int32, device=cache.position.device) return cache def append(self, xk, xv): @@ -148,6 +183,88 @@ def attend(self, xq, xk, xv): mask = tree.mask if tree is not None else self.chain_mask[:seq] return comfy_kitchen.flash_attention_decode_tree_merge(out, lse, xq, xk, xv, mask, torch.empty_like(out)) + def extend(self, xq, xk, xv): + # A prefill continuing a resumed prefix: the new rows attend the quantized prefix through the + # decode kernel and each other causally, then the quantized pages grow one page per update. + start, seq = self.index, xk.shape[2] + length = torch.full_like(self.position[:1], start) + self.packed.update(self.key, self.value, length) + step = max(1, 64 // (xq.shape[1] // xk.shape[1])) # query rows per call, no more than a verify step uses + out, lse = None, None + for r in range(0, seq, step): + o, s = self.packed.attend(xq[:, :, r:r + step].contiguous(), length) + if out is None: + out = o.new_empty((o.shape[0], seq, o.shape[2])) + lse = s.new_empty((s.shape[0], s.shape[1], seq)) + out[:, r:r + step] = o + lse[:, :, r:r + step] = s + out = causal_merge(out, lse, xq, xk, xv) + window, page = self.key.shape[2], self.packed.key.shape[3] + pos = start + while pos < start + seq: + end = min(start + seq, (pos // page + 1) * page) + slots = torch.arange(pos, end, device=xk.device).remainder(window) + self.key.index_copy_(2, slots, xk[:, :, pos - start:end - start]) + self.value.index_copy_(2, slots, xv[:, :, pos - start:end - start]) + self.packed.update(self.key, self.value, torch.full_like(length, end)) + pos = end + return out + + +CAUSAL_MERGE_ELEMENTS = 1 << 26 # float32 scores per chunk (256 MB) + + +def causal_merge(out, lse, q, k, v): + """Fold rows' causal attention over each other into their attention over a prefix. + + out [B, S, H*D] and lse [B, H, S] are every row's attention over the prefix; q [B, H, S, D] and + k/v [B, Hk, S, D] the rows themselves. Row i also attends rows <= i. Returns [B, S, H*D]. + The query heads sharing a key/value head go through one matmul, so k and v are never repeated.""" + batch, heads, rows, dim = q.shape + kv_heads = k.shape[1] + groups = heads // kv_heads + kf, vf = k.float(), v.float() + merged = torch.empty_like(out) + chunk = max(1, min(rows, CAUSAL_MERGE_ELEMENTS // (heads * rows))) + for r in range(0, rows, chunk): + n = min(chunk, rows - r) + qc = q[:, :, r:r + n].float().reshape(batch, kv_heads, groups * n, dim) + scores = torch.matmul(qc, kf[:, :, :r + n].transpose(-1, -2)).mul_(dim ** -0.5).view(batch, heads, n, r + n) + scores.masked_fill_(torch.ones(n, r + n, dtype=torch.bool, device=q.device).triu_(r + 1), float("-inf")) + prefix_lse = lse[:, :, r:r + n] + m = torch.maximum(prefix_lse, scores.amax(-1)) + w_prefix = torch.exp(prefix_lse - m) + w = scores.sub_(m.unsqueeze(-1)).exp_() + acc = torch.matmul(w.view(batch, kv_heads, groups * n, r + n), vf[:, :, :r + n]).view(batch, heads, n, dim) + acc.add_(out[:, r:r + n].view(batch, n, heads, dim).transpose(1, 2).float() * w_prefix.unsqueeze(-1)) + merged[:, r:r + n] = (acc / (w_prefix + w.sum(-1)).unsqueeze(-1)).transpose(1, 2).reshape(batch, n, heads * dim).to(out.dtype) + return merged + + +class KVCacheModel(nn.Module): + comfy_writable = True + + def __init__(self, caches, extra_caches=()): + super().__init__() + self.caches = caches + self.extra_caches = list(extra_caches) + self.layers = nn.ModuleList() + for cache in self.caches + self.extra_caches: + layer = nn.ModuleDict({key: comfy.ops.WritableWeight(getattr(owner, name).shape, getattr(owner, name).dtype) + for key, (owner, name) in cache.state_tensors().items()}) + self.layers.append(layer) + cache.storage = layer + + def get_dynamic_vram__units(self): + return self.layers, [] + + def checkpoint_state(self): + return [cache.checkpoint_state() for cache in self.caches + self.extra_caches] + + def restore_state(self, state): + for cache, saved in zip(self.caches + self.extra_caches, state, strict=True): + cache.restore_state(saved) + @dataclass class Llama2Config: vocab_size: int = 128320 @@ -1016,7 +1133,8 @@ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermed spec_decode = fixed_kv and any(isinstance(kv, FixedKVState) for kv in past_key_values) and 2 <= seq_len <= SPEC_ROWS and past_len > 0 and attention_mask is None if seq_len > 1 and not spec_decode: # spec verify: the staircase decode bias is causal - causal_mask = torch.empty(past_len + seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(1) + # rows are this call's tokens, columns the cached prefix plus them (a prefill continuing a prefix) + causal_mask = torch.empty(seq_len, past_len + seq_len, dtype=x.dtype, device=x.device).fill_(torch.finfo(x.dtype).min / 4).triu_(past_len + 1) if mask is not None: mask += causal_mask else: @@ -1048,7 +1166,8 @@ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermed elif intermediate_output < 0: intermediate_output = len(self.layers) + intermediate_output - prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), x.device, {"prefetch_dynamic_vbars": self.prefetch_dynamic_vbars and past_key_values is not None}) + prefetch_modules = [(layer, kv.storage) if kv.storage is not None else layer for layer, kv in zip(self.layers, past_key_values)] if fixed_kv else list(self.layers) + prefetch_queue = comfy.model_prefetch.make_prefetch_queue(prefetch_modules, x.device, {"prefetch_dynamic_vbars": self.prefetch_dynamic_vbars and past_key_values is not None}) next_key_values = list(past_key_values) if past_key_values is not None else [] for i, layer in enumerate(self.layers): if all_intermediate is not None: @@ -1067,13 +1186,14 @@ def core(): # Plain FixedKV shares GPU counters; prepare once, eager or graph-replayed. if i == 0 and type(past_kv) is FixedKV: past_kv.prepare(seq_len) - output, current_kv = layer( - x=x, - attention_mask=mask, - freqs_cis=freqs_cis, - optimized_attention=optimized_attention, - past_key_value=past_kv, - ) + with past_kv.cast(x.device) if fixed_kv else contextlib.nullcontext(): + output, current_kv = layer( + x=x, + attention_mask=mask, + freqs_cis=freqs_cis, + optimized_attention=optimized_attention, + past_key_value=past_kv, + ) if enable_graph: x.copy_(output) else: @@ -1191,7 +1311,27 @@ def logits(self, x): def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): return self.model.init_kv_cache(batch, max_cache_len, device, execution_dtype) - def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None, position_ids=None, deepstack_embeds=None, visual_pos_masks=None, embeds_info=None): + def prefill(self, embeds, past_key_values, position_ids=None, **kwargs): + return self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, position_ids=position_ids, **kwargs) + + def cache_capacity(self, needed, state): + """The cache length for a generation that needs this many tokens on top of the state's.""" + return needed + + def restore_generation_state(self, state, cache, generator): + if "caches" in state: + cache.restore_state(state["caches"]) + if generator is not None and "rng" in state: + generator.set_state(torch.tensor(state["rng"], dtype=torch.uint8)) + + def save_generation_state(self, state, cache, ids, generator, capacity, position): + # pending is the last sampled token, fed at position on the next turn + state.update(length=cache.caches[0].index, capacity=capacity, pending=ids[-1:], position=position, caches=cache.checkpoint_state(), + penalty_tokens=sorted(set(state.get("penalty_tokens", []) + ids))) + if generator is not None: + state["rng"] = generator.get_state().tolist() + + def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, initial_tokens=[], execution_dtype=None, min_tokens=0, presence_penalty=0.0, initial_input_ids=None, position_ids=None, deepstack_embeds=None, visual_pos_masks=None, embeds_info=None, model_state=None): device = embeds.device if stop_tokens is None: @@ -1207,10 +1347,15 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, if embeds.ndim == 2: embeds = embeds.unsqueeze(0) - max_cache_len = embeds.shape[1] + max_length - past_key_values = self.init_kv_cache(embeds.shape[0], max_cache_len, device, execution_dtype) - + state = {} if model_state is None else model_state.metadata + max_cache_len = self.cache_capacity(state.get("length", 0) + embeds.shape[1] + max_length, state) + initial_tokens = state.get("penalty_tokens", initial_tokens) generator = torch.Generator(device=device).manual_seed(seed) if do_sample else None + cache = past_key_values = self.init_kv_cache(embeds.shape[0], max_cache_len, device, execution_dtype) + if isinstance(cache, KVCacheModel): + cache = model_state.load(cache, device) + self.restore_generation_state(state, cache, generator) + past_key_values = cache.caches generated_token_ids = [] pbar = comfy.utils.ProgressBar(max_length) @@ -1256,7 +1401,10 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, if step == 0 and deepstack_embeds is not None: extra["deepstack_embeds"] = deepstack_embeds extra["visual_pos_masks"] = visual_pos_masks - x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids, position_ids=position_ids, **extra, embeds_info=(embeds_info if step == 0 else None)) + if step == 0: + x, _, past_key_values = self.prefill(embeds, past_key_values, position_ids=position_ids, input_ids=current_input_ids, **extra, embeds_info=embeds_info) + else: + x, _, past_key_values = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=past_key_values, input_ids=current_input_ids, position_ids=position_ids, **extra, embeds_info=None) logits = self.logits(x)[:, -1] if penalty_mask is None and do_sample and penalize: # allocated on the (unbracketed) first step; later steps only index_fill_ it @@ -1290,6 +1438,8 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, finally: comfy.model_prefetch.cleanup_prefetch_queues() + if isinstance(cache, KVCacheModel): + self.save_generation_state(state, cache, generated_token_ids, generator, max_cache_len, cache.caches[0].index if next_pos is None else next_pos) return generated_token_ids def processed_probs(self, logits, temperature, top_k, top_p, min_p, repetition_penalty, token_history, presence_penalty=0.0, penalty_mask=None): diff --git a/comfy/text_encoders/llm_state_cache.py b/comfy/text_encoders/llm_state_cache.py new file mode 100644 index 00000000000..77ac79cb918 --- /dev/null +++ b/comfy/text_encoders/llm_state_cache.py @@ -0,0 +1,522 @@ +"""Reuse a hybrid (attention + DeltaNet) LLM's prompt state across TextGenerate calls, kept in model +states (comfy.model_state) so the KV lives in dynamic VRAM, spills to RAM when another model needs the card, and +streams back per layer. + +The cache keeps a few resident states, one per conversation. A new prompt takes the state with the longest common +token prefix, resumes from the latest DeltaNet checkpoint inside it (a recurrent state can't be truncated; the +attention KV is truncated by moving its index back), and prefills the rest in batched segments that end on message +boundaries, where it takes the next checkpoints. Image positions get ids from a hash of the image, so image prompts +match too. + +Memory: states stay loaded between calls (nothing is written per call). Past COMFY_LLM_STATE_CACHE_GB of states, or +COMFY_LLM_STATE_CACHE_ENTRIES of them, the least recently used is saved to a file in COMFY_LLM_STATE_CACHE_DIR and +dropped; its next match loads the file. The shared start (the prompt up to the end of the system message) is saved +once to a file named by the hash of its tokens, if no such file exists, and a new conversation with no resident match +starts from it. + +COMFY_LLM_PREFIX_CACHE: off (default) | on. +With the int8 KV cache, a state resumes only where its bf16 window still holds the two pages before the resume point. +""" +import contextlib +import hashlib +import json +import logging +import os +import time + +import safetensors +import safetensors.torch +import torch + +MODE = os.environ.get("COMFY_LLM_PREFIX_CACHE", "off") +CACHE_DIR = os.environ.get("COMFY_LLM_STATE_CACHE_DIR", "/tmp/llm_state_cache") +BUDGET = int(float(os.environ.get("COMFY_LLM_STATE_CACHE_GB", "16")) * 2**30) +MAX_ENTRIES = int(os.environ.get("COMFY_LLM_STATE_CACHE_ENTRIES", "4")) +HEADROOM = int(os.environ.get("COMFY_LLM_PREFIX_CACHE_HEADROOM", "8192")) # capacity beyond this call, for later turns +CKPT_RECENT = int(os.environ.get("COMFY_LLM_PREFIX_CACHE_CKPTS", "6")) # message-boundary checkpoints kept +MSG_END = (248046, 198) # "<|im_end|>\n" in the Qwen3.5 vocab: a checkpoint may sit after it +MIN_SUFFIX = 9 # suffixes of llama.SPEC_ROWS (8) or fewer tokens would take the decode/verify paths +ROUND = 4096 + +stats = {} +_entries = [] # resident and spilled states, most recently used last +_tick = [0] + + +_call = [None] # a per-call mode set by the TextGenerate node, else MODE + + +def enabled(): + return (_call[0] or MODE) == "on" + + +@contextlib.contextmanager +def call_mode(mode): + _call[0] = mode + try: + yield + finally: + _call[0] = None + + +class _Entry: + """One conversation's state: a loaded ModelState (or a file it was spilled to), the tokens its KV holds, and DeltaNet + checkpoints {position: {layer: (conv, recurrent)}} on the CPU.""" + + def __init__(self, key, state, capacity): + self.key = key + self.state = state # comfy.model_state.ModelState, loaded; None once spilled + self.capacity = capacity + self.ids = [] + self.checkpoints = {} + self.file = None + self.used = 0 + self.window = _int8_window(state.patcher.model) if state is not None else None # (rows, page) for an int8 cache + self.written = 0 # rows written past the committed ones (draft rows included) bound what the window still holds + + def nbytes(self): + n = sum(c.nbytes + r.nbytes for st in self.checkpoints.values() for c, r in st.values()) + if self.state is not None and self.state.patcher is not None: + n += sum(t.nbytes for t in self.state.patcher.model.state_dict().values()) + return n + + +def _model_key(model, dtype, device): + from .qwen35 import GatedAttention + attn = next(m for m in model.modules() if isinstance(m, GatedAttention)) + return f"{getattr(model, 'model_type', type(model).__name__)}|{dtype}|{attn.kv_cache_type(device, dtype).__name__}" + + +def _int8_window(kvm): + from .llama import Int8FixedKVCache + kv = next((kv for kv in kvm.caches if isinstance(kv, Int8FixedKVCache)), None) + return None if kv is None else (kv.key.shape[2], kv.packed.key.shape[3]) + + +def _resumable(entry, pos): + """An int8 cache refreshes its pages from the bf16 window: the two pages before pos must still be in it.""" + if entry.window is None: + return True + rows, page = entry.window + return entry.written - rows <= max(0, (pos - 1) // page * page - page) + + +def _lcp(a, b): + n = 0 + for x, y in zip(a, b): + if x != y: + break + n += 1 + return n + + +def _best(key, ids): + """(entry, checkpoint position) with the most reusable tokens, or (None, 0).""" + best, best_pos = None, 0 + limit_all = len(ids) - MIN_SUFFIX + for e in _entries: + if e.key != key: + continue + lcp = _lcp(ids, e.ids) + usable = [p for p in e.checkpoints if p <= min(lcp, limit_all) and _resumable(e, p)] + if usable and max(usable) > best_pos: + best, best_pos = e, max(usable) + return best, best_pos + + +@contextlib.contextmanager +def _cast(model, kvs, device): + """Device views of these caches' tensors outside a forward (their storage streams through a prefetch queue).""" + import comfy.model_prefetch + kvs = [kv for kv in kvs if kv.storage is not None] + if not kvs: + yield + return + storages = tuple(kv.storage for kv in kvs) + queue = comfy.model_prefetch.make_prefetch_queue([storages], device, {"prefetch_dynamic_vbars": model.model.prefetch_dynamic_vbars}) + with contextlib.ExitStack() as casts: + comfy.model_prefetch.prefetch_queue_pop(queue, device, storages, core=lambda: [casts.enter_context(kv.cast(device)) for kv in kvs]) + yield + comfy.model_prefetch.prefetch_queue_pop(queue, device, None) + + +def _linear(cache): + from .qwen35 import LinearKV + return {i: kv for i, kv in enumerate(cache.caches) if isinstance(kv, LinearKV)} + + +def _snapshot(model, cache, device): + lin = _linear(cache) + with _cast(model, lin.values(), device): + out = {i: (kv.conv_state.to("cpu", copy=True), kv.recurrent_state.to("cpu", copy=True)) for i, kv in lin.items()} + return out + + +def _restore_linear(model, cache, states, device): + lin = _linear(cache) + with _cast(model, lin.values(), device): + for i, (c, r) in states.items(): + lin[i].conv_state.copy_(c) + lin[i].recurrent_state.copy_(r) + + +def _set_index(cache, n): + """Every layer at index n with no step in flight: slots from n on are masked by the next prepare().""" + from .llama import FixedKVState + from .qwen35 import LinearKV + for kv in cache.caches: + kv.index = n + if isinstance(kv, FixedKVState): + kv.tracker["step"] = -1 + kv.tracker["tree"] = None + elif isinstance(kv, LinearKV): + kv.uncommitted = 0 + kv.ctl_tracker.update(step=None, path=None, tree=None) + for kv in cache.extra_caches: # the MTP layer's cache starts empty on every call, as on a fresh generate + kv.index = 0 + if isinstance(kv, FixedKVState): + kv.tracker["step"] = -1 + kv.tracker["tree"] = None + + +def _capacity(need): + return (need + HEADROOM + ROUND - 1) // ROUND * ROUND + + +def _padded(tensors, model_sd): + """CPU tensors shaped like model_sd: equal shapes pass through, attention caches (capacity on dim 2, int8 pages on + dim 1) are copied into zeros of the new capacity, cut to it if longer.""" + out = {} + for name, like in model_sd.items(): + t = tensors.get(name) + if t is None: + out[name] = torch.zeros(like.shape, dtype=like.dtype) + elif tuple(t.shape) == tuple(like.shape): + out[name] = t + else: + z = torch.zeros(like.shape, dtype=like.dtype) + common = tuple(slice(0, min(a, b)) for a, b in zip(t.shape, like.shape)) + z[common] = t[common] + out[name] = z + return out + + +def _in_use(patcher, flag): + # only a dynamic-VRAM patcher pins for the current prompt (none on a CPU device) + if patcher.is_dynamic(): + patcher.set_in_use_by_current_prompt(flag) + + +def _load(model, capacity, device, dtype, tensors=None): + """A loaded ModelState holding a fresh KV cache model of this capacity, filled from tensors (CPU, any capacity).""" + import comfy.model_state + kvm = model.init_kv_cache(1, capacity, device, dtype, mtp=True) + state = comfy.model_state.ModelState() + if tensors: + state.tensors = _padded(tensors, kvm.state_dict()) + state.load(kvm, device) + _in_use(state.patcher, True) + return state, kvm + + +def _activate(entry, device): + import comfy.model_management + comfy.model_management.load_models_gpu([entry.state.patcher]) + _in_use(entry.state.patcher, True) + return entry.state.patcher.model + + +def _attention_layout(kvm): + """{state-dict name: (dim, rows per slot)} of the attention caches, the MTP layer's included: capacity on dim 2 (an + int8 cache's bf16 window too: row r sits in slot r % window), an int8 cache's pages on dim 1.""" + from .qwen35 import LinearKV + out = {} + for i, kv in enumerate(kvm.caches + kvm.extra_caches): + if isinstance(kv, LinearKV): + continue + for key in kv.state_tensors(): + out[f"layers.{i}.{key}.weight"] = (1, kv.packed.key.shape[3]) if key.startswith("packed_") else (2, 1) + return out + + +def _trimmed(t, n, layout): + dim, unit = layout + return t.narrow(dim, 0, min(t.shape[dim], -(-n // unit))).contiguous() + + +def _trimmed_tensors(state_tensors, n, attention): + """A state's CPU tensors with the attention caches cut to the n tokens used (the rest is masked anyway).""" + return {name: _trimmed(t, n, attention[name]) if name in attention else t for name, t in state_tensors.items()} + + +def _file_tensors(entry_tensors, checkpoints): + t = dict(entry_tensors) + for pos, st in checkpoints.items(): + for i, (c, r) in st.items(): + t[f"ckpt.{pos}.{i}.conv"] = c.contiguous() + t[f"ckpt.{pos}.{i}.rec"] = r.contiguous() + return t + + +def _write(path, tensors, meta): + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + f".tmp{os.getpid()}" + safetensors.torch.save_file(tensors, tmp, {"llm_state_cache": json.dumps(meta)}) + os.replace(tmp, path) + + +def _read(path): + with safetensors.safe_open(path, "pt") as f: + meta = json.loads(f.metadata()["llm_state_cache"]) + tensors, cps = {}, {} + for name in f.keys(): + if name.startswith("ckpt."): + _, pos, i, kind = name.split(".") + cps.setdefault(int(pos), {}).setdefault(int(i), [None, None])[0 if kind == "conv" else 1] = f.get_tensor(name) + else: + tensors[name] = f.get_tensor(name) + return tensors, {p: {i: tuple(v) for i, v in st.items()} for p, st in cps.items()}, meta + + +def _ids_json(ids): + return [list(t) if isinstance(t, tuple) else t for t in ids] + + +def _prefix_path(key, ids): + h = hashlib.blake2b(json.dumps([key, _ids_json(ids)]).encode(), digest_size=16).hexdigest() + return os.path.join(CACHE_DIR, f"prefix-{h}.safetensors") + + +def _spill(entry): + """Save a resident entry to a file and free its state; its next match loads the file.""" + t = time.perf_counter() + state = entry.state + attention = _attention_layout(state.patcher.model) + state.flush() + n = len(entry.ids) + path = os.path.join(CACHE_DIR, f"spill-{os.getpid()}-{id(entry)}.safetensors") + _write(path, _file_tensors(_trimmed_tensors(state.tensors, n, attention), entry.checkpoints), + {"key": entry.key, "ids": _ids_json(entry.ids)}) + entry.state = None + entry.file = path + entry.checkpoints = {p: {} for p in entry.checkpoints} # positions kept for matching; the states are in the file + logging.info("llm state cache: spilled %d tokens to disk in %.2f s", n, time.perf_counter() - t) + + +def _drop(entry): + if entry.state is not None: + entry.state.release() + if entry.file is not None: + try: + os.remove(entry.file) + except OSError: + pass + _entries.remove(entry) + + +def _enforce_budget(keep): + resident = [e for e in _entries if e.state is not None] + resident.sort(key=lambda e: e.used) + total = sum(e.nbytes() for e in resident) + for e in resident: + if e is keep: + continue + if total <= BUDGET and len(resident) <= MAX_ENTRIES: + break + total -= e.nbytes() + _spill(e) + resident = [r for r in resident if r is not e] + spilled = [e for e in _entries if e.state is None] + spilled.sort(key=lambda e: e.used) + while len(spilled) > MAX_ENTRIES: # spilled files are bounded too + _drop(spilled.pop(0)) + + +def _take(model, key, ids, need, device, dtype): + """(entry, cache model, prefix length) for this prompt, allocating or loading as needed.""" + entry, n = _best(key, ids) + start = _shared_start(ids) + path = _prefix_path(key, ids[:start]) if start else None + if entry is not None and n <= start < len(entry.ids) and path is not None and os.path.exists(path): + entry, n = None, 0 # another conversation: fork from the shared-start file, keep its state whole + stats["fork"] = True + if entry is not None: + _entries.remove(entry) + if entry.state is None: + tensors, cps, _ = _read(entry.file) + os.remove(entry.file) + entry.file = None + cap = max(_capacity(need), entry.capacity) + entry.state, kvm = _load(model, cap, device, dtype, tensors) + entry.capacity = cap + entry.checkpoints = {p: cps[p] for p in cps if p in entry.checkpoints} + stats["restore"] = "from-disk" + elif entry.capacity < need: + attention = _attention_layout(entry.state.patcher.model) + entry.state.flush() + cap = _capacity(need) + tensors = _trimmed_tensors(entry.state.tensors, n, attention) + entry.state, kvm = _load(model, cap, device, dtype, tensors) + entry.capacity = cap + stats["restore"] = "grow" + else: + kvm = _activate(entry, device) + stats["restore"] = "resident" + _restore_linear(model, kvm, entry.checkpoints[n], device) + _set_index(kvm, n) + entry.checkpoints = {p: st for p, st in entry.checkpoints.items() if p <= n} + entry.ids = list(ids[:n]) + return entry, kvm, n + cap = _capacity(need) + if path is not None and os.path.exists(path): + tensors, cps, _ = _read(path) + state, kvm = _load(model, cap, device, dtype, tensors) + entry = _Entry(key, state, cap) + entry.written = start + entry.checkpoints = {start: cps[start]} + _restore_linear(model, kvm, cps[start], device) + _set_index(kvm, start) + entry.ids = list(ids[:start]) + stats["restore"] = "shared-start-file" + return entry, kvm, start + state, kvm = _load(model, cap, device, dtype) + return _Entry(key, state, cap), kvm, 0 + + +def _boundaries(ids): + return [j for j in range(2, len(ids) + 1) if ids[j - 2] == MSG_END[0] and ids[j - 1] == MSG_END[1]] + + +def _shared_start(ids): + """The end of the system message (the first message boundary), where a conversation's shared start ends.""" + ends = _boundaries(ids) + if not ends or ends[0] < MIN_SUFFIX or len(ids) - ends[0] < MIN_SUFFIX: + return 0 + return ends[0] + + +def _checkpoint_positions(ids, start): + ends = _boundaries(ids) + want = sorted(set(ends[:1] + ends[-CKPT_RECENT:])) if CKPT_RECENT > 0 else [] + out, prev = [], start + for pos in want: + if pos - prev >= MIN_SUFFIX and len(ids) - pos >= MIN_SUFFIX: + out.append(pos) + prev = pos + return out + + +def prompt_ids(tokens, embeds_info, seq_len): + """Cache ids for one prompt, one per embed position: the token ids, and for each image span a key per position + made from a hash of the image. None when the prompt holds anything else (video, audio, raw embeddings) or the + spans don't line up.""" + spans = iter(embeds_info) + out = [] + for t in tokens: + if isinstance(t, int): + out.append(t) + continue + if not isinstance(t, dict) or t.get("type") != "image" or not torch.is_tensor(t.get("data")): + return None + info = next(spans, None) + if info is None or info.get("type") != "image" or info.get("index") != len(out): + return None + digest = _image_digest(t["data"]) + out.extend((digest, k) for k in range(info["size"])) + if next(spans, None) is not None or len(out) != seq_len: + return None + return out + + +def _image_digest(image): + data = image.detach().to("cpu").contiguous() + h = hashlib.blake2b(digest_size=16) + h.update(f"{tuple(data.shape)}|{data.dtype}".encode()) + h.update(data.view(torch.uint8).numpy().tobytes() if data.numel() else b"") + return "img:" + h.hexdigest() + + +_current = {} + + +def prefill(model, embeds, ids, need, forward, position_ids=None): + """Prefill the whole prompt `embeds`, reusing a cached prefix of `ids`. Returns (cache model, x of the last position, capacity). + `forward(embeds, caches, position_ids)` runs one segment.""" + if _current: + abort() # the previous generate failed after its prefill + stats.clear() + device, dtype = embeds.device, embeds.dtype + key = _model_key(model, dtype, device) + entry, kvm, n = _take(model, key, ids, need, device, dtype) + stats.update(prompt_len=len(ids), prefix_len=n) + start, x = n, None + shared = _shared_start(ids) + for pos in _checkpoint_positions(ids, n) + [len(ids)]: + x = forward(embeds[:, start:pos], kvm.caches, None if position_ids is None else position_ids[..., start:pos]) + entry.checkpoints[pos] = _snapshot(model, kvm, device) + if pos == shared and n < shared: + _save_shared_start(model, kvm, key, ids, shared, entry.checkpoints[pos], device) + start = pos + if len(entry.checkpoints) > CKPT_RECENT + 3: + keep = sorted(entry.checkpoints) + keep = keep[:1] + keep[-(CKPT_RECENT + 2):] + entry.checkpoints = {p: entry.checkpoints[p] for p in keep} + entry.ids = list(ids) + entry.written = len(ids) + _current.update(entry=entry, kvm=kvm, device=device, model=model) + return kvm, x, entry.capacity + + +def _save_shared_start(model, kvm, key, ids, pos, linear_states, device): + """Write the shared start (KV up to pos + the DeltaNet states at pos) once, if no file has it.""" + path = _prefix_path(key, ids[:pos]) + if os.path.exists(path): + return + linear = _linear(kvm) + attn = {i: kv for i, kv in enumerate(kvm.caches) if i not in linear} + layout = _attention_layout(kvm) + tensors = {} + with _cast(model, attn.values(), device): + for i, kv in attn.items(): + for key, (owner, field) in kv.state_tensors().items(): + name = f"layers.{i}.{key}.weight" + tensors[name] = _trimmed(getattr(owner, field), pos, layout[name]).to("cpu", copy=True) + _write(path, _file_tensors(tensors, {pos: linear_states}), {"key": key, "ids": _ids_json(ids[:pos])}) + stats["shared_start_saved"] = True + + +def finish(ids, generated): + """Checkpoint the prompt plus the generated tokens the DeltaNet state has committed, keep the state resident, and + spill others past the budget.""" + from . import llama + entry, kvm, device, model = (_current.get(k) for k in ("entry", "kvm", "device", "model")) + _current.clear() + if entry is None: + return + stats["gen_tokens"] = len(generated) + full = list(ids) + list(generated) + lin = next(iter(_linear(kvm).values())) + # deferred commit: a decode step leaves the DeltaNet state untouched until the next step, so the state lags the + # index by the last step's still-uncommitted tokens + idx = lin.index - lin.uncommitted + if len(ids) < idx <= len(full): + entry.checkpoints[idx] = _snapshot(model, kvm, device) + entry.ids = full[:idx] + entry.written = max(kv.index for kv in kvm.caches) + llama.SPEC_ROWS # a verify step writes up to SPEC_ROWS rows ahead + _in_use(entry.state.patcher, False) + _tick[0] += 1 + entry.used = _tick[0] + _entries.append(entry) + _enforce_budget(entry) + stats["entries"] = len(_entries) + stats["resident"] = sum(1 for e in _entries if e.state is not None) + stats["cache_gb"] = round(sum(e.nbytes() for e in _entries if e.state is not None) / 1e9, 3) + logging.info("LLM_PREFIX_CACHE %s", json.dumps(stats)) + + +def abort(): + """A generate failed after prefill: drop the half-written state.""" + entry = _current.get("entry") + _current.clear() + if entry is not None and entry.state is not None: + entry.state.release() diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index 4719c52dd2d..e3dbc523123 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -16,8 +16,9 @@ from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_for_device from comfy import sd1_clip import comfy.text_encoders.qwen_vl +from . import llm_state_cache -from .llama import SPEC_ROWS, BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix +from .llama import SPEC_ROWS, BaseLlama, BaseGenerate, FixedKV, FixedKVCache, Int8FixedKVCache, KVCacheModel, Llama2_, MLP, RMSNorm, apply_penalty, apply_rope, penalty_active, precompute_freqs_cis, rope_matrix @dataclass @@ -37,6 +38,17 @@ class LinearKV(FixedKV): ctl_tracker: dict = None uncommitted: int = 0 + def state_tensors(self): + return {**super().state_tensors(), **{name: (self, name) for name in ("qkv_buf", "proj_buf", "gates_buf", "sumsq_buf")}} + + def checkpoint_state(self): + # the accepted verify rows the next step still has to commit + return {"index": self.index, "uncommitted": self.uncommitted, "parity": self.ctl_tracker["parity"], "path": self.ctl_tracker["path"]} + + def restore_state(self, state): + self.index, self.uncommitted = state["index"], state["uncommitted"] + self.ctl_tracker.update(parity=state["parity"], path=state["path"], step=None) + @staticmethod def deferred_tracker(device): # ctl = {pending, parity, slot[sm], parent[sm], prog[sm]}; without MTP, only row 0 runs. @@ -369,6 +381,12 @@ def forward(self, x, past_key_value=None, **kwargs): proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim] z = self.in_proj_z(x) + # A restored cache can start on a decode step, before any prefill ran here. + if past_key_value is not None and past_key_value.g_decay is None: + past_key_value.g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp() + past_key_value.dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32) + past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device) + if past_key_value is not None and past_key_value.decode_step(seq_len): # decode: conv step into the side buffer, then gates + delta rule + gated norm in one # kernel; both first commit the previous step's accepted tokens @@ -392,21 +410,25 @@ def forward(self, x, past_key_value=None, **kwargs): a = self.in_proj_a(x) # Conv1d - if past_key_value is not None: - conv_state = past_key_value.conv_state - conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0)) - conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:]) - mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len]) + continuing = past_key_value is not None and past_key_value.index > 0 + if continuing: + # a prefill continuing a resumed state: the conv window starts from the cached one + window = torch.cat([past_key_value.conv_state, mixed_qkv], dim=-1) + past_key_value.conv_state.copy_(window[:, :, -past_key_value.conv_state.shape[-1]:]) + with comfy.ops.CastBiasWeightContext(self.conv1d, window, offloadable=True) as (conv_weight, conv_bias): + mixed_qkv = F.silu(F.conv1d(window, conv_weight, conv_bias, groups=self.conv1d.groups)) + else: + if past_key_value is not None: + conv_state = past_key_value.conv_state + conv_state_init = F.pad(mixed_qkv, (self.conv_kernel_size - mixed_qkv.shape[-1], 0)) + conv_state.copy_(conv_state_init[:, :, -conv_state.shape[-1]:]) + mixed_qkv = F.silu(self.conv1d(mixed_qkv)[:, :, :seq_len]) # Split QKV and compute beta/g mixed_qkv = mixed_qkv.transpose(1, 2) # [B, seq_len, conv_dim] query, key, value = mixed_qkv.split([self.key_dim, self.key_dim, self.value_dim], dim=-1) - g_decay = -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp() - dt_bias = comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32) - if past_key_value is not None: - past_key_value.g_decay = g_decay - past_key_value.dt_bias = dt_bias - past_key_value.norm_weight = comfy.model_management.cast_to(self.norm.weight, dtype=x.dtype, device=x.device) + g_decay = past_key_value.g_decay if past_key_value is not None else -comfy.model_management.cast_to_device(self.A_log, x.device, torch.float32).exp() + dt_bias = past_key_value.dt_bias if past_key_value is not None else comfy.model_management.cast_to_device(self.dt_bias, x.device, torch.float32) beta = b.sigmoid() g = g_decay * F.softplus(a.float() + dt_bias) @@ -421,7 +443,7 @@ def forward(self, x, past_key_value=None, **kwargs): # Delta rule core_attn_out, last_recurrent_state = torch_chunk_gated_delta_rule( query, key, value, g=g, beta=beta, - initial_state=None, + initial_state=past_key_value.recurrent_state if continuing else None, output_final_state=past_key_value is not None, ) @@ -481,11 +503,11 @@ def kv_cache_type(self, device, dtype): return Int8FixedKVCache return FixedKVCache - def init_kv_cache(self, batch, capacity, device, dtype, shared): + def init_kv_cache(self, batch, capacity, device, dtype, shared, storage_device=None): cache_type = self.kv_cache_type(device, dtype) if cache_type not in shared: shared[cache_type] = cache_type.shared(batch, device) - return cache_type.zeros(batch, self.num_kv_heads, capacity, self.head_dim, device, dtype, shared[cache_type]) + return cache_type.zeros(batch, self.num_kv_heads, capacity, self.head_dim, device if storage_device is None else storage_device, dtype, shared[cache_type]) def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=None, past_key_value=None): batch_size, seq_length, _ = x.shape @@ -516,6 +538,8 @@ def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=No if decode and (isinstance(past_key_value, Int8FixedKVCache) or self.comfy_attention.method is None and not comfy.model_management.comfy_kitchen_attention_enabled()): output = past_key_value.attend(xq, xk, xv) + elif not decode and isinstance(past_key_value, Int8FixedKVCache) and past_key_value.index > 0: + output = past_key_value.extend(xq, xk, xv) # a prefill continuing a resumed state else: if decode: xk, xv = past_key_value.key, past_key_value.value @@ -842,6 +866,11 @@ def forward(self, embeds, hidden, freqs_cis, past_key_value): # Model Wrapper +# Rows per prefill pass: one pass over a long prompt needs enough working memory to page the +# weights out of VRAM, and the decode after it would read them back from system memory. +PREFILL_ROWS = 4096 + + class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module): model_type = "qwen35_2b" @@ -886,6 +915,70 @@ def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermed position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device) return super().forward(x, attention_mask=attention_mask, embeds=embeds, num_tokens=num_tokens, intermediate_output=intermediate_output, final_layer_norm_intermediate=final_layer_norm_intermediate, dtype=dtype, position_ids=position_ids, past_key_values=past_key_values) + def prefill(self, embeds, past_key_values, position_ids=None, **kwargs): + rows = embeds.shape[1] + batched = past_key_values[0].index == 0 or self.batched_continue(rows, past_key_values) + if batched and rows > PREFILL_ROWS and not kwargs: + # a long prompt goes in equal pieces, each continuing the one before + size = -(-rows // -(-rows // PREFILL_ROWS)) + for start in range(0, rows, size): + positions = position_ids[..., start:start + size] if position_ids is not None else None + result = super().prefill(embeds[:, start:start + size], past_key_values, positions) + return result + if batched: + return super().prefill(embeds, past_key_values, position_ids, **kwargs) + # Otherwise the new tokens go through the deferred decode kernels in rows of 1, 4 or 8, + # which also commit any rows the saved state left uncommitted. A multi-row + # chunk is a verify tree that is one chain: each row attends and commits after the + # row before it. + kwargs.pop("decode_buffers", None) + trackers = [kv.ctl_tracker if isinstance(kv, LinearKV) else kv.tracker for kv in past_key_values] + start = 0 + while start < embeds.shape[1]: + size = next(s for s in (SPEC_ROWS, 4, 1) if start + s <= embeds.shape[1]) + spec = [] + for _ in range(size - 1): + spec = [spec] + tree = VerifyTree(spec, size, embeds.device) if size > 1 else None + for tracker in trackers: + tracker["tree"] = tree + positions = position_ids[..., start:start + size] if position_ids is not None else None + result = super().prefill(embeds[:, start:start + size], past_key_values, positions, **kwargs) + start += size + for tracker in trackers: + tracker["tree"] = None + return result + + @staticmethod + def batched_continue(num_tokens, past_key_values): + # A resumed state continues as one batched prefill when no DeltaNet rows wait for a commit; + # up to SPEC_ROWS tokens would take the decode path inside the layers. + return num_tokens > SPEC_ROWS and all(kv.uncommitted == 0 for kv in past_key_values if isinstance(kv, LinearKV)) + + def save_generation_state(self, state, cache, ids, generator, capacity, position): + # The last step's accepted tokens sit in the attention caches but not yet in the DeltaNet state + # (deferred commit). Save the state before them and re-feed them with the pending token, so the + # next turn can continue as one batched prefill. + pkv = cache.caches + replay = next((kv.uncommitted for kv in pkv if isinstance(kv, LinearKV)), 0) + if 0 < replay < len(ids): + for kv in pkv: + kv.index -= replay + if isinstance(kv, LinearKV): + kv.uncommitted = 0 + kv.ctl_tracker["path"] = None + super().save_generation_state(state, cache, ids, generator, capacity, position - replay) + state["pending"] = ids[-1 - replay:] + return + super().save_generation_state(state, cache, ids, generator, capacity, position) + + def cache_capacity(self, needed, state): + # a saved state fixes its capacity when first created, rounded up so later turns fit + capacity = state.get("capacity", (needed + 4095) // 4096 * 4096) + if needed > capacity: + raise ValueError(f"The saved cache holds {capacity} tokens but this generation needs {needed}") + return capacity + def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.9, min_p=0.0, repetition_penalty=1.0, seed=42, stop_tokens=None, **kwargs): mtp = kwargs.pop("mtp", True) if self.mtp is None or not mtp or kwargs.get("initial_input_ids") is not None: @@ -898,9 +991,10 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, "repetition_penalty": repetition_penalty, "presence_penalty": kwargs.get("presence_penalty", 0.0) or 0.0, "seed": seed if seed is not None else 42} - return self._generate_mtp(embeds, max_length, stop_tokens, mtp, sampling=sampling, position_ids=kwargs.get("position_ids")) + return self._generate_mtp(embeds, max_length, stop_tokens, mtp, sampling=sampling, position_ids=kwargs.get("position_ids"), + model_state=kwargs.get("model_state"), cache_ids=kwargs.get("cache_ids")) - def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, position_ids=None): + def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, position_ids=None, model_state=None, cache_ids=None): device = embeds.device cfg = self.model.config if stop_tokens is None: @@ -909,10 +1003,21 @@ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, pos embeds = embeds.to(dt) if embeds.ndim == 2: embeds = embeds.unsqueeze(0) - cap = embeds.shape[1] + max_length + 7 - pkv = self.init_kv_cache(embeds.shape[0], cap, device, dt) + state = model_state.metadata + past_length = state.get("length", 0) + cap = self.cache_capacity(past_length + embeds.shape[1] + max_length + 7, state) + generator = torch.Generator(device=device).manual_seed(sampling["seed"]) if sampling is not None else None + if cache_ids is not None: + # the prompt state cache: a resident state with the longest matching prefix, prefilled to the end of the prompt + cache, x, cap = llm_state_cache.prefill(self, embeds, cache_ids, cap, + lambda e, kv, p: self.prefill(e, kv, p)[0], + position_ids=position_ids) + else: + cache = model_state.load(self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True), device) + self.restore_generation_state(state, cache, generator) + pkv = cache.caches # repair window: drafting ahead plus a near-full rollback - mtp_kv = self.mtp.layers[0].self_attn.init_kv_cache(embeds.shape[0], cap, device, dt, {}) + mtp_kv = cache.extra_caches[0] head = self.model.lm_head if hasattr(self.model, "lm_head") else self.model.embed_tokens def verify_logits(x): @@ -921,13 +1026,10 @@ def verify_logits(x): with comfy.ops.CastBiasWeightContext(head, x, offloadable=True) as (w, _bias): return F.linear(x, w) - generator = None - if sampling is not None: - generator = torch.Generator(device=device).manual_seed(sampling["seed"]) penalized = sampling is not None and penalty_active(sampling["repetition_penalty"], sampling["presence_penalty"]) # after an image, text positions run behind the cache index (MRoPE); decode continues from the last prefill position - rope_offset = 0 if position_ids is None else int(position_ids[:, -1].max()) + 1 - embeds.shape[1] + rope_offset = 0 if position_ids is None else int(position_ids[:, -1].max()) + 1 - past_length - embeds.shape[1] # rope table once per generate, sliced per draft ftab = rope_matrix(self.model.compute_freqs_cis(torch.arange(rope_offset, cap + rope_offset, device=device, dtype=torch.float).unsqueeze(0), device)) @@ -938,14 +1040,17 @@ def freqs_at(p, n=1): # cross-step state lives in static carriers: nothing allocated inside a step may outlive it nt_buf = torch.empty((embeds.shape[0], 1), device=device, dtype=torch.long) h_buf = torch.empty((embeds.shape[0], 1, cfg.hidden_size), device=device, dtype=dt) - x, _, _ = self.model.forward(None, embeds=embeds, attention_mask=None, past_key_values=pkv, position_ids=position_ids) + if cache_ids is None: + x, _, _ = self.prefill(embeds, pkv, position_ids) lg0 = self.logits(x)[:, -1] + pen_mask = torch.zeros((lg0.shape[-1],), device=device, dtype=torch.bool) if penalized else None + if penalized and "penalty_tokens" in state: + pen_mask.index_fill_(0, torch.tensor(state["penalty_tokens"], device=device, dtype=torch.long), True) if sampling is None: nt_buf.copy_(lg0.argmax(dim=-1, keepdim=True)) else: nt_buf.copy_(self.sample_token(lg0, sampling["temperature"], sampling["top_k"], sampling["top_p"], sampling["min_p"], - sampling["repetition_penalty"], [], generator, presence_penalty=sampling["presence_penalty"])) - pen_mask = torch.zeros((lg0.shape[-1],), device=device, dtype=torch.bool) if penalized else None + sampling["repetition_penalty"], [], generator, presence_penalty=sampling["presence_penalty"], penalty_mask=pen_mask)) h_buf.copy_(x[:, -1:, :]) del x, lg0 if hasattr(self.model.embed_tokens, "_v"): @@ -955,7 +1060,7 @@ def freqs_at(p, n=1): ids = [nt_buf[0].item()] if penalized: pen_mask.index_fill_(0, nt_buf.reshape(-1), True) - pos = embeds.shape[1] + pos = pkv[0].index progress = comfy.utils.ProgressBar(max_length) console = tqdm(total=max_length, desc="Generating tokens", initial=1) @@ -979,6 +1084,8 @@ def update_progress(n): kv.ctl_tracker["tree"] = tree else: kv.tracker["tree"] = tree + # only attention caches move rows on commit; linear caches record the path for the next forward + attention_kv = [kv for kv in pkv if not isinstance(kv, LinearKV)] verify_tokens = torch.zeros((embeds.shape[0], tree.rows), device=device, dtype=torch.long) verify_buffers = (torch.empty((embeds.shape[0], tree.rows, cfg.hidden_size), device=device, dtype=dt), freqs_at(pos, tree.rows).clone()) draft_top = torch.empty((embeds.shape[0], 7 if adaptive else tree.fanout), device=device, dtype=torch.long) @@ -998,14 +1105,15 @@ def draft(token, hidden, p): draft_freqs.copy_(freqs_at(p)) host_rows = hasattr(embed, "_v") and embed.weight_lowvram_function is None and len(embed.weight_function) == 0 and comfy.ops.vbar_above_watermark(embed) queue = comfy.model_prefetch.make_prefetch_queue( - [None if host_rows else embed, (head, self.mtp)], device, + [None if host_rows else embed, [head, self.mtp, mtp_kv.storage]], device, {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars}) def embed_core(): draft_emb.copy_(embed.host_rows(draft_token, out_dtype=dt) if host_rows else embed(draft_token).to(dt)) def core(): - n1, r1 = self.mtp(draft_emb, draft_hidden, draft_freqs, mtp_kv) + with mtp_kv.cast(device): + n1, r1 = self.mtp(draft_emb, draft_hidden, draft_freqs, mtp_kv) draft_hidden.copy_(r1) logits = self.logits(n1)[:, -1] if adaptive: @@ -1080,36 +1188,45 @@ def step(): packet = [toks[0], verify_tokens[0, 1:]] if adaptive: packet.append(draft_ranks.reshape(-1)) - vals = torch.cat(packet).tolist() - t, dr = vals[:tree.rows], [None] + vals[tree.rows:2 * tree.rows - 1] - # Continue only along a child containing the main model's chosen token. - path = [0] - while True: - node = next((c for c in tree.children[path[-1]] if dr[c] == t[path[-1]]), None) - if node is None: - break - path.append(node) - ranks = [] - if adaptive: - for node in path: - if node >= depth: + # the accepted rows may move within the attention caches: pin and cast them while the GPU + # is still verifying, so only the moves themselves run after the accept sync + storages = tuple(kv.storage for kv in attention_kv) + queue = comfy.model_prefetch.make_prefetch_queue([storages], device, + {"prefetch_dynamic_vbars": self.model.prefetch_dynamic_vbars}) + with contextlib.ExitStack() as casts: + comfy.model_prefetch.prefetch_queue_pop(queue, device, storages, + core=lambda: [casts.enter_context(kv.cast(device)) for kv in attention_kv]) + vals = torch.cat(packet).tolist() + t, dr = vals[:tree.rows], [None] + vals[tree.rows:2 * tree.rows - 1] + # Continue only along a child containing the main model's chosen token. + path = [0] + while True: + node = next((c for c in tree.children[path[-1]] if dr[c] == t[path[-1]]), None) + if node is None: break - candidates = vals[15 + 7 * node:22 + 7 * node] - ranks.append(next((r for r, token in enumerate(candidates) if token == t[node]), 7)) - if timers: - clocks[0], clocks[1], clocks[2] = 0.0, timers[0].elapsed_time(timers[1]), timers[0].elapsed_time(timers[2]) - else: - clocks[:] = [t * 1000 for t in clocks] - commit = tuple(dr[r] for r in path[1:]) + (t[path[-1]],) - row = path[-1] - next_tok = toks[:, row:row + 1] - if len(path) < tree.rows: - for kv in pkv: - kv.rollback(tree.rows - len(path)) - on_chain = row == len(path) - 1 - if not on_chain: - for kv in pkv: - kv.commit_path(pos, path) + path.append(node) + ranks = [] + if adaptive: + for node in path: + if node >= depth: + break + candidates = vals[15 + 7 * node:22 + 7 * node] + ranks.append(next((r for r, token in enumerate(candidates) if token == t[node]), 7)) + if timers: + clocks[0], clocks[1], clocks[2] = 0.0, timers[0].elapsed_time(timers[1]), timers[0].elapsed_time(timers[2]) + else: + clocks[:] = [t * 1000 for t in clocks] + commit = tuple(dr[r] for r in path[1:]) + (t[path[-1]],) + row = path[-1] + next_tok = toks[:, row:row + 1] + if len(path) < tree.rows: + for kv in pkv: + kv.rollback(tree.rows - len(path)) + on_chain = row == len(path) - 1 + if not on_chain: + for kv in pkv: + kv.commit_path(pos, path) + comfy.model_prefetch.prefetch_queue_pop(queue, device, None) # mtp entries fed by a rejected or off-chain draft token kept = min(len(path) if on_chain else len(path) - 1, depth) if kept < depth: @@ -1121,12 +1238,12 @@ def step(): pen_mask.index_fill_(0, verify_tokens[:, r], True) pen_mask.index_fill_(0, nt_buf.reshape(-1), True) pos += len(commit) - return commit, ranks + return commit, ranks, kept try: while len(ids) < max_length and ids[-1] not in stop_tokens: with (comfy.model_prefetch.malloc_graph_scope(device) if compile_allocations else contextlib.nullcontext()): - commit, ranks = step() + commit, ranks, kept = step() if adaptive: steps += 1 for row in hits: @@ -1145,18 +1262,30 @@ def step(): widths = best tree.update(tree_spec(widths)) depth = tree.chain + accepted = len(commit) commit = list(commit[:max_length - len(ids)]) stop = next((i for i, t in enumerate(commit) if t in stop_tokens), None) if stop is not None: del commit[stop + 1:] + if accepted > len(commit): + for kv in pkv: + kv.rollback(accepted - len(commit)) + mtp_kv.rollback(max(0, kept - len(commit))) ids.extend(commit) update_progress(len(commit)) finally: console.close() comfy.model_prefetch.cleanup_prefetch_queues() + # A resumed cache decodes as a chain until its next verify installs a tree. + for kv in pkv: + (kv.ctl_tracker if isinstance(kv, LinearKV) else kv.tracker)["tree"] = None + if cache_ids is not None: + llm_state_cache.finish(cache_ids, ids) + return ids + self.save_generation_state(state, cache, ids, generator, cap, pkv[0].index + rope_offset) return ids - def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): + def init_kv_cache(self, batch, max_cache_len, device, execution_dtype, mtp=False): model_config = self.model.config past_key_values = [] shared = {} @@ -1165,21 +1294,22 @@ def init_kv_cache(self, batch, max_cache_len, device, execution_dtype): if model_config.layer_types[i] == "linear_attention": recurrent_state = torch.zeros( [batch, model_config.linear_num_value_heads, model_config.linear_key_head_dim, model_config.linear_value_head_dim], - device=device, dtype=torch.float32 + device="meta", dtype=torch.float32 ) conv_dim = model_config.linear_num_key_heads * model_config.linear_key_head_dim * 2 + model_config.linear_num_value_heads * model_config.linear_value_head_dim conv_state = torch.zeros( [batch, conv_dim, model_config.conv_kernel_size - 1], - device=device, dtype=execution_dtype + device="meta", dtype=execution_dtype ) kv = LinearKV(conv_state, recurrent_state, 0, None, None) kv.qkv_buf, kv.proj_buf, kv.gates_buf, kv.sumsq_buf = comfy_kitchen.gated_delta_deferred_buffers( - batch, conv_dim, model_config.linear_num_value_heads, model_config.linear_num_key_heads, execution_dtype, device) + batch, conv_dim, model_config.linear_num_value_heads, model_config.linear_num_key_heads, execution_dtype, "meta") kv.ctl, kv.ctl_tracker = tracker["ctl"], tracker past_key_values.append(kv) else: - past_key_values.append(self.model.layers[i].self_attn.init_kv_cache(batch, max_cache_len, device, execution_dtype, shared)) - return past_key_values + past_key_values.append(self.model.layers[i].self_attn.init_kv_cache(batch, max_cache_len, device, execution_dtype, shared, storage_device="meta")) + extra = [self.mtp.layers[0].self_attn.init_kv_cache(batch, max_cache_len, device, execution_dtype, {}, storage_device="meta")] if mtp else [] + return KVCacheModel(past_key_values, extra) # Tokenizer and Text Encoder Wrappers @@ -1204,7 +1334,7 @@ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None if image is not None and len(images) == 0: images = [image[i:i + 1] for i in range(image.shape[0])] - skip_template = False + skip_template = kwargs.get("skip_template", False) if text.startswith('<|im_start|>'): skip_template = True if prevent_empty_text and text == '': @@ -1255,10 +1385,25 @@ def generate(self, tokens, do_sample, max_length, temperature, top_k, top_p, min if isinstance(tokens, dict): tokens = next(iter(tokens.values())) tokens_only = [[t[0] for t in b] for b in tokens] + state = self.model_state.metadata + resume = "pending" in state + if resume: + if len(tokens_only) != 1 or any(not isinstance(t, int) for t in tokens_only[0]): + raise ValueError("Continuing a saved generation supports one text sequence") + tokens_only[0] = state["pending"] + tokens_only[0] embeds, _, _, embeds_info = self.process_tokens(tokens_only, self.execution_device) position_ids = comfy.text_encoders.qwen_vl.qwen2vl_mrope_position_ids(embeds_info, embeds.shape[1], embeds.device) + if resume: + position_ids = torch.arange(state["position"], state["position"] + embeds.shape[1], device=embeds.device).unsqueeze(0) + extra = {} + if llm_state_cache.enabled() and not state and mtp and len(tokens_only) == 1: + # no explicit state: the prompt state cache finds a matching prefix itself (image positions are hashed) + cache_ids = llm_state_cache.prompt_ids(tokens_only[0], embeds_info, embeds.shape[1]) + if cache_ids is not None: + extra["cache_ids"] = cache_ids return self.transformer.generate(embeds, do_sample, max_length, temperature, top_k, top_p, min_p, repetition_penalty, seed, - presence_penalty=presence_penalty, position_ids=position_ids, mtp=mtp) + presence_penalty=presence_penalty, position_ids=position_ids, mtp=mtp, + model_state=self.model_state, **extra) class Qwen35TEModel(sd1_clip.SD1ClipModel): diff --git a/comfy_api/latest/_io.py b/comfy_api/latest/_io.py index 42d2d1f374f..09d3349b30e 100644 --- a/comfy_api/latest/_io.py +++ b/comfy_api/latest/_io.py @@ -21,6 +21,7 @@ from comfy.controlnet import ControlNet from comfy.hooks import HookGroup, HookKeyframeGroup from comfy.model_patcher import ModelPatcher + from comfy.model_state import ModelState as ModelState_ from comfy.samplers import CFGGuider, Sampler from comfy.sd import CLIP, VAE from comfy.sd import StyleModel as StyleModel_ @@ -615,6 +616,12 @@ class Model(ComfyTypeIO): if TYPE_CHECKING: Type = ModelPatcher +@comfytype(io_type="MODEL_STATE") +class ModelState(ComfyTypeIO): + """A writable model's values over a file, such as an LLM's KV cache.""" + if TYPE_CHECKING: + Type = ModelState_ + @comfytype(io_type="BACKGROUND_REMOVAL") class BackgroundRemoval(ComfyTypeIO): if TYPE_CHECKING: @@ -2646,6 +2653,7 @@ def as_dict(self): "ControlNet", "Vae", "Model", + "ModelState", "ModelPatch", "ClipVision", "ClipVisionOutput", diff --git a/comfy_extras/nodes_model_state.py b/comfy_extras/nodes_model_state.py new file mode 100644 index 00000000000..5463b420484 --- /dev/null +++ b/comfy_extras/nodes_model_state.py @@ -0,0 +1,79 @@ +import os + +from typing_extensions import override + +import comfy.model_state +import folder_paths +from comfy_api.latest import ComfyExtension, io + + +def state_path(filename): + output_dir = folder_paths.get_output_directory() + path = os.path.abspath(os.path.join(output_dir, filename)) + if not folder_paths.is_within_directory(output_dir, path): + raise ValueError(f"Model state files must be inside the output folder: {filename}") + os.makedirs(os.path.dirname(path), exist_ok=True) + return path + + +class OpenModelState(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="OpenModelState", + display_name="Open Model State", + category="model_state", + search_aliases=["KV cache", "conversation", "resume"], + description="Opens a model state file in the output folder, such as a text generation conversation, so a sampler can continue from it. The file is only written by Save Model State.", + inputs=[ + io.String.Input("filename", default="model_states/ComfyUI.safetensors"), + io.Combo.Input("create", options=["if_missing", "never", "always"], default="if_missing", tooltip="What to do when the file is missing: start from an empty state, or fail. always ignores the file and starts empty on every run."), + ], + outputs=[io.ModelState.Output(display_name="state")], + ) + + @classmethod + def fingerprint_inputs(cls, filename, create): + if create == "always": + return float("NaN") + path = state_path(filename) + return os.stat(path).st_mtime_ns if os.path.exists(path) else None + + @classmethod + def execute(cls, filename, create) -> io.NodeOutput: + return io.NodeOutput(comfy.model_state.ModelState.open(state_path(filename), create)) + + +class SaveModelState(io.ComfyNode): + @classmethod + def define_schema(cls) -> io.Schema: + return io.Schema( + node_id="SaveModelState", + display_name="Save Model State", + category="model/merging", + description="Writes a model state's values to a file in the folder output/model_states.", + is_output_node=True, + inputs=[ + io.ModelState.Input("state"), + io.String.Input("filename", default="model_states/ComfyUI.safetensors"), + ], + outputs=[io.ModelState.Output(display_name="state", tooltip="The saved state, for chaining or ending a loop.")], + ) + + @classmethod + def execute(cls, state, filename) -> io.NodeOutput: + state.save(state_path(filename)) + return io.NodeOutput(state) + + +class ModelStateExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[io.ComfyNode]]: + return [ + OpenModelState, + SaveModelState, + ] + + +async def comfy_entrypoint() -> ModelStateExtension: + return ModelStateExtension() diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py index 0e7d3a9af6c..ed8d66699a9 100644 --- a/comfy_extras/nodes_textgen.py +++ b/comfy_extras/nodes_textgen.py @@ -1,6 +1,16 @@ from comfy_api.latest import ComfyExtension, io from typing_extensions import override +import json +import time + +import comfy.model_state +from comfy.text_encoders import llm_state_cache + +# older clients send these mode names and read LLMPrefixCacheStats +LEGACY_CACHE_MODES = {"vram": "on", "gpu": "on", "cpu": "on", "disk": "on", "stats": "off"} + + class TextGenerate(io.ComfyNode): @classmethod def define_schema(cls): @@ -41,19 +51,32 @@ def define_schema(cls): io.Boolean.Input("use_default_template", optional=True, default=True, tooltip="Use the built in system prompt/template if the model has one.", advanced=True), io.Combo.Input("mtp", options=["auto", "off", "2", "3", "4", "5", "tree2s1", "tree2s2", "tree3s1", "tree4s1m8"], default="auto", optional=True, tooltip="Speculative decoding with the checkpoint's multi-token-prediction head. No effect without MTP weights. auto adapts the draft depth and breadth to observed token matches and execution cost; 2-5 pins a constant draft depth; tree modes pin both breadth and depth. Sampled output stays correctly distributed but differs from non-MTP output for the same seed."), io.String.Input("system_prompt", force_input=True, optional=True, tooltip="Replaces the system prompt in the model's chat template. Ignored when the default template is not used."), + io.Combo.Input("prefix_cache", options=["env", "off", "on"] + list(LEGACY_CACHE_MODES), default="env", optional=True, advanced=True, tooltip="Reuse the prompt state of an earlier call that shares this prompt's start (one kept per conversation). env follows COMFY_LLM_PREFIX_CACHE. vram, gpu, disk and stats are older clients' names."), + io.ModelState.Input("state", optional=True, tooltip="Continues the conversation held by this model state and extends it with this turn. The prompt should be a single text turn."), ], outputs=[ io.String.Output(display_name="generated_text"), io.String.Output(display_name="thinking"), + io.ModelState.Output(display_name="state", tooltip="A copy of the input state extended by this turn; the input is left as it was."), ], + hidden=[io.Hidden.unique_id, io.Hidden.dynprompt], ) @classmethod - def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="") -> io.NodeOutput: + def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="", state=None, prefix_cache="env") -> io.NodeOutput: + # the mode applies to this call only; env follows COMFY_LLM_PREFIX_CACHE + with llm_state_cache.call_mode(None if prefix_cache == "env" else LEGACY_CACHE_MODES.get(prefix_cache, prefix_cache)): + return cls._generate(clip, prompt, max_length, sampling_mode, image, thinking, use_default_template, video, audio, mtp, system_prompt, state) + @classmethod + def _generate(cls, clip, prompt, max_length, sampling_mode, image, thinking, use_default_template, video, audio, mtp, system_prompt, state) -> io.NodeOutput: mtp = {"off": False, "auto": True}.get(mtp, mtp) - tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt if use_default_template else "") + # the system prompt opens a conversation; later turns continue it + if not use_default_template or (state is not None and state.metadata): + system_prompt = "" + state = comfy.model_state.ModelState() if state is None else state.copy() + tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt) # Get sampling parameters from dynamic combo do_sample = sampling_mode.get("sampling_mode") == "on" @@ -76,8 +99,13 @@ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=F repetition_penalty=repetition_penalty, presence_penalty=presence_penalty, seed=seed, - mtp=mtp + mtp=mtp, + state=state, ) + # the output is written back only when something consumes it; otherwise it is released after the node + dynprompt = cls.hidden.dynprompt + if any(value == [cls.hidden.unique_id, 2] for node_id in dynprompt.all_node_ids() for value in dynprompt.get_node(node_id)["inputs"].values()): + state.flush() generated_text = clip.decode(generated_ids) @@ -90,7 +118,7 @@ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=F if not prompt.rstrip().endswith("") and not clip.decode(prompt_ids[-4:]).rstrip().endswith(""): reasoning, text = "", generated_text - return io.NodeOutput(text.strip(), reasoning.replace("", "", 1).strip()) + return io.NodeOutput(text.strip(), reasoning.replace("", "", 1).strip(), state) LTX2_T2V_SYSTEM_PROMPT = """You are a Creative Assistant. Given a user's raw input prompt describing a scene or concept, expand it into a detailed video generation prompt with specific visuals and integrated audio to guide a text-to-video model. @@ -233,8 +261,9 @@ def define_schema(cls): node_id="TextGenerateLTX2Prompt", display_name="Generate LTX2 Prompt", category=parent_schema.category, - inputs=parent_schema.inputs, - outputs=parent_schema.outputs, + inputs=[i for i in parent_schema.inputs if i.id not in ("state", "prefix_cache")], + outputs=parent_schema.outputs[:2], + hidden=parent_schema.hidden, search_aliases=["prompt enhance", "LLM", "gemma"], ) @@ -274,12 +303,32 @@ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=F return io.NodeOutput(out.args[0] or prompt, out.args[1]) +class LLMPrefixCacheStats(io.ComfyNode): + """The last generate's prompt state cache line, as JSON (for older clients); wire the generated text in so it runs after.""" + @classmethod + def define_schema(cls): + return io.Schema(node_id="LLMPrefixCacheStats", category="text", + inputs=[io.String.Input("text", force_input=True)], + outputs=[io.String.Output(display_name="stats")]) + + @classmethod + def fingerprint_inputs(cls, **kwargs): + return time.time() + + @classmethod + def execute(cls, text) -> io.NodeOutput: + st = dict(llm_state_cache.stats) + st.setdefault("hit", st.get("prefix_len", 0) > 0) + return io.NodeOutput(json.dumps(st)) + + class TextgenExtension(ComfyExtension): @override async def get_node_list(self) -> list[type[io.ComfyNode]]: return [ TextGenerate, TextGenerateLTX2Prompt, + LLMPrefixCacheStats, ] async def comfy_entrypoint() -> TextgenExtension: diff --git a/execution.py b/execution.py index a8918ada17a..f56bd0925cb 100644 --- a/execution.py +++ b/execution.py @@ -561,6 +561,7 @@ def pre_execute_cb(call_index): comfy.model_prefetch.cleanup_prefetch_queues() comfy.model_management.reset_cast_buffers() comfy_aimdo.model_vbar.vbars_reset_watermark_limits() + comfy.model_management.cleanup_ephemeral_models() if has_pending_tasks: pending_async_nodes[unique_id] = output_data diff --git a/main.py b/main.py index dc484c7eb01..64988f13467 100644 --- a/main.py +++ b/main.py @@ -381,7 +381,8 @@ def prompt_worker(q, server_instance, asset_manager): asset_manager.pause_background_scan() background_scan_paused = True - e.execute(item[2], prompt_id, extra_data, item[4]) + with comfy.model_management.model_execution_lock: + e.execute(item[2], prompt_id, extra_data, item[4]) need_gc = True diff --git a/nodes.py b/nodes.py index e279ae2fec4..89ca4196817 100644 --- a/nodes.py +++ b/nodes.py @@ -2527,6 +2527,7 @@ async def init_builtin_extra_nodes(): "nodes_glsl.py", "nodes_lora_debug.py", "nodes_textgen.py", + "nodes_model_state.py", "nodes_text_overlay.py", "nodes_color.py", "nodes_toolkit.py", diff --git a/requirements.txt b/requirements.txt index beddb6f33a0..0a5ab43f9cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,9 @@ SQLAlchemy>=2.0.0 filelock av>=17.0.0 comfy-aimdo==0.6.0 -comfy-kitchen==0.2.40 +# Pinned to comfy-kitchen#303's head (3- and 5-bit weight widths) for our evals and demos, until it merges and is released. +# Boxes with no GitHub access or no CUDA toolkit install the CUDA wheel #303's CI built from this same commit instead. +comfy-kitchen @ git+https://github.com/kijai/comfy-kitchen.git@34bc5830941b7d356343f2bceeb72712d4efade6 requests simpleeval>=1.0.0 blake3 diff --git a/server.py b/server.py index 3eb75d925bb..409e597a4ab 100644 --- a/server.py +++ b/server.py @@ -273,6 +273,8 @@ def __init__(self, loop, asset_manager: "AssetManager"): self.prompt_queue = execution.PromptQueue(self) self.loop = loop self.messages = asyncio.Queue() + self.device_memory = {} + self.read_device_memory_unless_a_prompt_runs([comfy.model_management.get_torch_device()] + list(comfy.model_management.get_all_torch_devices())) self.client_session:Optional[aiohttp.ClientSession] = None self.number = 0 @@ -790,10 +792,11 @@ async def system_stats(request): else: torch_devices = [primary_device] + list(torch_devices) + self.read_device_memory_unless_a_prompt_runs(torch_devices) + device_entries = [] for d in torch_devices: - vram_total, torch_vram_total = comfy.model_management.get_total_memory(d, torch_total_too=True) - vram_free, torch_vram_free = comfy.model_management.get_free_memory(d, torch_free_too=True) + vram_total, torch_vram_total, vram_free, torch_vram_free = self.device_memory[str(d)] device_entries.append({ "name": comfy.model_management.get_torch_device_name(d), "type": d.type, @@ -1319,6 +1322,16 @@ async def setup(self): timeout = aiohttp.ClientTimeout(total=None) # no timeout self.client_session = aiohttp.ClientSession(timeout=timeout) + def read_device_memory_unless_a_prompt_runs(self, devices): + if not comfy.model_management.model_execution_lock.acquire(blocking=False): + return + try: + for d in devices: + self.device_memory[str(d)] = (comfy.model_management.get_total_memory(d, torch_total_too=True) + + comfy.model_management.get_free_memory(d, torch_free_too=True)) + finally: + comfy.model_management.model_execution_lock.release() + def add_routes(self): # The comfy-agent host mounts its own /api/agent and /api/workflows # routes (the proxy + trust boundary now live in the wheel). Do it before diff --git a/tests-unit/app_test/test_prompt_worker.py b/tests-unit/app_test/test_prompt_worker.py index 16e7a11065b..be04be7f8b3 100644 --- a/tests-unit/app_test/test_prompt_worker.py +++ b/tests-unit/app_test/test_prompt_worker.py @@ -155,3 +155,34 @@ def test_prompt_worker_resumes_scan_once_the_queue_is_empty(monkeypatch) -> None def test_prompt_worker_keeps_scan_paused_while_prompts_are_queued(monkeypatch) -> None: assert _paused_when_the_worker_asks_for_the_next_prompt(monkeypatch, queued_after_first=1) == [True] + + +class LockCheckingExecutor(Executor): + held_during_execute = None + + def execute(self, *args, **kwargs) -> None: + LockCheckingExecutor.held_during_execute = main.comfy.model_management.model_execution_lock.locked() + + +def test_prompt_worker_holds_model_execution_lock_only_while_executing(monkeypatch) -> None: + monkeypatch.setattr(main.execution, "PromptExecutor", LockCheckingExecutor) + monkeypatch.setattr(main.time, "perf_counter", lambda: 1.0) + LockCheckingExecutor.held_during_execute = None + lock = main.comfy.model_management.model_execution_lock + held_while_waiting: list[bool] = [] + queue = Queue(on_second_get=lambda: held_while_waiting.append(lock.locked())) + + with pytest.raises(LoopEscape, match="^prompt worker requested a second item$"): + main.prompt_worker(queue, Server(), AssetManager()) + + assert LockCheckingExecutor.held_during_execute is True + assert held_while_waiting == [False] + + +def test_prompt_worker_releases_model_execution_lock_when_execute_raises(monkeypatch) -> None: + monkeypatch.setattr(main.execution, "PromptExecutor", ExecuteFailureExecutor) + + with pytest.raises(RuntimeError, match="^forced execute failure$"): + main.prompt_worker(Queue(), Server(), AssetManager()) + + assert not main.comfy.model_management.model_execution_lock.locked() diff --git a/tests-unit/comfy_test/llm_state_cache_test.py b/tests-unit/comfy_test/llm_state_cache_test.py new file mode 100644 index 00000000000..1bf528451ad --- /dev/null +++ b/tests-unit/comfy_test/llm_state_cache_test.py @@ -0,0 +1,120 @@ +import types + +import pytest +import torch + +from qwen35_decode_test import _ExactPacked, _tiny_model + +import comfy.text_encoders.llm_state_cache as cache # noqa: E402 + +END = list(cache.MSG_END) + + +class _Pages16(_ExactPacked): + page = 16 # a 48-row window: room to resume a turn after the previous reply's draft rows + + +@pytest.fixture(params=["bf16", "int8"]) +def model(request, monkeypatch, tmp_path): + import comfy.text_encoders.llama as llama + import comfy.text_encoders.qwen35 as qwen35 + model = _tiny_model(monkeypatch) + if request.param == "int8": + monkeypatch.setattr(llama, "Int8DecodeCache", _Pages16, raising=False) + monkeypatch.setattr(qwen35.GatedAttention, "kv_cache_type", lambda self, device, dtype: qwen35.Int8FixedKVCache) + monkeypatch.setattr(cache, "CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(cache, "ROUND", 64) + monkeypatch.setattr(cache, "HEADROOM", 0) # a later turn outgrows the first one's capacity + cache._entries.clear() + cache._current.clear() + yield model + for entry in list(cache._entries): + cache._drop(entry) + + +_table = {} + + +def _embeds(model, ids): + # generated tokens embed as the model embeds them; other ids (message ends, prompt text) get fixed vectors + rows = [] + for i in ids: + if i < 97: + rows.append(model.model.embed_tokens(torch.tensor([i]))[0]) + else: + rows.append(_table.setdefault(i, torch.randn(64, generator=torch.Generator().manual_seed(i)))) + return torch.stack(rows).unsqueeze(0) + + +def _text(seed, n): + return [1000 + seed * 100 + k for k in range(n)] + + +def _generate(model, ids, max_length, mode): + """(generated ids, first logits after the prompt) with the cache on or off.""" + first = [] + logits = model.logits + + def record(x): + out = logits(x) + if not first: + first.append(out.detach().clone()) + return out + model.logits = record + try: + with cache.call_mode(mode): + kwargs = {"cache_ids": list(ids)} if mode == "on" else {} + out = model.generate(_embeds(model, ids), do_sample=False, max_length=max_length, mtp=True, **kwargs) + finally: + model.logits = logits + return out, first[0] + + +def _check(model, ids, max_length=8): + got, got_logits = _generate(model, ids, max_length, "on") + stats = dict(cache.stats) + want, want_logits = _generate(model, ids, max_length, "off") + assert got == want + assert torch.allclose(got_logits, want_logits, atol=1e-4), stats + return stats + + +def test_next_turn_resumes_the_conversation(model): + system = _text(0, 12) + END + turn1 = system + _text(1, 20) + END + stats = _check(model, turn1) + assert stats["prefix_len"] == 0 and stats.get("shared_start_saved") + reply = cache._entries[-1].ids[len(turn1):] # the reply tokens the state holds + turn2 = turn1 + reply + END + _text(2, 30) + END + stats = _check(model, turn2) + assert stats["prefix_len"] >= len(turn1) and stats["restore"] in ("resident", "grow") + + +def test_spilled_conversation_reloads_from_disk(model, monkeypatch): + monkeypatch.setattr(cache, "MAX_ENTRIES", 1) + # longer than the int8 window, so the pages must survive the spill + a = _text(0, 12) + END + _text(1, 70) + END + b = _text(5, 12) + END + _text(6, 20) + END # another system prompt: no shared start + _check(model, a) + reply = cache._entries[-1].ids[len(a):] + _check(model, b) + assert sum(e.state is None for e in cache._entries) == 1 # a was written to disk + stats = _check(model, a + reply + END + _text(2, 15) + END) + assert stats["restore"] == "from-disk" and stats["prefix_len"] >= len(a) + + +def test_new_conversation_starts_from_the_shared_start_file(model): + system = _text(0, 70) + END # longer than the int8 window, so the file must hold its pages + _check(model, system + _text(1, 20) + END) + for entry in list(cache._entries): # as after a restart: only the files remain + cache._drop(entry) + stats = _check(model, system + _text(3, 25) + END) + assert stats["restore"] == "shared-start-file" and stats["prefix_len"] == len(system) + + +def test_int8_resumes_only_inside_its_window(): + entry = types.SimpleNamespace(window=(48, 16), written=200) + # the refresh before pos reads the page holding pos - 1 and the one before it; the window holds rows 152..199 + assert not cache._resumable(entry, 170) # needs rows from 144 + assert cache._resumable(entry, 180) # needs rows from 160 + assert cache._resumable(types.SimpleNamespace(window=None, written=200), 10) diff --git a/tests-unit/comfy_test/model_prefetch_cleanup_test.py b/tests-unit/comfy_test/model_prefetch_cleanup_test.py new file mode 100644 index 00000000000..4ccaa71474c --- /dev/null +++ b/tests-unit/comfy_test/model_prefetch_cleanup_test.py @@ -0,0 +1,128 @@ +import pytest +import torch + +from comfy.cli_args import args as cli_args + +if not torch.cuda.is_available(): + cli_args.cpu = True + +import comfy.model_prefetch as model_prefetch # noqa: E402 + + +class MTPHead(torch.nn.Module): + pass + + +@pytest.fixture(autouse=True) +def _reset_queues(): + model_prefetch.PREFETCH_QUEUES = [] + yield + model_prefetch.PREFETCH_QUEUES = [] + + +def _prefetched(module): + module._prefetch = {"signature": None} + return (None, (module, [module])) + + +@pytest.mark.parametrize("pending_unit", [tuple, list]) +def test_cleanup_skips_unconsumed_grouped_unit(pending_unit): + # Qwen3.5 MTP draft queue interrupted after the embed pop: the embed slot has + # been pinned, the grouped (head, mtp) slot has not been reached yet. + embed = torch.nn.Embedding(4, 4) + head = torch.nn.Linear(4, 4) + queue = [_prefetched(embed), pending_unit((head, MTPHead())), None] + model_prefetch.PREFETCH_QUEUES = [queue] + + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + assert not hasattr(embed, "_prefetch") + + +def test_cleanup_resets_queues_when_entry_cleanup_fails(monkeypatch): + def boom(module, comfy_modules): + raise RuntimeError("unpin failed") + + monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", boom) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(torch.nn.Linear(4, 4)), None]] + + with pytest.raises(RuntimeError): + model_prefetch.cleanup_prefetch_queues() + # A failed cleanup must not poison every later node on the worker. + assert model_prefetch.PREFETCH_QUEUES == [] + model_prefetch.cleanup_prefetch_queues() + + +@pytest.mark.parametrize("pending_unit", [tuple, list]) +def test_cleanup_skips_unreached_grouped_head(pending_unit): + # Interrupted between popping the consumed slot and pinning the next one: the + # head of the queue is still the raw (head, mtp) group, not a pinned slot. + queue = [pending_unit((torch.nn.Linear(4, 4), MTPHead())), None] + model_prefetch.PREFETCH_QUEUES = [queue] + + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + + +def test_cleanup_unpins_later_queues_when_an_earlier_one_fails(monkeypatch): + real_cleanup = model_prefetch.cleanup_prefetched_modules + failing = torch.nn.Linear(4, 4) + later = torch.nn.Linear(4, 4) + + def cleanup(module, comfy_modules): + if module is failing: + raise RuntimeError("unpin failed") + real_cleanup(module, comfy_modules) + + monkeypatch.setattr(model_prefetch, "cleanup_prefetched_modules", cleanup) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(failing), None], [_prefetched(later), None]] + + with pytest.raises(RuntimeError, match="unpin failed"): + model_prefetch.cleanup_prefetch_queues() + # The first failure is re-raised, but only after every other queue was cleaned up. + assert not hasattr(later, "_prefetch") + assert model_prefetch.PREFETCH_QUEUES == [] + + +def test_failed_unpin_still_clears_prefetch_markers(monkeypatch): + # A module whose unpin raised must not keep _prefetch, or a later cast would skip + # the fresh cast and reuse the stale transfer state. The rest of the group is still cleaned. + calls = [] + + def vbar_unpin(v): + calls.append(v) + if v == "first": + raise RuntimeError("unpin failed") + + monkeypatch.setattr(model_prefetch.comfy_aimdo.model_vbar, "vbar_unpin", vbar_unpin) + first = torch.nn.Linear(4, 4) + second = torch.nn.Linear(4, 4) + for module, v in ((first, "first"), (second, "second")): + module._v = v + module._prefetch = {"signature": "sig"} + model_prefetch.PREFETCH_QUEUES = [[(None, (first, [first, second])), None]] + + with pytest.raises(RuntimeError, match="unpin failed"): + model_prefetch.cleanup_prefetch_queues() + assert calls == ["first", "second"] + assert not hasattr(first, "_prefetch") + assert not hasattr(second, "_prefetch") + + +def test_malloc_graph_cleanup_failure_still_cleans_queues(monkeypatch): + # After a GPU failure the graph abort can raise too. Queues and _prefetch markers must not + # survive into the next node, and the graph error still reaches the caller. + def boom(): + raise RuntimeError("graph abort failed") + + monkeypatch.setattr(model_prefetch, "cleanup_malloc_graph", boom) + module = torch.nn.Linear(4, 4) + model_prefetch.PREFETCH_QUEUES = [[_prefetched(module), None]] + + with pytest.raises(RuntimeError, match="graph abort failed"): + model_prefetch.cleanup_prefetch_queues() + + assert model_prefetch.PREFETCH_QUEUES == [] + assert not hasattr(module, "_prefetch") diff --git a/tests-unit/comfy_test/qwen21_cache_admission_test.py b/tests-unit/comfy_test/qwen21_cache_admission_test.py index d8f3f020504..0b27ce6c41b 100644 --- a/tests-unit/comfy_test/qwen21_cache_admission_test.py +++ b/tests-unit/comfy_test/qwen21_cache_admission_test.py @@ -43,6 +43,7 @@ def test_cache_budget_admits_all_conditions_together(monkeypatch, vram_margin, r monkeypatch.setattr(system_memory, "virtual_memory_available", lambda: 3 * 1024 ** 3 + cache_bytes + ram_margin) monkeypatch.setattr(model_management, "free_pins", lambda size, **kwargs: min(size, reclaimable)) monkeypatch.setattr(patcher, "get_free_memory", lambda device: 4 * cache_bytes + vram_margin) + monkeypatch.setattr(patcher, "is_dynamic", lambda: True) cache = model.get_writable_model(conditions, (1, 4, 3, 5), {"transformer_options": transformer_options}, torch.device("cpu")) assert (cache is not None) == admitted diff --git a/tests-unit/comfy_test/qwen35_decode_test.py b/tests-unit/comfy_test/qwen35_decode_test.py index efd322865cf..0475bdedcb1 100644 --- a/tests-unit/comfy_test/qwen35_decode_test.py +++ b/tests-unit/comfy_test/qwen35_decode_test.py @@ -9,7 +9,10 @@ if not torch.cuda.is_available(): cli_args.cpu = True +import comfy.model_management # noqa: E402 import comfy.ops # noqa: E402 +import comfy.model_state # noqa: E402 +from comfy.model_patcher import ModelPatcher # noqa: E402 from comfy.text_encoders.llama import MLP, RMSNorm # noqa: E402 @@ -337,6 +340,17 @@ class Tiny(qwen35.Qwen35): for name, p in model.named_parameters(): p.data = torch.randn_like(p) * (0.05 if "norm" in name else 0.3) model.model.lm_head.weight.data[:8] *= 4 # a few likely tokens, so some drafts survive + + # Generation loads its KV cache into a model state, as CLIP.generate does; each test ends its node. + monkeypatch.setattr(comfy.model_management, "EPHEMERAL_MODELS", []) + generate = model.generate + + def generate_with_cache(*args, model_state=None, **kwargs): + if model_state is None: + model_state = comfy.model_state.ModelState() + comfy.model_management.register_ephemeral_model(model_state) + return generate(*args, model_state=model_state, **kwargs) + model.generate = generate_with_cache return model @@ -410,6 +424,199 @@ def best_widths(widths, hits, draft_ms, verify_ms): assert len(set(switches)) == 4, switches +@pytest.mark.parametrize("mtp", [False, "3", "tree2s2"]) +def test_continuation_from_model_state_matches_uninterrupted_generation(monkeypatch, tmp_path, mtp): + model = _tiny_model(monkeypatch) + embeds = torch.randn(1, 5, 64) + expected = model.generate(embeds.clone(), do_sample=False, max_length=20, mtp=False) + for cutoff in (2, 4, 7, 9): + path = str(tmp_path / f"{cutoff}.safetensors") + state = comfy.model_state.ModelState.open(path) + first = model.generate(embeds.clone(), do_sample=False, max_length=cutoff, mtp=mtp, model_state=state) + # pending is the last sampled token, after the last step's tokens the DeltaNet state hadn't committed + pending_ids = state.metadata["pending"] + assert pending_ids == first[-len(pending_ids):] + assert state.metadata["length"] == embeds.shape[1] + cutoff - len(pending_ids) + state.flush() + state.save(path) + + # The next turn reopens the saved file, as a later prompt would. + saved = comfy.model_state.ModelState.open(path, create="never") + pending = model.model.embed_tokens(torch.tensor([saved.metadata["pending"]])) + position = saved.metadata["position"] + rest = model.generate(pending, do_sample=False, max_length=20 - cutoff, mtp=mtp, model_state=saved, + position_ids=torch.arange(position, position + len(pending_ids)).unsqueeze(0)) + saved.release() + assert first + rest == expected + + +def test_batched_continue_matches_one_prefill(monkeypatch): + # a prefill continuing a cached prefix (DeltaNet conv window and recurrent state, attention prefix) + # gives the same hidden states as prefilling the whole sequence at once + model = _tiny_model(monkeypatch) + embeds = torch.randn(1, 30, 64) + + def prefill(*parts): + state = comfy.model_state.ModelState() + comfy.model_management.register_ephemeral_model(state) + cpu = torch.device("cpu") + cache = state.load(model.init_kv_cache(1, 32, cpu, torch.float32, mtp=True), cpu) + out = [model.prefill(embeds[:, part].clone(), cache.caches)[0] for part in parts] # the forward reuses its input buffer + state.release() + return out + + whole, = prefill(slice(0, 30)) + _, rest = prefill(slice(0, 11), slice(11, 30)) + assert torch.allclose(rest, whole[:, 11:], atol=1e-4) + +@pytest.mark.parametrize("kv", ["bf16", "int8"]) +def test_long_prompt_prefills_in_pieces(monkeypatch, kv): + # a prompt over PREFILL_ROWS goes in equal pieces that continue each other, and its last hidden state + # and the cache it leaves match one pass over the whole prompt + import comfy.text_encoders.llama as llama + import comfy.text_encoders.qwen35 as qwen35 + model = _tiny_model(monkeypatch) + if kv == "int8": + monkeypatch.setattr(llama, "Int8DecodeCache", _ExactPacked, raising=False) + monkeypatch.setattr(qwen35.GatedAttention, "kv_cache_type", lambda self, device, dtype: qwen35.Int8FixedKVCache) + embeds, more = torch.randn(1, 30, 64), torch.randn(1, 12, 64) + passes = [] + forward = model.model.forward + monkeypatch.setattr(model.model, "forward", lambda *a, **kw: passes.append(kw["embeds"].shape[1]) or forward(*a, **kw)) + + def prefill(rows): + monkeypatch.setattr(qwen35, "PREFILL_ROWS", rows) + state = comfy.model_state.ModelState() + comfy.model_management.register_ephemeral_model(state) + cpu = torch.device("cpu") + cache = state.load(model.init_kv_cache(1, 48, cpu, torch.float32, mtp=True), cpu) + last = model.prefill(embeds.clone(), cache.caches)[0][:, -1] # the forward reuses its input buffer + monkeypatch.setattr(qwen35, "PREFILL_ROWS", 4096) + after = model.prefill(more.clone(), cache.caches)[0] + state.release() + return last, after + + whole = prefill(4096) + passes.clear() + pieces = prefill(11) + assert passes == [10, 10, 10, 12] + assert torch.allclose(pieces[0], whole[0], atol=1e-4) + assert torch.allclose(pieces[1], whole[1], atol=1e-4) + +@pytest.mark.parametrize("mtp", [False, "3", "tree2s2"]) +def test_long_continuation_prefills_in_one_batch(monkeypatch, tmp_path, mtp): + # a continued turn with more than SPEC_ROWS new tokens prefills as one batch, from the saved + # DeltaNet conv/recurrent state and the attention prefix, and matches reading the whole prompt at once + import comfy.text_encoders.qwen35 as qwen35 + model = _tiny_model(monkeypatch) + batched = [] + check = qwen35.Qwen35.batched_continue + monkeypatch.setattr(qwen35.Qwen35, "batched_continue", staticmethod(lambda n, kvs: batched.append(check(n, kvs)) or batched[-1])) + turn1, turn2 = torch.randn(1, 5, 64), torch.randn(1, 13, 64) + path = str(tmp_path / "state.safetensors") + state = comfy.model_state.ModelState.open(path) + first = model.generate(turn1.clone(), do_sample=False, max_length=6, mtp=mtp, model_state=state) + state.flush() + state.save(path) + + saved = comfy.model_state.ModelState.open(path, create="never") + pending_ids, position = saved.metadata["pending"], saved.metadata["position"] + embeds = torch.cat([model.model.embed_tokens(torch.tensor([pending_ids])), turn2], dim=1) + batched.clear() + rest = model.generate(embeds, do_sample=False, max_length=12, mtp=mtp, model_state=saved, + position_ids=torch.arange(position, position + embeds.shape[1]).unsqueeze(0)) + saved.release() + assert batched[0] is True + + whole = torch.cat([turn1, model.model.embed_tokens(torch.tensor([first])), turn2], dim=1) + assert rest == model.generate(whole, do_sample=False, max_length=12, mtp=False) + + + +class _ExactPacked: + # Int8DecodeCache stand-in that pages exact rows, `page` per page, so a continued int8 prefill can be checked + # against bf16. A refresh reads only what the kernel may: the last two pages, from the circular window. + dim, page = 64, 4 + + def __init__(self, batch, heads, capacity, device): + shape = (batch, -(-capacity // self.page), heads, self.page, self.dim) + self.key, self.value = torch.zeros(shape, device=device), torch.zeros(shape, device=device) + self.key_scale, self.value_scale = (torch.zeros(shape[:3] + (1,), device=device) for _ in range(2)) # per page, as the kernel's + + def update(self, key, value, length, initialize=False): + n, page = int(length.item()), self.page + first = 0 if initialize else max(0, (n - 1) // page * page - page) + for i in range(first, n): + src = i if initialize else i % key.shape[2] + self.key[:, i // page, :, i % page] = key[:, :, src] + self.value[:, i // page, :, i % page] = value[:, :, src] + + def attend(self, query, length): + n = int(length.item()) + batch, heads, seq, dim = query.shape + k, v = (t.permute(0, 2, 1, 3, 4).reshape(batch, t.shape[2], -1, dim)[:, :, :n] for t in (self.key, self.value)) + k, v = (t.repeat_interleave(heads // t.shape[1], dim=1) for t in (k, v)) + scores = torch.matmul(query.float(), k.float().transpose(-1, -2)) * dim ** -0.5 + out = torch.matmul(scores.softmax(-1), v.float()) + return out.transpose(1, 2).reshape(batch, seq, heads * dim).to(query.dtype), scores.logsumexp(-1) + + +@pytest.mark.parametrize("merge_elements", [1 << 26, 4 * 19 * 5]) +def test_int8_extend_matches_full_attention(monkeypatch, merge_elements): + # a continued prefill over an int8 prefix: each new row sees the prefix and the new rows up to itself, + # and the pages afterwards hold every row, across page and window boundaries; the small merge budget + # takes the new rows in chunks of 5 + import comfy.text_encoders.llama as llama + monkeypatch.setattr(llama, "Int8DecodeCache", _ExactPacked, raising=False) + monkeypatch.setattr(llama, "CAUSAL_MERGE_ELEMENTS", merge_elements) + kv = llama.Int8FixedKVCache.zeros(1, 2, 64, 64, "cpu", torch.float32, llama.FixedKVCache.shared(1, "cpu")) + q, k, v = torch.randn(1, 4, 30, 64), torch.randn(1, 2, 30, 64), torch.randn(1, 2, 30, 64) + + def full(rows, length): + scores = torch.matmul(q[:, :, rows], k.repeat_interleave(2, dim=1)[:, :, :length].transpose(-1, -2)) * 64 ** -0.5 + scores.masked_fill_(torch.arange(length) > torch.arange(length)[rows].view(-1, 1), float("-inf")) + return torch.matmul(scores.softmax(-1), v.repeat_interleave(2, dim=1)[:, :, :length]).transpose(1, 2).reshape(1, -1, 256) + + kv.prepare(11) + kv.append(k[:, :, :11], v[:, :, :11]) + kv.advance(11) + kv.prepare(19) + assert torch.allclose(kv.extend(q[:, :, 11:], k[:, :, 11:], v[:, :, 11:]), full(slice(11, 30), 30), atol=1e-5) + out, _ = kv.packed.attend(q[:, :, 29:], torch.tensor([30])) + assert torch.allclose(out, full(slice(29, 30), 30), atol=1e-5) + +@pytest.mark.parametrize("mtp", [False, "3", "tree2s2"]) +def test_int8_long_continuation_matches_bf16(monkeypatch, tmp_path, mtp): + # the int8 cache continues a long turn through extend(): new rows attend the paged prefix and each + # other, then the pages grow through the window; with exact pages it must match the bf16 cache + import comfy.text_encoders.llama as llama + import comfy.text_encoders.qwen35 as qwen35 + model = _tiny_model(monkeypatch) + turn1, turn2 = torch.randn(1, 5, 64), torch.randn(1, 13, 64) + + def two_turns(name): + path = str(tmp_path / f"{name}.safetensors") + state = comfy.model_state.ModelState.open(path) + first = model.generate(turn1.clone(), do_sample=False, max_length=6, mtp=mtp, model_state=state) + state.flush() + state.save(path) + saved = comfy.model_state.ModelState.open(path, create="never") + pending_ids, position = saved.metadata["pending"], saved.metadata["position"] + embeds = torch.cat([model.model.embed_tokens(torch.tensor([pending_ids])), turn2], dim=1) + rest = model.generate(embeds, do_sample=False, max_length=12, mtp=mtp, model_state=saved, + position_ids=torch.arange(position, position + embeds.shape[1]).unsqueeze(0)) + saved.release() + return first, rest + + expected = two_turns("bf16") + monkeypatch.setattr(llama, "Int8DecodeCache", _ExactPacked, raising=False) + monkeypatch.setattr(qwen35.GatedAttention, "kv_cache_type", lambda self, device, dtype: qwen35.Int8FixedKVCache) + extended = [] + extend = llama.Int8FixedKVCache.extend + monkeypatch.setattr(llama.Int8FixedKVCache, "extend", lambda self, q, k, v: extended.append(k.shape[2]) or extend(self, q, k, v)) + assert two_turns("int8") == expected + assert extended and min(extended) > 8 + class _FakePacked: # Int8DecodeCache boundary with 4-row pages: records the BF16 refreshes it is asked for def __init__(self, batch, heads, capacity, device): @@ -508,8 +715,6 @@ def _attention_config(config): def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch): import comfy.ldm.modules.attention as attention import comfy.lora - from comfy.model_patcher import ModelPatcher - import comfy.text_encoders.llama as llama import comfy.text_encoders.qwen35 as qwen35 from comfy.text_encoders.llama import FixedKVCache @@ -517,7 +722,6 @@ def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch): monkeypatch.setattr(qwen35.comfy_kitchen, "int8_attention_is_available", lambda device: True) monkeypatch.setattr(qwen35.comfy_kitchen, "sol_attn_is_available", lambda device: False) monkeypatch.setattr(qwen35, "int8_decode_is_available", lambda device: True) - monkeypatch.setattr(llama, "Int8DecodeCache", _FakePacked) monkeypatch.setattr(cli_args, "use_ck_attention", False) model = _tiny_model(monkeypatch, layers=8, head_dim=256) key = "model.layers.3.self_attn.comfy_attention.config" @@ -532,7 +736,7 @@ def test_checkpoint_and_lora_attention_select_each_kv_layer(monkeypatch): def cache_types(): kv = model.init_kv_cache(1, 32, device, torch.bfloat16) draft = model.mtp.layers[0].self_attn.init_kv_cache(1, 32, device, torch.bfloat16, {}) - return type(kv[3]), type(kv[7]), type(draft) + return type(kv.caches[3]), type(kv.caches[7]), type(draft) assert cache_types() == (qwen35.Int8FixedKVCache, FixedKVCache, qwen35.Int8FixedKVCache) monkeypatch.setattr(qwen35.comfy_kitchen, "sol_attn_is_available", lambda device: True) diff --git a/tests-unit/comfy_test/qwen_writable_cache_cuda.py b/tests-unit/comfy_test/qwen_writable_cache_cuda.py new file mode 100644 index 00000000000..783f6a1085c --- /dev/null +++ b/tests-unit/comfy_test/qwen_writable_cache_cuda.py @@ -0,0 +1,170 @@ +"""Bounded hybrid Qwen generation against plain, resident, pinned and file-only model states. Run on a disposable CUDA host.""" +import json +import os +from pathlib import Path +import sys +import tempfile +import weakref + +os.environ.setdefault("PYTORCH_ALLOC_CONF", "backend:cudaMallocAsync") +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import safetensors.torch +import torch +import comfy_aimdo.control as aimdo + +assert aimdo.init("cuda") and aimdo.init_device(0) +from comfy.cli_args import args +args.use_ck_attention = True +import comfy.ldm.wan.model_animate2 # noqa: F401 -- initialize the model registry before the patcher +from comfy import memory_management, model_management as mm, model_patcher, model_prefetch as prefetch, model_state, ops, storage +from comfy.text_encoders import qwen35 as qwen +from comfy_kitchen.int8_decode import is_available + +memory_management.aimdo_enabled = True +device = torch.device("cuda", 0) +qwen.QWEN35_MODELS["probe"] = dict(vision=dict(hidden_size=32, num_heads=2, intermediate_size=64, depth=1)) + +class Probe(qwen.Qwen35): + model_type = "probe" + +config = dict(vocab_size=97, hidden_size=64, intermediate_size=128, num_hidden_layers=4, + layer_types=qwen._qwen35_layer_types(4), num_attention_heads=2, num_key_value_heads=1, + head_dim=256, linear_num_key_heads=2, linear_num_value_heads=2, + linear_key_head_dim=128, linear_value_head_dim=128, mtp=True, lm_head=True, stop_tokens=[]) + +with torch.inference_mode(), tempfile.TemporaryDirectory() as tmp: + torch.manual_seed(37) + memory_management.aimdo_enabled = False + model = Probe(config, torch.bfloat16, "cpu", ops.manual_cast) + memory_management.aimdo_enabled = True + for name, p in model.named_parameters(): + p.copy_(torch.randn_like(p) * (0.05 if "norm" in name else 0.08)) + model.model.lm_head.weight[:8].mul_(4) + patcher = model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu")) + mm.load_models_gpu([patcher]) + patcher.set_in_use_by_current_prompt(True) + original_forward = model.model.forward + outputs = [] + captured = set() + states = [] + mode = "hard" + + class _ModeState(model_state.ModelState): + # The hard mode loads without dynamic VRAM; the others toggle it back on for the model under test. + def __init__(self, mode): + super().__init__() + self.mode = mode + + def load(self, model, device): + states.append(self) + memory_management.aimdo_enabled = self.mode != "hard" + loaded = super().load(model, device) + memory_management.aimdo_enabled = True + return loaded + + def forward(*args, **kwargs): + for s in states: + if mode in ("ram", "disk") and not outputs or mode == "evict" and len(outputs) == 4: + s.patcher._vbar_get().set_watermark(0) + if mode == "evict" and len(outputs) == 8: + mm.DISABLE_PINNED_MEMORY = True + s.patcher.partially_unload_ram(1e32) + result = original_forward(*args, **kwargs) + for s in states: + if mode in ("ram", "disk") or mode == "evict" and len(outputs) >= 4: + assert s.patcher.loaded_size() == 0 + if mode == "disk" or mode == "evict" and len(outputs) >= 8: + assert s.patcher.loaded_ram_size() == 0 + outputs.append(result[0].cpu().clone()) + captured.update(i for i, layer in enumerate(model.model.layers) if hasattr(layer, "_comfy_graph")) + return result + + model.model.forward = forward + for attention, length in (("bf16", 19), ("int8", 19), ("int8", 1533)): + mm.args.use_ck_attention = attention == "int8" + embeds = torch.randn(1, length, 64, device=device, dtype=torch.bfloat16) + continuation = torch.randn(1, 1, 64, device=device, dtype=torch.bfloat16) + mm.args.disable_fast_disk = length <= 512 + storage.anonymous_fast_disk.cache_clear() + if attention == "int8": + assert is_available(device) + for layer in [model.model.layers[3], model.mtp.layers[0]]: + layer.self_attn.comfy_attention.method = "comfy_kitchen_int8" if attention == "int8" else "sdpa" + for graphs, queue in ((False, True), (True, True)): + mm.args.disable_cuda_graphs = not graphs + model.model.prefetch_dynamic_vbars = queue + for mtp in (False, "3"): + baseline = None + for mode in ("hard", "resident", "ram", "disk", "evict"): + mm.DISABLE_PINNED_MEMORY = mode == "disk" + outputs.clear() + captured.clear() + # The resident run is saved to a named file; the others stay anonymous. + path = os.path.join(tmp, f"{attention}-{length}-{graphs}-{mtp}.safetensors") if mode == "resident" else None + state = _ModeState(mode) + mm.register_ephemeral_model(state) + ids = model.generate(embeds.clone(), do_sample=True, seed=71, max_length=24, mtp=mtp, model_state=state) + # A consumer flushes the values back to CPU tensors, which drops the loaded model. + dead = [weakref.ref(s.patcher) for s in states] + [weakref.ref(s.patcher.model) for s in states] + states.clear() + state.flush() + assert state.patcher is None + saved = state.tensors + if mode == "hard": + # A plain model state restores its tensors into device memory. + reopened = state.load(model.init_kv_cache(1, state.metadata["capacity"], device, torch.bfloat16, mtp=bool(mtp)), device) + assert all(torch.equal(v.cpu(), saved[k]) for k, v in reopened.state_dict().items()) + del reopened + state.release() + if path is not None: + state.save(path) + assert all(torch.equal(v, saved[k]) for k, v in safetensors.torch.load_file(path).items()) + # Node end: the flushed patcher and its model must both be dead with nothing else holding them. + mm.reset_cast_buffers() + mm.cleanup_ephemeral_models() + assert not mm.EPHEMERAL_MODELS and all(ref() is None for ref in dead), (mode, dead) + assert any(cache.get("uncommitted") for cache in state.metadata["caches"]) # The last DeltaNet update has not been committed. + # The next turn copies the state, reopened by name or still in memory, onto a fresh cache. + source = state if path is None else model_state.ModelState.open(path, create="never") + assert source.metadata == json.loads(json.dumps(state.metadata)) + turn = source.copy() + assert turn.metadata == source.metadata and turn.metadata is not source.metadata + assert all(torch.equal(v, saved[k]) for k, v in turn.tensors.items()) + restored = turn.load(model.init_kv_cache(1, turn.metadata["capacity"], device, torch.bfloat16, mtp=bool(mtp)), device) + restored.restore_state(turn.metadata["caches"]) + resumed = original_forward(None, embeds=continuation, attention_mask=None, past_key_values=restored.caches)[0].clone() + prefetch.cleanup_prefetch_queues() + turn.release() + del restored + if baseline is None: + baseline = ids, list(outputs), resumed + else: + assert ids == baseline[0], (attention, graphs, queue, mtp, mode, ids, baseline[0]) + assert len(outputs) == len(baseline[1]) + error = max((a.float() - b.float()).abs().max().item() for a, b in zip(outputs, baseline[1])) + assert error == 0, (attention, graphs, queue, mtp, mode, error) + assert torch.equal(resumed, baseline[2]), ("resume", attention, graphs, queue, mtp, mode) + if graphs and mode in ("hard", "resident", "evict"): + assert captured + if mode in ("ram", "disk"): + assert not captured + print("QWEN_WRITABLE_PASS", attention, length, graphs, queue, mtp, mode, len(ids), len(captured), flush=True) # noqa: T201 + pending = torch.nn.Sequential(ops.WritableWeight((1,), torch.float32)) + pending.comfy_writable = True + output = torch.empty(2 ** 20, device=device) + state = model_state.ModelState() + mm.register_ephemeral_model(state) + state.load(pending, device) + with prefetch.malloc_graph_scope(device): + temporary = torch.full_like(output, 1.25) + torch.cuda._sleep(50_000_000) + output.copy_(temporary) + del temporary + del pending + prefetch.cleanup_prefetch_queues() + mm.reset_cast_buffers() + mm.cleanup_ephemeral_models() + assert torch.all(output.cpu() == 1.25), "Writable teardown freed pages before pending work finished" + mm.unload_model_and_clones(patcher) +print("QWEN_WRITABLE_ALL_PASS", flush=True) # noqa: T201 diff --git a/tests-unit/comfy_test/tensor_file_test.py b/tests-unit/comfy_test/tensor_file_test.py index abe30152cd8..6c343ca6d4c 100644 --- a/tests-unit/comfy_test/tensor_file_test.py +++ b/tests-unit/comfy_test/tensor_file_test.py @@ -5,8 +5,14 @@ import safetensors.torch import torch -from comfy.memory_management import read_tensor_file_slice_into, write_tensor_file_slice -from comfy.utils import create_safetensors +from comfy.cli_args import args as cli_args + +if not torch.cuda.is_available(): + cli_args.cpu = True + +import comfy.model_state # noqa: E402 +from comfy.memory_management import read_tensor_file_slice_into, write_tensor_file_slice # noqa: E402 +from comfy.utils import create_safetensors # noqa: E402 pytestmark = pytest.mark.skipif( os.name != "nt" and sys.platform != "linux", @@ -65,3 +71,94 @@ def test_anonymous_safetensors(): assert torch.equal(safetensors.torch.load(file.read())["kv"], expected) finally: file.close() + + +class _KV(torch.nn.Module): + def __init__(self, rows=4): + super().__init__() + self.register_buffer("k", torch.empty(rows, 8)) + self.register_buffer("v", torch.empty(rows, 8, dtype=torch.float16)) + + +def test_model_state_persists_values_and_metadata(tmp_path): + cpu = torch.device("cpu") + path = str(tmp_path / "state.safetensors") + state = comfy.model_state.ModelState.open(path) + assert state.tensors == {} and state.metadata == {} + model = state.load(_KV(), cpu) + assert not model.k.count_nonzero() and not model.v.count_nonzero() + model.k.fill_(3.0) + model.v.fill_(5.0) + state.metadata["position"] = 7 + state.flush() + assert state.patcher is None + state.save(path) + + saved = safetensors.torch.load_file(path) + assert torch.equal(saved["k"], torch.full((4, 8), 3.0)) and torch.equal(saved["v"], torch.full((4, 8), 5.0, dtype=torch.float16)) + + reopened = comfy.model_state.ModelState.open(path, create="never") + assert reopened.metadata == {"position": 7} + + # The next turn continues on a copy; the opened state only reaches the file when saved. + turn = reopened.copy() + model = turn.load(_KV(), cpu) + assert torch.equal(model.k, saved["k"]) and torch.equal(model.v, saved["v"]) + model.k[0, 0] = 9.0 + turn.metadata["position"] = 8 + turn.flush() + reopened.save(path) + assert safetensors.torch.load_file(path)["k"][0, 0] == 3.0 + turn.save(path) + assert safetensors.torch.load_file(path)["k"][0, 0] == 9.0 + assert comfy.model_state.ModelState.open(path).metadata == {"position": 8} + + with pytest.raises(FileNotFoundError): + comfy.model_state.ModelState.open(str(tmp_path / "missing.safetensors"), create="never") + with pytest.raises(RuntimeError): + comfy.model_state.ModelState.open(path).load(_KV(rows=1), cpu) + + recreated = comfy.model_state.ModelState.open(path, create="always") + assert recreated.metadata == {} and recreated.tensors == {} + assert not recreated.load(_KV(), cpu).k.count_nonzero() + assert safetensors.torch.load_file(path)["k"][0, 0] == 9.0 + + +def test_model_state_rejects_plain_safetensors(tmp_path): + path = str(tmp_path / "plain.safetensors") + safetensors.torch.save_file({"k": torch.zeros(4, 8)}, path) + with pytest.raises(ValueError): + comfy.model_state.ModelState.open(path) + + +def test_model_state_copies_are_independent(tmp_path): + cpu = torch.device("cpu") + first = comfy.model_state.ModelState() + model = first.load(_KV(), cpu) + model.k.fill_(2.0) + first.metadata["position"] = 3 + first.flush() + + second = first.copy() + assert second.metadata == {"position": 3} and second.metadata is not first.metadata + model = second.load(_KV(), cpu) + model.k[0, 0] = 1.0 + second.metadata["position"] = 4 + second.flush() + + third = second.copy() + assert third.metadata == {"position": 4} + assert torch.equal(third.load(_KV(), cpu).k, model.k) and model.k[0, 0] == 1.0 + assert torch.equal(first.load(_KV(), cpu).k, torch.full((4, 8), 2.0)) and first.metadata == {"position": 3} + + # Releasing a loaded state drops values its metadata may already describe, so it cannot reach another node. + first.patcher.model.k.fill_(5.0) + first.release() + with pytest.raises(RuntimeError): + first.copy() + with pytest.raises(RuntimeError): + first.save(str(tmp_path / "released.safetensors")) + # A written back state stays usable. + third.flush() + third.release() + assert torch.equal(third.copy().load(_KV(), cpu).k, model.k) diff --git a/tests-unit/server_test/test_system_stats.py b/tests-unit/server_test/test_system_stats.py new file mode 100644 index 00000000000..f82fec3b763 --- /dev/null +++ b/tests-unit/server_test/test_system_stats.py @@ -0,0 +1,45 @@ +from unittest.mock import MagicMock + +import pytest +from aiohttp.test_utils import TestClient, TestServer + +import comfy.model_management +import server + + +@pytest.mark.asyncio +async def test_system_stats_reports_last_values_while_a_prompt_runs(monkeypatch): + memory = {"total": (100, 6), "free": (14, 4)} + queried = [] + + def get_total_memory(dev=None, torch_total_too=False): + queried.append("total") + return memory["total"] if torch_total_too else memory["total"][0] + + def get_free_memory(dev=None, torch_free_too=False): + queried.append("free") + return memory["free"] if torch_free_too else memory["free"][0] + + monkeypatch.setattr(comfy.model_management, "get_total_memory", get_total_memory) + monkeypatch.setattr(comfy.model_management, "get_free_memory", get_free_memory) + prompt_server = server.PromptServer(None, MagicMock(enabled=False)) + handler = next(r.handler for r in prompt_server.routes if getattr(r, "path", None) == "/system_stats") + prompt_server.app.router.add_get("/system_stats", handler) + + def device(stats): + d = stats["devices"][0] + return d["vram_total"], d["torch_vram_total"], d["vram_free"], d["torch_vram_free"] + + async with TestClient(TestServer(prompt_server.app)) as client: + with comfy.model_management.model_execution_lock: + queried.clear() + assert device(await (await client.get("/system_stats")).json()) == (100, 6, 14, 4), \ + "a first request during a prompt gets the values read at startup" + assert queried == ["total", "free"], "only the cpu's RAM is read while a prompt runs" + + memory.update(total=(200, 7), free=(30, 5)) + assert device(await (await client.get("/system_stats")).json()) == (200, 7, 30, 5) + + memory.update(total=(300, 8), free=(40, 6)) + with comfy.model_management.model_execution_lock: + assert device(await (await client.get("/system_stats")).json()) == (200, 7, 30, 5)