diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index ecc5c52f85d..a649dc55b5b 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -962,6 +962,8 @@ def unpin_weight(self, key): self.pinned.remove(key) def unpin_all_weights(self): + for callback in self.get_all_callbacks(CallbacksMP.ON_UNLOAD_RAM): + callback(self) for key in list(self.pinned): self.unpin_weight(key) @@ -1823,7 +1825,10 @@ def is_dynamic(self): return True def discard_writes(self): - """Forget pinned writes so unpinning does not flush them to the backing file.""" + """Forget resident and pinned writes without flushing them to the backing file.""" + for module in self.model.modules(): + if hasattr(module, "_v_signature"): + module._v_signature = None pin_state = self.model.dynamic_pins[self.load_device] for subset in ("weights", "weights-loaded", "weights-fast"): for module, _ in pin_state[subset][1]: @@ -1887,6 +1892,7 @@ def unpin_weight(self, key): raise RuntimeError("unpin_weight invalid for dymamic weight loading") def unpin_all_weights(self): + super().unpin_all_weights() self.partially_unload_ram(1e32) def __del__(self): @@ -2167,8 +2173,11 @@ def unregister_inactive_pins(self, ram_to_unload, subsets=[ "weights-loaded", "p return freed def partially_unload_ram(self, ram_to_unload, subsets=[ "weights-fast", "patches-fast", "weights-loaded", "patches-loaded", "weights", "patches" ]): - freed = 0 pin_state = self.model.dynamic_pins[self.load_device] + if ram_to_unload > 0 and any(pin_state[subset][1] for subset in subsets): + for callback in self.get_all_callbacks(CallbacksMP.ON_UNLOAD_RAM): + callback(self) + freed = 0 for subset in subsets: hostbuf, stack, stack_split, pinned_size, *_ = pin_state[subset] while len(stack) > 0: diff --git a/comfy/model_state.py b/comfy/model_state.py index 8b9416257c3..215cff61433 100644 --- a/comfy/model_state.py +++ b/comfy/model_state.py @@ -1,38 +1,143 @@ import copy +import itertools import json import os +import tempfile +import weakref import safetensors -import safetensors.torch import torch import comfy.memory_management import comfy.model_management import comfy.model_patcher -import comfy.model_prefetch import comfy.storage import comfy.utils METADATA_KEY = "comfy_model_state" +SAVED_STATES = weakref.WeakValueDictionary() +_STATE_VERSIONS = itertools.count() + + +def _save_tensors(tensors, path, metadata): + fd, temporary = tempfile.mkstemp(dir=os.path.dirname(path) or ".", prefix=".model-state-") + os.close(fd) + try: + comfy.utils.save_torch_file(tensors, temporary, {METADATA_KEY: metadata}) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def _read_tensors(patcher): + comfy.model_management.synchronize() + state_dict = patcher.model.state_dict() + if patcher.is_dynamic(): + patcher.flush_writes() + tensors = {name: torch.empty(value.shape, dtype=value.dtype) for name, value in state_dict.items()} + for name, value in state_dict.items(): + if not comfy.memory_management.read_tensor_file_slice_into(value, tensors[name]): + raise RuntimeError(f"Could not read model state tensor {name}") + else: + tensors = {name: value.to(device="cpu", copy=True) for name, value in state_dict.items()} + return tensors + + +class _StateValues: + # Patcher-owned save records must not own their patcher: a GC cycle can destroy + # the VBAR before the patcher's final writeback, particularly on Windows. + def __init__(self, tensors, metadata): + self.tensors = tensors + self.metadata = metadata + self.version = next(_STATE_VERSIONS) + self.patcher = lambda: None + self.saved_versions = {} + + def check_valid(self, patcher): + if self.tensors is None and (patcher is None or patcher.attachments.get("model_state") is not self): + raise RuntimeError("The model state was released without being written back; rerun the node that produced it") + + def read(self, patcher): + self.check_valid(patcher) + return self.tensors if self.tensors is not None else _read_tensors(patcher) + + def flush(self, patcher): + self.tensors = self.read(patcher) + + def remove_pending(self, path): + patcher = self.patcher() + if patcher is not None: + saves = patcher.attachments.get("model_state_saves", {}) + if saves.get(path) is self: + saves.pop(path) + + def write_pending(self, patcher): + saves = patcher.attachments.get("model_state_saves", {}) + paths = {path for path, values in saves.items() if values is self} + if not paths: + return + for path in paths: + saves.pop(path) # readback can itself reclaim RAM pins + try: + tensors = None + for path in tuple(paths): + if ModelState._saved(path)[1] is self: + if tensors is None: + tensors = self.read(patcher) + metadata = self.saved_versions[path][1] + _save_tensors(tensors, path, metadata) + self.saved_versions[path] = ModelState.file_version(path), metadata + paths.remove(path) + except Exception: + saves.update(dict.fromkeys(paths, self)) + raise + + +def _save_before_ram_unload(patcher): + for values in set(patcher.attachments.get("model_state_saves", {}).values()): + values.write_pending(patcher) + + +def _save_on_detach(patcher, unpatch_all): + if unpatch_all: + _save_before_ram_unload(patcher) class ModelState: - """A writable model's values as CPU tensors with its generation metadata, passed between nodes by value. + """A snapshot of a writable model. Compatible snapshots share one managed model.""" - Loading copies the values into the model's own backing, so a state's tensors - are never written in place and copies share them. Flushing reads the loaded - model's values back and drops it; a node that loads a state flushes or - releases it before the state reaches another node. - """ + def __init__(self, tensors=None, metadata=None, _values=None): + self._values = _values or _StateValues({} if tensors is None else tensors, {} if metadata is None else metadata) + self.patcher = self._values.patcher() + self._owner = lambda: None - def __init__(self, tensors=None, metadata=None): - self.tensors = {} if tensors is None else tensors - self.metadata = {} if metadata is None else metadata - self.patcher = None + @property + def tensors(self): + return self._values.tensors + + @property + def metadata(self): + return self._values.metadata + + @property + def owner(self): + return self._owner() + + @owner.setter + def owner(self, patcher): + self._owner = weakref.ref(patcher) if patcher is not None else lambda: None @classmethod def open(cls, path, create="if_missing"): - """Read the state file at path; a missing file is an empty state unless create is never, and always starts empty.""" + """Open the latest saved snapshot, including an unwritten resident save.""" + path = os.path.abspath(path) + _, saved = cls._saved(path) + if create != "always" and saved is not None: + metadata = saved.saved_versions[path][1] + state = cls(_values=saved).copy() + state._values.metadata = json.loads(metadata) + return state if create != "always" and os.path.exists(path): with safetensors.safe_open(path, "pt") as file: metadata = file.metadata() or {} @@ -43,61 +148,146 @@ def open(cls, path, create="if_missing"): raise FileNotFoundError(f"Model state file does not exist: {path}") return cls() - def save(self, path): - if self.tensors is None: - raise RuntimeError("The model state was released without being written back; rerun the node that produced it") - safetensors.torch.save_file(self.tensors, path, {METADATA_KEY: json.dumps(self.metadata)}) + @staticmethod + def file_version(path): + try: + stat = os.stat(path) + except FileNotFoundError: + return None + return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns + + @classmethod + def _saved(cls, path): + version = cls.file_version(path) + saved = SAVED_STATES.get(path) + if saved is not None and version != saved.saved_versions[path][0]: + saved.remove_pending(path) + saved.saved_versions.pop(path) + SAVED_STATES.pop(path) + saved = None + return version, saved + + @classmethod + def fingerprint(cls, path): + path = os.path.abspath(path) + version, saved = cls._saved(path) + return (saved.version, saved.saved_versions[path][1]) if saved is not None else version + + def save(self, path, writethrough=True): + self._values.check_valid(self.patcher) + path = os.path.abspath(path) + metadata = json.dumps(self.metadata) + if writethrough or self.patcher is None: + _save_tensors(self._values.read(self.patcher), path, metadata) + previous = SAVED_STATES.get(path) + if previous is not None and previous is not self._values: + previous.remove_pending(path) + previous.saved_versions.pop(path) + SAVED_STATES[path] = self._values + self._values.saved_versions[path] = self.file_version(path), metadata + if not writethrough and self.patcher is not None: + self.patcher.attachments.setdefault("model_state_saves", {})[path] = self._values + else: + self._values.remove_pending(path) + + def _comfy_cache_tensors(self): + if self.tensors is None and self.patcher is not None and not self.patcher.is_dynamic(): + return self.patcher.model.state_dict() + return self.tensors def copy(self): - if self.tensors is None: - raise RuntimeError("The model state was released without being written back; rerun the node that produced it") - return ModelState(dict(self.tensors), copy.deepcopy(self.metadata)) + self.flush() + state = ModelState(dict(self.tensors), copy.deepcopy(self.metadata)) + state.patcher, state.owner = self.patcher, self.owner + state._values.version = self._values.version + state._values.patcher = self._values.patcher + return state def load(self, model, device): - """Load model with this state's values, or zeros, and return it ready to run.""" - if comfy.memory_management.aimdo_enabled and not comfy.model_management.is_device_cpu(device): - # Dynamic VRAM pages the model from an anonymous file and writes it back there. - tensors, _ = comfy.utils.create_safetensors(model.state_dict()) - for name, value in self.tensors.items(): - comfy.memory_management.write_tensor_file_slice(tensors[name], value) - model.load_state_dict(tensors, assign=True) - self.patcher = comfy.model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu"), fast_disk=comfy.storage.anonymous_fast_disk()) - else: - model.to_empty(device="cpu") - for name, value in model.state_dict().items(): - if self.tensors: - value.copy_(self.tensors[name]) + """Select this snapshot in the owner's managed model and return it ready to run.""" + self._values.check_valid(self.patcher) + if self._values.saved_versions: + values = self.copy()._values + # Generation already holds a reference to this metadata. + values.metadata = self.metadata + self._values = values + patcher = self.patcher + owner = self.owner + if owner is not None: + while owner.parent is not None and owner.model is owner.parent.model: + owner = owner.parent + attached = owner.get_attachment("model_state_model") + if attached is None or attached[0]() is not owner.model: + attached = [weakref.ref(owner.model), None] + owner.set_attachments("model_state_model", attached) + patcher = attached[1] + if patcher is not self.patcher: + self.flush() + self.release() + previous = None + if patcher is not None: + previous = patcher.attachments.get("model_state") + if previous is not None and previous is not self._values: + previous.flush(patcher) + geometry = {k: (v.shape, v.dtype) for k, v in model.state_dict().items()} + if geometry != {k: (v.shape, v.dtype) for k, v in patcher.model.state_dict().items()} or device != patcher.load_device: + self.flush() + self.release() + comfy.model_management.unload_model_and_clones(patcher, all_devices=True) + patcher = None + resident = patcher is not None and previous is not None and previous.version == self._values.version + if patcher is not None: + if not resident: + patcher.unpatch_model(patcher.offload_device) + model = patcher.model + self._values.version = next(_STATE_VERSIONS) + if not resident: + dynamic = comfy.memory_management.aimdo_enabled and not comfy.model_management.is_device_cpu(device) + if dynamic: + tensors, _ = comfy.utils.create_safetensors(model.state_dict()) + for name, value in self.tensors.items(): + comfy.memory_management.write_tensor_file_slice(tensors[name], value) + model.load_state_dict(tensors, assign=True) + else: + model.to_empty(device="cpu") + for name, value in model.state_dict().items(): + if self.tensors: + value.copy_(self.tensors[name]) + else: + value.zero_() + if patcher is None: + if dynamic: + patcher = comfy.model_patcher.ModelPatcherDynamic(model, device, torch.device("cpu"), fast_disk=comfy.storage.anonymous_fast_disk()) else: - value.zero_() - self.patcher = comfy.model_patcher.ModelPatcher(model, device, torch.device("cpu")) - comfy.model_management.load_models_gpu([self.patcher], force_full_load=not self.patcher.is_dynamic()) - if self.patcher.is_dynamic(): - self.patcher.set_in_use_by_current_prompt(True) + patcher = comfy.model_patcher.ModelPatcher(model, device, torch.device("cpu")) + patcher.add_callback_with_key(comfy.model_patcher.CallbacksMP.ON_UNLOAD_RAM, "model_state", _save_before_ram_unload) + patcher.add_callback_with_key(comfy.model_patcher.CallbacksMP.ON_DETACH, "model_state", _save_on_detach) + self.patcher = patcher + self._values.patcher = weakref.ref(patcher) + patcher.attachments["model_state"] = self._values + if owner is not None: + attached[1] = patcher + comfy.model_management.load_models_gpu([patcher], force_full_load=not patcher.is_dynamic()) + if patcher.is_dynamic(): + patcher.set_in_use_by_current_prompt(True) + self._values.tensors = None return model def flush(self): - """Take the loaded model's values and drop it.""" - if self.patcher is None: - return - # The last forward may still be using the allocation graph's pages. - comfy.model_management.synchronize() - comfy.model_prefetch.cleanup_prefetch_queues() - state_dict = self.patcher.model.state_dict() - if self.patcher.is_dynamic(): - self.patcher.flush_writes() - self.tensors = {name: torch.empty(value.shape, dtype=value.dtype) for name, value in state_dict.items()} - for name, value in state_dict.items(): - if not comfy.memory_management.read_tensor_file_slice_into(value, self.tensors[name]): - raise RuntimeError(f"Could not read model state tensor {name}") - else: - self.tensors = {name: value.cpu() for name, value in state_dict.items()} - self.patcher = None + """Materialize an immutable CPU snapshot without unloading the model.""" + self._values.flush(self.patcher) def release(self): - """Drop a still loaded model, discarding its values; the metadata may already describe them, so the state is unusable.""" + """Discard an unwired result, leaving its shared model available for reuse.""" if self.patcher is None: return - if self.patcher.is_dynamic(): - self.patcher.discard_writes() + self._values.write_pending(self.patcher) + if self.patcher.attachments.get("model_state") is self._values: + self.patcher.attachments.pop("model_state") + if self.patcher.is_dynamic(): + self.patcher.discard_writes() + if self.tensors is None: + for path in self._values.saved_versions: + if SAVED_STATES.get(path) is self._values: + SAVED_STATES.pop(path) self.patcher = None - self.tensors = None diff --git a/comfy/patcher_extension.py b/comfy/patcher_extension.py index 189ee84caa3..606ff8096ba 100644 --- a/comfy/patcher_extension.py +++ b/comfy/patcher_extension.py @@ -6,6 +6,7 @@ class CallbacksMP: ON_MATCH_MULTIGPU_CLONES = "on_match_multigpu_clones" ON_LOAD = "on_load_after" ON_DETACH = "on_detach_after" + ON_UNLOAD_RAM = "on_unload_ram" ON_CLEANUP = "on_cleanup" ON_PRE_RUN = "on_pre_run" ON_PREPARE_STATE = "on_prepare_state" diff --git a/comfy/sd.py b/comfy/sd.py index d3ea95672f5..f22af2fb008 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -478,7 +478,7 @@ def get_key_patches(self): return self.patcher.get_key_patches() def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_k=50, top_p=0.95, min_p=0.0, repetition_penalty=1.0, seed=None, presence_penalty=0.0, mtp=True, state=None): - """Generate tokens into a model state, continuing the generation it holds. The state stays loaded until flushed, or released after the node.""" + """Generate tokens into a model state, reusing the CLIP's managed state model.""" self.cond_stage_model.reset_clip_options() self.load_model(tokens) @@ -488,7 +488,8 @@ def generate(self, tokens, do_sample=True, max_length=256, temperature=1.0, top_ if state is None: state = comfy.model_state.ModelState() - model_management.register_ephemeral_model(state) + model_management.register_ephemeral_model(state) + state.owner = self.patcher with model_management.cuda_device_context(device), comfy.ops.use_quantized_matmul(self.cond_stage_model, device): self.cond_stage_model.set_clip_options({"model_state": state}) return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty, mtp=mtp) diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index 1701d954944..4fde6d6fc90 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -200,6 +200,13 @@ def __init__(self, caches, extra_caches=()): def get_dynamic_vram__units(self): return self.layers, [] + def load(self, state, device): + model = state.load(self, device) + model.caches, model.extra_caches = self.caches, self.extra_caches + for cache, layer in zip(model.caches + model.extra_caches, model.layers, strict=True): + cache.storage = layer + return model + def checkpoint_state(self): return [cache.checkpoint_state() for cache in self.caches + self.extra_caches] @@ -1294,7 +1301,7 @@ def generate(self, embeds=None, do_sample=True, max_length=256, temperature=1.0, generator = torch.Generator(device=device).manual_seed(seed) if do_sample else None cache = past_key_values = self.init_kv_cache(embeds.shape[0], max_cache_len, device, execution_dtype) if isinstance(cache, KVCacheModel): - cache = model_state.load(cache, device) + cache = cache.load(model_state, device) self.restore_generation_state(state, cache, generator) past_key_values = cache.caches diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index f928295b9bf..6edfdcf8e52 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -959,7 +959,7 @@ def _generate_mtp(self, embeds, max_length, stop_tokens, mtp, sampling=None, pos past_length = state.get("length", 0) cap = self.cache_capacity(past_length + embeds.shape[1] + max_length + 7, state) generator = torch.Generator(device=device).manual_seed(sampling["seed"]) if sampling is not None else None - cache = model_state.load(self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True), device) + cache = self.init_kv_cache(embeds.shape[0], cap, device, dt, mtp=True).load(model_state, device) self.restore_generation_state(state, cache, generator) pkv = cache.caches # repair window: drafting ahead plus a near-full rollback diff --git a/comfy_extras/nodes_model_state.py b/comfy_extras/nodes_model_state.py index 5463b420484..31c3f496623 100644 --- a/comfy_extras/nodes_model_state.py +++ b/comfy_extras/nodes_model_state.py @@ -37,7 +37,7 @@ def fingerprint_inputs(cls, filename, create): if create == "always": return float("NaN") path = state_path(filename) - return os.stat(path).st_mtime_ns if os.path.exists(path) else None + return comfy.model_state.ModelState.fingerprint(path) @classmethod def execute(cls, filename, create) -> io.NodeOutput: @@ -56,13 +56,14 @@ def define_schema(cls) -> io.Schema: inputs=[ io.ModelState.Input("state"), io.String.Input("filename", default="model_states/ComfyUI.safetensors"), + io.Boolean.Input("writethrough", default=True, optional=True, tooltip="Write the file now. When disabled, keep the saved state resident and write it when its model is evicted from RAM."), ], outputs=[io.ModelState.Output(display_name="state", tooltip="The saved state, for chaining or ending a loop.")], ) @classmethod - def execute(cls, state, filename) -> io.NodeOutput: - state.save(state_path(filename)) + def execute(cls, state, filename, writethrough=True) -> io.NodeOutput: + state.save(state_path(filename), writethrough=writethrough) return io.NodeOutput(state) diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py index dba50f8375d..1dfc8d79bb0 100644 --- a/comfy_extras/nodes_textgen.py +++ b/comfy_extras/nodes_textgen.py @@ -2,6 +2,7 @@ from typing_extensions import override import comfy.model_state +import comfy.model_management class TextGenerate(io.ComfyNode): @classmethod @@ -62,6 +63,9 @@ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=F if not use_default_template or (state is not None and state.metadata): system_prompt = "" state = comfy.model_state.ModelState() if state is None else state.copy() + dynprompt = cls.hidden.dynprompt + if not any(value == [cls.hidden.unique_id, 2] for node_id in dynprompt.all_node_ids() for value in dynprompt.get_node(node_id)["inputs"].values()): + comfy.model_management.register_ephemeral_model(state) tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt) # Get sampling parameters from dynamic combo @@ -88,11 +92,6 @@ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=F mtp=mtp, state=state, ) - # the output is written back only when something consumes it; otherwise it is released after the node - dynprompt = cls.hidden.dynprompt - if any(value == [cls.hidden.unique_id, 2] for node_id in dynprompt.all_node_ids() for value in dynprompt.get_node(node_id)["inputs"].values()): - state.flush() - generated_text = clip.decode(generated_ids) # ends reasoning; without one, reasoning opened by the output or prompt was cut off by max_length diff --git a/tests-unit/comfy_test/persistent_state_cuda.py b/tests-unit/comfy_test/persistent_state_cuda.py new file mode 100644 index 00000000000..a61947aa6ee --- /dev/null +++ b/tests-unit/comfy_test/persistent_state_cuda.py @@ -0,0 +1,155 @@ +"""Bounded persistent-state lifecycle checks; run on a disposable CUDA host.""" +import os +import gc +from pathlib import Path +import sys +import tempfile + +os.environ.setdefault("PYTORCH_ALLOC_CONF", "backend:cudaMallocAsync") +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +import safetensors.torch +import torch +import comfy_aimdo.control as aimdo + +assert aimdo.init("cuda") and aimdo.init_device(0) +import comfy.ldm.wan.model_animate2 # noqa: F401 -- initialize the registry before the patcher +from comfy import memory_management, model_management as mm, model_patcher, model_prefetch, model_state, ops +import comfy.system_memory + +memory_management.aimdo_enabled = True +device = torch.device("cuda", 0) + + +class Cache(torch.nn.Module): + comfy_writable = True + + def __init__(self): + super().__init__() + self.k = ops.WritableWeight((512, 1024), torch.float32) + self.v = ops.WritableWeight((512, 1024), torch.float32) + + +def write(model, value): + queue = model_prefetch.make_prefetch_queue([model], device, {"prefetch_dynamic_vbars": True}) + def core(): + with model.k.access(device) as (k, _), model.v.access(device) as (v, _): + k.fill_(value) + v.fill_(-value) + model_prefetch.prefetch_queue_pop(queue, device, module=model, core=core) + model_prefetch.prefetch_queue_pop(queue, device, None) + + +with torch.inference_mode(), tempfile.TemporaryDirectory() as directory: + owner = model_patcher.ModelPatcher(torch.nn.Linear(1, 1), torch.device("cpu"), torch.device("cpu")) + path = os.path.join(directory, "state.safetensors") + first = model_state.ModelState() + first.owner = owner + cache = first.load(Cache(), device) + patcher = first.patcher + patch_uuid = patcher.patches_uuid + write(cache, 3) + address = cache.k._v[1] + first.metadata["turn"] = 1 + first.save(path, writethrough=False) + assert first.tensors is None and not os.path.exists(path) + reopened = model_state.ModelState.open(path, "never") + assert reopened._values.version == first._values.version and reopened.patcher is first.patcher + del reopened + second = first.copy() + assert second.load(Cache(), device) is cache + assert second.patcher is patcher and cache.k._v[1] == address + assert not os.path.exists(path) + write(cache, 5) + second.metadata["turn"] = 2 + second.save(path, writethrough=False) + assert not os.path.exists(path) + # A VRAM eviction writes into RAM, but does not commit the named save. + patcher._vbar_get().set_watermark(0) + assert patcher.loaded_size() == 0 and patcher.loaded_ram_size() > 0 + assert not os.path.exists(path) + patcher.partially_unload_ram(1e32) + assert patcher.loaded_ram_size() == 0 + assert second.tensors is None + assert safetensors.torch.load_file(path)["k.weight"].eq(5).all() + assert not patcher.get_attachment("model_state_saves") + second.flush() + assert second.tensors["v.weight"].eq(-5).all() + assert first.load(Cache(), device) is cache + first.flush() + assert first.tensors["k.weight"].eq(3).all() + assert second.load(Cache(), device) is cache + second.flush() + assert second.tensors["k.weight"].eq(5).all() + assert patcher.patches_uuid == patch_uuid and cache.current_weight_patches_uuid == patch_uuid + # Explicit unload must commit a still-resident dirty state too. + third = second.copy() + third.load(Cache(), device) + write(cache, 7) + cache.k.writeback(cache.k._v_weight) + third.save(path, writethrough=False) + queue = model_prefetch.make_prefetch_queue([cache], device, {"prefetch_dynamic_vbars": True}) + model_prefetch.prefetch_queue_pop(queue, device, cache) + prefetched = queue[0] + patcher.partially_unload_ram(1e32) + assert queue in model_prefetch.PREFETCH_QUEUES and queue[0] is prefetched + model_prefetch.prefetch_queue_pop(queue, device, None) + assert safetensors.torch.load_file(path)["k.weight"].eq(7).all() + # Explicit unload must also commit a still-resident snapshot. + fourth = third.copy() + fourth.load(Cache(), device) + write(cache, 9) + fourth.save(path, writethrough=False) + mm.unload_model_and_clones(patcher) + assert safetensors.torch.load_file(path)["k.weight"].eq(9).all() + collected = model_state.ModelState() + orphan = collected.load(Cache(), device) + write(orphan, 11) + collected.save(path, writethrough=False) + del collected, orphan + gc.collect() + assert safetensors.torch.load_file(path)["k.weight"].eq(11).all() + assert safetensors.torch.load_file(path)["v.weight"].eq(-11).all() + pressure = model_state.ModelState() + pressured = pressure.load(Cache(), device) + write(pressured, 13) + pressure.patcher.partially_unload_ram(1e32) + pressured.k.writeback(pressured.k._v_weight) + assert pressure.patcher.loaded_ram_size() == pressured.k.weight.nbytes + pressure.save(path, writethrough=False) + pressure.patcher.set_in_use_by_current_prompt(False) + mm.reset_cast_buffers() + available = comfy.system_memory.virtual_memory_available + comfy.system_memory.virtual_memory_available = lambda: 0 + try: + # Reclaim the first RAM pin while the second weight's native pageout is running. + pressure.patcher._vbar_get().set_watermark(0) + finally: + comfy.system_memory.virtual_memory_available = available + assert safetensors.torch.load_file(path)["k.weight"].eq(13).all() + assert safetensors.torch.load_file(path)["v.weight"].eq(-13).all() + assert not pressure.patcher.get_attachment("model_state_saves") + assert pressure.tensors is None + mm.unload_model_and_clones(pressure.patcher) + other_owner = model_patcher.ModelPatcher(torch.nn.Linear(1, 1), torch.device("cpu"), torch.device("cpu")) + moved = model_state.ModelState() + moved.owner = other_owner + write(moved.load(Cache(), device), 17) + source_patcher = moved.patcher + moved.owner = owner + write(moved.load(Cache(), device), 19) + assert moved.patcher is patcher + other = model_state.ModelState() + other.owner = other_owner + write(other.load(Cache(), device), 23) + moved.flush() + assert moved.tensors["k.weight"].eq(19).all() + assert moved.tensors["v.weight"].eq(-19).all() + unused = moved.copy() + signature = cache.k._v_signature + unused.release() + assert cache.k._v_signature == signature and signature is not None + assert patcher.get_attachment("model_state") is moved._values + mm.unload_model_and_clones(source_patcher) + mm.unload_model_and_clones(patcher) + print("PERSISTENT_STATE_CUDA_PASS", flush=True) # noqa: T201 diff --git a/tests-unit/comfy_test/qwen_writable_cache_cuda.py b/tests-unit/comfy_test/qwen_writable_cache_cuda.py index 783f6a1085c..d5c83fc2fbf 100644 --- a/tests-unit/comfy_test/qwen_writable_cache_cuda.py +++ b/tests-unit/comfy_test/qwen_writable_cache_cuda.py @@ -98,6 +98,7 @@ def forward(*args, **kwargs): baseline = None for mode in ("hard", "resident", "ram", "disk", "evict"): mm.DISABLE_PINNED_MEMORY = mode == "disk" + states.clear() outputs.clear() captured.clear() # The resident run is saved to a named file; the others stay anonymous. @@ -105,22 +106,25 @@ def forward(*args, **kwargs): state = _ModeState(mode) mm.register_ephemeral_model(state) ids = model.generate(embeds.clone(), do_sample=True, seed=71, max_length=24, mtp=mtp, model_state=state) - # A consumer flushes the values back to CPU tensors, which drops the loaded model. + # A snapshot does not unload its managed model. dead = [weakref.ref(s.patcher) for s in states] + [weakref.ref(s.patcher.model) for s in states] states.clear() + loaded_patcher = state.patcher state.flush() - assert state.patcher is None + assert state.patcher is loaded_patcher + del loaded_patcher saved = state.tensors if mode == "hard": # A plain model state restores its tensors into device memory. reopened = state.load(model.init_kv_cache(1, state.metadata["capacity"], device, torch.bfloat16, mtp=bool(mtp)), device) assert all(torch.equal(v.cpu(), saved[k]) for k, v in reopened.state_dict().items()) del reopened + state.flush() state.release() if path is not None: state.save(path) assert all(torch.equal(v, saved[k]) for k, v in safetensors.torch.load_file(path).items()) - # Node end: the flushed patcher and its model must both be dead with nothing else holding them. + # An ephemeral state without an owner still releases its model at node end. mm.reset_cast_buffers() mm.cleanup_ephemeral_models() assert not mm.EPHEMERAL_MODELS and all(ref() is None for ref in dead), (mode, dead) @@ -131,7 +135,7 @@ def forward(*args, **kwargs): turn = source.copy() assert turn.metadata == source.metadata and turn.metadata is not source.metadata assert all(torch.equal(v, saved[k]) for k, v in turn.tensors.items()) - restored = turn.load(model.init_kv_cache(1, turn.metadata["capacity"], device, torch.bfloat16, mtp=bool(mtp)), device) + restored = model.init_kv_cache(1, turn.metadata["capacity"], device, torch.bfloat16, mtp=bool(mtp)).load(turn, device) restored.restore_state(turn.metadata["caches"]) resumed = original_forward(None, embeds=continuation, attention_mask=None, past_key_values=restored.caches)[0].clone() prefetch.cleanup_prefetch_queues() @@ -150,6 +154,42 @@ def forward(*args, **kwargs): if mode in ("ram", "disk"): assert not captured print("QWEN_WRITABLE_PASS", attention, length, graphs, queue, mtp, mode, len(ids), len(captured), flush=True) # noqa: T201 + if attention == "bf16" and mtp == "3": + mode = "resident" + states.clear() + outputs.clear() + mm.DISABLE_PINNED_MEMORY = False + first = model_state.ModelState() + first.owner = patcher + model.generate(embeds.clone(), do_sample=True, seed=71, max_length=24, mtp=mtp, model_state=first) + managed = first.patcher + turn = first.copy() + reference = model_state.ModelState(dict(turn.tensors), json.loads(json.dumps(turn.metadata))) + prefetch.cleanup_prefetch_queues() + mm.reset_cast_buffers() + mm.cleanup_ephemeral_models() + assert turn.patcher is managed and patcher.get_attachment("model_state_model")[1] is managed + runs = [] + for current in (turn, reference): + outputs.clear() + captured.clear() + if current is reference: + mm.register_ephemeral_model(current) + ids = model.generate(continuation.clone(), do_sample=True, seed=71, max_length=8, mtp=mtp, model_state=current) + if current is turn: + assert current.patcher is managed + if graphs: + assert captured + runs.append((ids, list(outputs))) + prefetch.cleanup_prefetch_queues() + mm.reset_cast_buffers() + mm.cleanup_ephemeral_models() + assert runs[0][0] == runs[1][0] and len(runs[0][1]) == len(runs[1][1]) + assert all(torch.equal(a, b) for a, b in zip(runs[0][1], runs[1][1])) + turn.release() + mm.unload_model_and_clones(managed) + assert not mm.EPHEMERAL_MODELS + print("QWEN_PERSISTENT_PASS", graphs, flush=True) # noqa: T201 pending = torch.nn.Sequential(ops.WritableWeight((1,), torch.float32)) pending.comfy_writable = True output = torch.empty(2 ** 20, device=device) diff --git a/tests-unit/comfy_test/tensor_file_test.py b/tests-unit/comfy_test/tensor_file_test.py index 6c343ca6d4c..509c0648a3d 100644 --- a/tests-unit/comfy_test/tensor_file_test.py +++ b/tests-unit/comfy_test/tensor_file_test.py @@ -1,5 +1,7 @@ import os +import gc import sys +import weakref import pytest import safetensors.torch @@ -11,6 +13,7 @@ cli_args.cpu = True import comfy.model_state # noqa: E402 +import comfy.model_patcher # noqa: E402 from comfy.memory_management import read_tensor_file_slice_into, write_tensor_file_slice # noqa: E402 from comfy.utils import create_safetensors # noqa: E402 @@ -91,7 +94,7 @@ def test_model_state_persists_values_and_metadata(tmp_path): model.v.fill_(5.0) state.metadata["position"] = 7 state.flush() - assert state.patcher is None + assert state.patcher.model is model state.save(path) saved = safetensors.torch.load_file(path) @@ -160,5 +163,380 @@ def test_model_state_copies_are_independent(tmp_path): first.save(str(tmp_path / "released.safetensors")) # A written back state stays usable. third.flush() + expected = third.tensors["k"].clone() third.release() - assert torch.equal(third.copy().load(_KV(), cpu).k, model.k) + assert torch.equal(third.copy().load(_KV(), cpu).k, expected) + + +def test_shared_model_survives_nodes_and_snapshot_switches(): + cpu = torch.device("cpu") + owner = comfy.model_patcher.ModelPatcher(_KV(), cpu, cpu) + cloned_owner = owner.clone() + first = comfy.model_state.ModelState() + first.owner = owner + model = first.load(_KV(), cpu) + patcher = first.patcher + model.k.fill_(2) + second = first.copy() + assert second.load(_KV(), cpu) is model + assert second.patcher is patcher + model.k.fill_(3) + other = comfy.model_state.ModelState() + other.owner = cloned_owner + assert other.load(_KV(), cpu) is model + assert other.patcher is patcher + assert not model.k.count_nonzero() + assert first.load(_KV(), cpu) is model + assert torch.equal(model.k, torch.full((4, 8), 2.0)) + assert second.load(_KV(), cpu) is model + assert torch.equal(model.k, torch.full((4, 8), 3.0)) + late_clone = owner.clone() + old_patcher = weakref.ref(patcher) + for state in (first, second, other): + state.release() + replacement = comfy.model_state.ModelState() + replacement.owner = late_clone + assert replacement.load(_KV(rows=1), cpu).k.shape == (1, 8) + assert late_clone.get_attachment("model_state_model") is owner.get_attachment("model_state_model") + del patcher, model + gc.collect() + assert old_patcher() is None + + +def test_deferred_save_uses_resident_snapshot_and_flushes_on_eviction(tmp_path): + cpu = torch.device("cpu") + path = str(tmp_path / "deferred.safetensors") + state = comfy.model_state.ModelState() + model = state.load(_KV(), cpu) + model.k.fill_(7) + state.metadata["position"] = 9 + state.save(path, writethrough=False) + assert not os.path.exists(path) + assert state.tensors is None + reopened = comfy.model_state.ModelState.open(path, "never") + assert reopened._values.version == state._values.version and reopened.patcher is state.patcher + assert reopened.tensors is not state.tensors and state.tensors["k"].eq(7).all() + state.patcher.unpin_all_weights() + assert safetensors.torch.load_file(path)["k"].eq(7).all() + assert not state.patcher.get_attachment("model_state_saves") + assert comfy.model_state.ModelState.open(path).metadata == {"position": 9} + + +def test_deferred_save_keeps_latest_snapshot_and_external_edits_win(tmp_path): + cpu = torch.device("cpu") + path = str(tmp_path / "latest.safetensors") + first = comfy.model_state.ModelState() + model = first.load(_KV(), cpu) + model.k.fill_(1) + first.save(path, writethrough=False) + second = first.copy() + model = second.load(_KV(), cpu) + model.k.fill_(2) + second.save(path, writethrough=False) + assert not os.path.exists(path) + second.patcher.unpin_all_weights() + assert safetensors.torch.load_file(path)["k"].eq(2).all() + safetensors.torch.save_file({"k": torch.ones(4, 8)}, path, {comfy.model_state.METADATA_KEY: '{"external": true}'}) + assert comfy.model_state.ModelState.open(path).metadata == {"external": True} + + +def test_deferred_save_survives_state_collection(tmp_path): + cpu = torch.device("cpu") + path = str(tmp_path / "collected.safetensors") + state = comfy.model_state.ModelState() + model = state.load(_KV(), cpu) + model.k.fill_(11) + state.save(path, writethrough=False) + assert not os.path.exists(path) + del state, model + gc.collect() + assert safetensors.torch.load_file(path)["k"].eq(11).all() + + +@pytest.mark.parametrize("resident_target", [False, True]) +def test_live_state_preserves_values_when_changing_owner(resident_target): + cpu = torch.device("cpu") + owner = comfy.model_patcher.ModelPatcher(_KV(), cpu, cpu) + target_owner = comfy.model_patcher.ModelPatcher(_KV(), cpu, cpu) + state = comfy.model_state.ModelState() + state.owner = owner + source = state.load(_KV(), cpu) + source.k.fill_(2) + state.metadata["position"] = 3 + if resident_target: + target_state = comfy.model_state.ModelState() + target_state.owner = target_owner + target = target_state.load(_KV(), cpu) + target.k.fill_(9) + state.owner = target_owner + moved = state.load(_KV(), cpu) + assert moved is not source and moved.k.eq(2).all() + assert state.metadata == {"position": 3} + moved.k.fill_(4) + assert source.k.eq(2).all() + if resident_target: + assert moved is target + assert target_state.load(_KV(), cpu).k.eq(9).all() + + +def test_incompatible_live_geometry_preserves_state(): + cpu = torch.device("cpu") + state = comfy.model_state.ModelState() + model = state.load(_KV(), cpu) + model.k.fill_(2) + model.v.fill_(3) + state.metadata["position"] = 4 + with pytest.raises(RuntimeError): + state.load(_KV(rows=1), cpu) + assert state.tensors["k"].eq(2).all() and state.tensors["v"].eq(3).all() + restored = state.load(_KV(), cpu) + assert restored.k.eq(2).all() and restored.v.eq(3).all() + assert state.metadata == {"position": 4} + + +def test_independent_model_override_does_not_inherit_state_model(): + cpu = torch.device("cpu") + owner = comfy.model_patcher.ModelPatcher(_KV(), cpu, cpu) + state = comfy.model_state.ModelState() + state.owner = owner + source = state.load(_KV(), cpu) + source.k.fill_(2) + independent = comfy.model_patcher.ModelPatcher(_KV(), cpu, cpu) + clone = owner.clone(model_override=independent.get_clone_model_override()) + other = comfy.model_state.ModelState() + other.owner = clone + model = other.load(_KV(), cpu) + assert other.patcher is not state.patcher and model is not source + assert not model.k.count_nonzero() + model.k.fill_(3) + assert source.k.eq(2).all() + assert state.load(_KV(), cpu) is source + + +@pytest.mark.parametrize("trigger", ["eviction", "collection"]) +def test_external_replacement_cancels_pending_save(tmp_path, trigger): + path = str(tmp_path / "replaced.safetensors") + state = comfy.model_state.ModelState() + model = state.load(_KV(), torch.device("cpu")) + model.k.fill_(7) + state.metadata["position"] = 1 + state.save(path) + state.save(path, writethrough=False) + stat = os.stat(path) + replacement = str(tmp_path / "external.safetensors") + safetensors.torch.save_file( + {"k": torch.full_like(model.k, 99), "v": torch.full_like(model.v, -99)}, + replacement, {comfy.model_state.METADATA_KEY: '{"position": 2}'}) + os.utime(replacement, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + os.replace(replacement, path) + assert os.stat(path).st_mtime_ns == stat.st_mtime_ns + assert os.stat(path).st_size == stat.st_size + external = (tmp_path / "replaced.safetensors").read_bytes() + if trigger == "eviction": + state.patcher.unpin_all_weights() + assert not state.patcher.get_attachment("model_state_saves") + else: + del state, model + gc.collect() + assert (tmp_path / "replaced.safetensors").read_bytes() == external + opened = comfy.model_state.ModelState.open(path, "never") + assert opened.metadata == {"position": 2} and opened.tensors["k"].eq(99).all() + + +@pytest.mark.parametrize("writethrough", [False, True]) +def test_loading_saved_state_does_not_mutate_saved_snapshot(tmp_path, writethrough): + path = str(tmp_path / "snapshot.safetensors") + state = comfy.model_state.ModelState() + model = state.load(_KV(), torch.device("cpu")) + model.k.fill_(7) + state.metadata["turn"] = {"position": 9} + state.save(path, writethrough=writethrough) + state.metadata["turn"]["position"] = 10 + metadata = state.metadata + model = state.load(_KV(), torch.device("cpu")) + assert state.metadata is metadata + model.k.fill_(8) + state.metadata["turn"]["position"] = 11 + opened = comfy.model_state.ModelState.open(path, "never") + assert opened.metadata == {"turn": {"position": 9}} + assert opened.tensors["k"].eq(7).all() + opened.load(_KV(), torch.device("cpu")).k.fill_(12) + opened.metadata["turn"]["position"] = 13 + reopened = comfy.model_state.ModelState.open(path, "never") + assert reopened.metadata == {"turn": {"position": 9}} and reopened.tensors["k"].eq(7).all() + + +@pytest.mark.parametrize("prior_deferred", [False, True]) +def test_failed_writethrough_preserves_saved_catalog(tmp_path, monkeypatch, prior_deferred): + path = str(tmp_path / "failed.safetensors") + cpu = torch.device("cpu") + if prior_deferred: + previous = comfy.model_state.ModelState() + previous.load(_KV(), cpu).k.fill_(1) + previous.save(path, writethrough=False) + state = comfy.model_state.ModelState() + state.load(_KV(), cpu).k.fill_(2) + + def fail_save(*args, **kwargs): + raise OSError("injected save failure") + + with monkeypatch.context() as patch: + patch.setattr(comfy.utils, "save_torch_file", fail_save) + with pytest.raises(OSError, match="injected save failure"): + state.save(path) + assert not os.path.exists(path) and not state.patcher.get_attachment("model_state_saves") + if prior_deferred: + assert previous.patcher.get_attachment("model_state_saves")[path] is previous._values + assert comfy.model_state.ModelState.open(path, "never").tensors["k"].eq(1).all() + previous.patcher.unpin_all_weights() + assert safetensors.torch.load_file(path)["k"].eq(1).all() + else: + with pytest.raises(FileNotFoundError): + comfy.model_state.ModelState.open(path, "never") + + +@pytest.mark.parametrize("writethrough", [False, True]) +@pytest.mark.parametrize("trigger", ["eviction", "release"]) +def test_save_release_does_not_retain_readback_or_catalog(tmp_path, writethrough, trigger): + paths = [str(tmp_path / name) for name in ("readback.safetensors", "alias.safetensors")] + state = comfy.model_state.ModelState() + state.load(_KV(), torch.device("cpu")).k.fill_(7) + for path in paths: + state.save(path, writethrough=writethrough) + assert state.tensors is None + if trigger == "eviction": + state.patcher.unpin_all_weights() + assert state.tensors is None and not state.patcher.get_attachment("model_state_saves") + state.release() + assert state.tensors is None and state.patcher is None + for path in paths: + assert path not in comfy.model_state.SAVED_STATES + opened = comfy.model_state.ModelState.open(path, "never") + assert opened.patcher is None and opened.tensors["k"].eq(7).all() + + +@pytest.mark.parametrize("writethrough", [False, True]) +def test_metadata_resave_changes_fingerprint_without_changing_opened_snapshot(tmp_path, writethrough): + path = str(tmp_path / "metadata.safetensors") + state = comfy.model_state.ModelState() + state.load(_KV(), torch.device("cpu")).k.fill_(7) + state.metadata["turn"] = {"position": 1} + state.save(path, writethrough=writethrough) + opened = comfy.model_state.ModelState.open(path, "never") + before = comfy.model_state.ModelState.fingerprint(path) + state.metadata["turn"]["position"] = 2 + assert comfy.model_state.ModelState.fingerprint(path) == before + state.save(path, writethrough=writethrough) + after = comfy.model_state.ModelState.fingerprint(path) + assert after != before + assert opened.metadata == {"turn": {"position": 1}} and opened.tensors["k"].eq(7).all() + assert comfy.model_state.ModelState.open(path, "never").metadata == {"turn": {"position": 2}} + state.patcher.unpin_all_weights() + assert comfy.model_state.ModelState.fingerprint(path) == after + + +@pytest.mark.parametrize("canceled", [False, True]) +def test_partial_write_failure_retries_only_unfinished_destination(tmp_path, monkeypatch, canceled): + paths = [str(tmp_path / name) for name in ("first.safetensors", "second.safetensors")] + state = comfy.model_state.ModelState() + model = state.load(_KV(), torch.device("cpu")) + model.k.fill_(7) + for path in paths: + state.save(path) + state.save(path, writethrough=False) + save = comfy.model_state._save_tensors + file_version = comfy.model_state.ModelState.file_version + finished, calls = [], [] + failing = True + + def replace_first(path): + if not finished: + save({"k": torch.full_like(model.k, 99), "v": model.v}, path, + "{}") + finished.append(path) + return file_version(path) + + def fail_partial_save(tensors, path, metadata=None): + with open(path, "wb") as file: + file.write(b"partial safetensors write") + raise OSError("injected second destination failure") + + def save_with_failure(tensors, path, metadata=None): + calls.append(path) + if failing and finished: + with monkeypatch.context() as patch: + patch.setattr(comfy.utils, "save_torch_file", fail_partial_save) + save(tensors, path, metadata) + save(tensors, path, metadata) + finished.append(path) + + if canceled: + monkeypatch.setattr(comfy.model_state.ModelState, "file_version", staticmethod(replace_first)) + monkeypatch.setattr(comfy.model_state, "_save_tensors", save_with_failure) + with pytest.raises(OSError, match="injected second destination failure"): + state.patcher.unpin_all_weights() + unfinished = calls[-1] + assert unfinished != finished[0] + assert state.patcher.get_attachment("model_state_saves") == {unfinished: state._values} + assert not list(tmp_path.glob(".model-state-*")) + completed = safetensors.torch.load_file(finished[0])["k"].clone() + assert completed.eq(99 if canceled else 7).all() + failing = False + calls.clear() + state.patcher.unpin_all_weights() + assert calls == [unfinished] and not state.patcher.get_attachment("model_state_saves") + assert torch.equal(safetensors.torch.load_file(finished[0])["k"], completed) + assert safetensors.torch.load_file(unfinished)["k"].eq(7).all() + + +def test_cpu_cache_tensor_accounting_before_and_after_flush(): + state = comfy.model_state.ModelState() + model = state.load(_KV(), torch.device("cpu")) + resident = state._comfy_cache_tensors() + expected = model.k.nbytes + model.v.nbytes + assert state.tensors is None + assert sum(value.nbytes for value in resident.values()) == expected + assert resident["k"].data_ptr() == model.k.data_ptr() + state.flush() + snapshot = state._comfy_cache_tensors() + assert snapshot is state.tensors + assert sum(value.nbytes for value in snapshot.values()) == expected + assert all(value.data_ptr() != resident[name].data_ptr() for name, value in snapshot.items()) + + +def test_snapshot_reuse_does_not_change_patcher_identity(monkeypatch): + cpu = torch.device("cpu") + first = comfy.model_state.ModelState() + model = first.load(_KV(), cpu) + model.k.fill_(3) + patcher = first.patcher + patch_uuid = patcher.patches_uuid + version = first._values.version + second = first.copy() + + def unexpected_restore(*args, **kwargs): + pytest.fail("The resident snapshot should not be restored") + + with monkeypatch.context() as patch: + patch.setattr(model, "to_empty", unexpected_restore) + assert second.load(_KV(), cpu) is model + assert first._values.version == version and second._values.version != version + model.k.fill_(5) + assert first.load(_KV(), cpu) is model and model.k.eq(3).all() + assert second.load(_KV(), cpu) is model and model.k.eq(5).all() + assert patcher.patches_uuid == patch_uuid + assert model.current_weight_patches_uuid == patch_uuid + + +def test_releasing_unused_copy_does_not_discard_active_writes(monkeypatch): + state = comfy.model_state.ModelState() + state.load(_KV(), torch.device("cpu")).k.fill_(7) + unused = state.copy() + discarded = [] + monkeypatch.setattr(state.patcher, "is_dynamic", lambda: True) + monkeypatch.setattr(state.patcher, "discard_writes", lambda: discarded.append(True), raising=False) + unused.release() + assert not discarded + assert state.patcher.get_attachment("model_state") is state._values + state.release() + assert discarded == [True]