From af7e0eda9128fb8fb7c66f6ce3d0e3dd1587b9fc Mon Sep 17 00:00:00 2001 From: Comfy Org PR Bot Date: Wed, 4 Nov 2026 10:10:35 +0900 Subject: [PATCH 1/8] Bump comfyui-frontend-package to 1.58.5 (#17442) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3f5b0ac2b87..603f7e00d4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -comfyui-frontend-package==1.57.6 +comfyui-frontend-package==1.58.5 comfyui-workflow-templates==0.11.101 comfyui-embedded-docs==0.5.16 torch From 7027f2570e918c0a6da102ead29fa050ee98eefe Mon Sep 17 00:00:00 2001 From: comfyanonymous <121283862+comfyanonymous@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:59:21 -0800 Subject: [PATCH 2/8] Add better error for when pixel_space VAE wrongly selected. (#17518) --- execution.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/execution.py b/execution.py index a8918ada17a..0d50df2fdb4 100644 --- a/execution.py +++ b/execution.py @@ -650,6 +650,8 @@ async def await_completion(): comfy.model_management.unload_all_models() elif isinstance(ex, RuntimeError) and ("mat1 and mat2 shapes" in str(ex)) and "Sampler" in class_type: tips = "\n\nTIPS: If you have any \"Load CLIP\" or \"*CLIP Loader\" nodes in your workflow connected to this sampler node make sure the correct file(s) and type is selected." + elif isinstance(ex, TypeError) and "Cannot handle this data type: (1, 1," in str(ex) and "|u1" in str(ex): + tips = "\n\nTIPS: This can happen when 'pixel_space' is selected in the Load VAE node. Select the VAE that matches your model instead." error_details = { "node_id": real_node_id, From e8f3a927908b7013a9d566ffe48a5f9a067120ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Sepp=C3=A4nen?= <40791699+kijai@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:20:26 +0200 Subject: [PATCH 3/8] Use adaptive slicing in seedvr2 decode (#17508) --- comfy/ldm/seedvr/constants.py | 10 ++ comfy/ldm/seedvr/vae.py | 164 ++++++++++++++---- comfy/model_prefetch.py | 3 + comfy/sd.py | 2 + .../comfy_test/test_seedvr2_internals.py | 67 ++++++- 5 files changed, 210 insertions(+), 36 deletions(-) diff --git a/comfy/ldm/seedvr/constants.py b/comfy/ldm/seedvr/constants.py index 98fb5057b5a..facefe69285 100644 --- a/comfy/ldm/seedvr/constants.py +++ b/comfy/ldm/seedvr/constants.py @@ -21,6 +21,16 @@ SEEDVR2_VAE_CACHE_QUANT_BYTES = 64 * 1024 ** 2 # tails at or above this size are packed. SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL = 6200 # host-held caches and parked frames, per output-frame pixel. +# Free memory above the floor goes to the caches on the device first, then to longer slices. +SEEDVR2_MAX_SLICE_LATENTS = 4 +SEEDVR2_SLICE_MEM_HEADROOM = 0.8 +SEEDVR2_DECODE_SLICE_BYTES_PER_FRAME_PIXEL = 5300 # per latent frame of a slice, per output-frame pixel. +SEEDVR2_ENCODE_SLICE_BYTES_PER_PIXEL = 3150 # per input-frame pixel. +SEEDVR2_DECODE_TAIL_BYTES_PER_FRAME_PIXEL = 5550 # caches on the device; the fixed part covers tiles too small to pack. +SEEDVR2_DECODE_TAIL_FIXED_BYTES = 768 * 1024 ** 2 +SEEDVR2_ENCODE_TAIL_BYTES_PER_PIXEL = 2700 +SEEDVR2_ENCODE_TAIL_FIXED_BYTES = 512 * 1024 ** 2 + # VAE decode estimate: the free VRAM an untiled decode needs under cudaMallocAsync (ComfyUI's default), # fitted 2-7% above the measured floor from 480p to 4K. Flat in clip length: the decoded frames go # to sd.py's output buffer off the GPU slice by slice (1088p: 5.33 GiB allocated at 61 and 121 frames). diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.py index 6c8032855f6..4437caee451 100644 --- a/comfy/ldm/seedvr/vae.py +++ b/comfy/ldm/seedvr/vae.py @@ -21,6 +21,14 @@ SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL, SEEDVR2_DECODE_FIXED_BYTES, SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL, + SEEDVR2_DECODE_SLICE_BYTES_PER_FRAME_PIXEL, + SEEDVR2_DECODE_TAIL_BYTES_PER_FRAME_PIXEL, + SEEDVR2_DECODE_TAIL_FIXED_BYTES, + SEEDVR2_ENCODE_SLICE_BYTES_PER_PIXEL, + SEEDVR2_ENCODE_TAIL_BYTES_PER_PIXEL, + SEEDVR2_ENCODE_TAIL_FIXED_BYTES, + SEEDVR2_MAX_SLICE_LATENTS, + SEEDVR2_SLICE_MEM_HEADROOM, SEEDVR2_DECODE_LAB_BYTES_PER_OUTPUT_PIXEL, SEEDVR2_ENCODE_BYTES_PER_PIXEL, SEEDVR2_ENCODE_FIXED_BYTES, @@ -33,11 +41,13 @@ from comfy.ldm.modules.diffusionmodules.model import vae_attention import collections +import contextlib import dataclasses import math from enum import Enum import comfy.model_management +import comfy.model_prefetch import comfy.ops import comfy.quant_ops import comfy_kitchen @@ -72,7 +82,8 @@ def _match_layout(t, ref): def _ndhwc_filter(conv): """Store a resident filter NDHWC once, so cast_bias_weight hands it back without a per-call copy.""" if conv.weight.is_cuda and not conv.weight.is_contiguous(memory_format=_CL): - conv.weight.data = conv.weight.data.contiguous(memory_format=_CL) + with comfy.model_prefetch.pause_malloc_graph(): + conv.weight.data = conv.weight.data.contiguous(memory_format=_CL) ops = comfy.ops.manual_cast @@ -107,7 +118,10 @@ def tiled_vae(x, vae_model, tile_size=(512, 512), tile_overlap=(64, 64), encode= def run(tile): tile = tile.contiguous() - out = vae_model.encode(tile) if encode else vae_model.slicing_decode(tile) + if encode: + out = vae_model.encode(tile) + else: + out = vae_model.slicing_decode(tile, torch.empty(vae_model.decode_output_shape(tile.shape), device=storage_device, dtype=tile.dtype)) if out.ndim == 4: # encode drops the time axis of a single-frame latent out = out.unsqueeze(2) return out.to(storage_device).float() @@ -194,12 +208,15 @@ class CausalMemoryCache: Large channels_last CUDA tails are int8-packed (ConvRot, per-token scales); with ``offload`` the packed tails live in host memory and the next reader's tail is prefetched while this one runs. A tail can be a ring of per-frame entries, so extending it packs one frame. free() ends the pass. + A ``compiled`` pass runs its slices under the memory compiler: an activation dies with its slice, and + the tails, which outlive it, stay on the device outside the compiler's memory, so neither fragments the other. """ CONVROT_GROUPSIZE = 256 - def __init__(self, offload=False): + def __init__(self, offload=False, compiled=False): self.offload = offload + self.compiled = compiled self.plain = {} self.packed = {} # key -> (qdata, params, shape), on the host when offloaded self._order = [] # keys in packing order, which is the order the next slice reads them @@ -319,6 +336,13 @@ def _packable(self, value): ) def __setitem__(self, key, value): + if self.compiled: + with comfy.model_prefetch.pause_malloc_graph(): + self._store(key, value if self._packable(value) else value.clone()) + else: + self._store(key, value) + + def _store(self, key, value): self.discard(key) # a whole tail replaces a ring too if self._packable(value): c = value.shape[1] @@ -393,6 +417,26 @@ def discard(self, key): for t in (entry[0],) if entry[1] is None else (entry[0], entry[1].scale): self._spare.setdefault((t.shape, t.dtype), []).append(t) + @contextlib.contextmanager + def scope(self, device): + """The compiler scope of a pass; a graph of its own is released at the end so the next plan sees + the memory, another model's parked graph is left as found.""" + if not self.compiled: + yield + return + parked = comfy.model_prefetch.malloc_graph_recording() + with comfy.model_prefetch.malloc_graph_scope(device): + yield + if not parked: + comfy.model_prefetch.cleanup_malloc_graph() + + def next_slice(self, device, stage=None, frames=None): + """Close the open slice scope and open one for ``frames`` latents through ``stage``; none to only close.""" + if self.compiled: + comfy.model_prefetch.prefetch_queue_pop(None, device, None, malloc_scope="slice") + if stage is not None: + comfy.model_prefetch.prefetch_queue_pop(None, device, stage, malloc_scope=f"slice{frames}") + def free(self): """Unpin the host buffers at the end of the pass; the cache is empty afterwards.""" if self._pinned: @@ -417,6 +461,26 @@ def _offload_caches_for(frame_pixels): return comfy.model_management.ensure_pin_budget(frame_pixels * SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL, evict_active=False) +def _plan_pass(device, free_memory, frame_pixels, latents, working_bytes, slice_bytes, tail_bytes, dtype): + """Latent frames per slice (0: a frame at a time) and the cache for a pass. A clip that fits runs + whole. Otherwise fp16 keeps the caches on the device under the compiler and the rest of the free + memory sets the slice length; without room for them, they go to host memory, or the pass stays at + its floor.""" + mm = comfy.model_management + slice_latents, compiled = 0, False + if mm.vram_state not in (mm.VRAMState.LOW_VRAM, mm.VRAMState.NO_VRAM) and mm.is_device_cuda(device): + spare = (free_memory - mm.extra_reserved_memory()) * SEEDVR2_SLICE_MEM_HEADROOM - working_bytes + if latents <= SEEDVR2_MAX_SLICE_LATENTS and spare >= latents * slice_bytes: + return latents, CausalMemoryCache() + # the other dtypes' tails are not stored this way + compiled = dtype == torch.float16 and spare >= tail_bytes and comfy.model_prefetch.malloc_graph_enabled(device) + if compiled: + spare -= tail_bytes + slice_latents = max(0, min(SEEDVR2_MAX_SLICE_LATENTS, int(spare / slice_bytes))) + offload = not compiled and _offload_caches_for(frame_pixels) + return slice_latents if compiled or offload else 0, CausalMemoryCache(offload=offload, compiled=compiled) + + class MemoryState(Enum): DISABLED = 0 INITIALIZING = 1 @@ -1165,20 +1229,18 @@ def __init__(self, in_channels: int, out_channels: int, block_out_channels, laye self.conv_norm_out = ops.GroupNorm(num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6) self.conv_out = InflatedCausalConv3d(block_out_channels[-1], 2 * out_channels, 3, padding=1) - # Frames per pass through the full-resolution, spatial-only head; None runs a slice's frames at once. - head_frames = 1 - def _head(self, source, memory_state, memory_cache): box = [self.conv_in(source.pop(), memory_state=memory_state, memory_cache=memory_cache)] for down_block in self.down_blocks[:self.head_blocks]: box = [down_block.run(box, memory_state, memory_cache)] return box.pop() - def forward(self, sample, memory_state, memory_cache): + def forward(self, sample, memory_state, memory_cache, head_frames): + """``head_frames`` per run through the spatial-only head; None runs the slice at once.""" frame_pixels = sample.size(-2) * sample.size(-1) source = [sample] del sample - box = [_frame_chunked(self, self._head, source, self.head_frames, memory_state, memory_cache, frame_pixels)] + box = [_frame_chunked(self, self._head, source, head_frames, memory_state, memory_cache, frame_pixels)] for down_block in self.down_blocks[self.head_blocks:]: box = [down_block.run(box, memory_state, memory_cache)] sample = self.mid_block(box.pop(), memory_state, memory_cache) @@ -1210,9 +1272,6 @@ def __init__(self, in_channels: int, out_channels: int, block_out_channels, laye self.conv_norm_out = ops.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6) self.conv_out = InflatedCausalConv3d(block_out_channels[0], out_channels, 3, padding=1) - # Frames per pass through the spatial-only tail; None runs a slice's frames at once. - tail_frames = 1 - def _tail(self, source, memory_state, memory_cache): for up_block in self.up_blocks[self.temporal_up_num:]: source = [up_block.run(source, memory_state, memory_cache)] @@ -1222,7 +1281,8 @@ def _tail(self, source, memory_state, memory_cache): out = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache) return out - def forward(self, sample, memory_state, memory_cache): + def forward(self, sample, memory_state, memory_cache, tail_frames): + """``tail_frames`` per run through the spatial-only tail; None runs the slice at once.""" sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache) sample = self.mid_block(sample, memory_state, memory_cache) @@ -1247,11 +1307,10 @@ def upsample_and_tail(source, state, cache): for n, box in enumerate(up.frames(source, state, cache))] return torch.cat(outs, dim=2) box = [up.run(source, state, cache)] - return _frame_chunked(self, self._tail, box, self.tail_frames, state, cache, frame_pixels) + return _frame_chunked(self, self._tail, box, tail_frames, state, cache, frame_pixels) source = [sample] del sample - return _frame_chunked(last, upsample_and_tail, source, self.tail_frames and max(1, self.tail_frames // up.temporal_ratio), - memory_state, memory_cache, frame_pixels) + return _frame_chunked(last, upsample_and_tail, source, tail_frames, memory_state, memory_cache, frame_pixels) class VideoAutoencoderKLWrapper(nn.Module): @@ -1266,6 +1325,7 @@ def __init__(self): super().__init__() self.encoder = Encoder3D(3, SEEDVR2_LATENT_CHANNELS, BYTEDANCE_BLOCK_OUT_CHANNELS, layers_per_block=2, norm_num_groups=32, temporal_down_num=2) + self.current_patcher = None # set by the VAE wrapper self.decoder = Decoder3D(SEEDVR2_LATENT_CHANNELS, 3, BYTEDANCE_BLOCK_OUT_CHANNELS, layers_per_block=2, norm_num_groups=32, temporal_up_num=2) set_norm_limit(BYTEDANCE_VAE_NORM_MEM_GIB) @@ -1273,30 +1333,51 @@ def __init__(self): if isinstance(m, InflatedCausalConv3d): m.memory_limit = BYTEDANCE_VAE_CONV_MEM_GIB - def _encode(self, x, memory_state=MemoryState.DISABLED, memory_cache=None): - h = self.encoder(x.to(self.device), memory_state=memory_state, memory_cache=memory_cache) + def _encode(self, x, head_frames, memory_state=MemoryState.DISABLED, memory_cache=None): + h = self.encoder(x.to(self.device), memory_state=memory_state, memory_cache=memory_cache, head_frames=head_frames) return h.to(x.device) - def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None): - output = self.decoder(z.to(self.device), memory_state=memory_state, memory_cache=memory_cache) + def _decode(self, z, tail_frames, memory_state=MemoryState.DISABLED, memory_cache=None): + output = self.decoder(z.to(self.device), memory_state=memory_state, memory_cache=memory_cache, tail_frames=tail_frames) return output.to(z.device) def slicing_encode(self, x: torch.Tensor) -> torch.Tensor: if x.size(0) > 1: # a video at a time: the buffered path and the memory estimate are per video return torch.cat([self.slicing_encode(x[i:i + 1]) for i in range(x.size(0))]) - if (x.shape[2] - 1) <= self.slicing_sample_min_size: - return self._encode(x.to(self.device)) - memory_cache = CausalMemoryCache(offload=_offload_caches_for(x.shape[-2] * x.shape[-1])) - split_size = max(self.slicing_sample_min_size, self.temporal_downsample_factor) + area = x.shape[-2] * x.shape[-1] + factor = _eager_factor(x.dtype, SEEDVR2_EAGER_ENCODE_FACTOR) + latents = -(-(x.shape[2] - 1) // self.temporal_downsample_factor) + slice_bytes = area * SEEDVR2_ENCODE_SLICE_BYTES_PER_PIXEL * factor + short_tail = (x.shape[2] - 1) % self.temporal_downsample_factor / self.temporal_downsample_factor # joins the slice before it + slice_latents, memory_cache = _plan_pass(self.device, self.current_patcher.get_free_memory(self.device), area, latents, self.comfy_memory_used_encode(x.shape, x.dtype) + short_tail * slice_bytes, + slice_bytes, area * SEEDVR2_ENCODE_TAIL_BYTES_PER_PIXEL + SEEDVR2_ENCODE_TAIL_FIXED_BYTES, x.dtype) + head_frames = None if slice_latents else 1 + split_size = max(self.slicing_sample_min_size, self.temporal_downsample_factor, slice_latents * self.temporal_downsample_factor) + if (x.shape[2] - 1) <= split_size: + memory_cache.free() + return self._encode(x.to(self.device), head_frames=head_frames) x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2)) # _encode moves each to the device if len(x_slices) > 1 and x_slices[-1].shape[2] < self.temporal_downsample_factor: x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2) x_slices.pop() + encoded_slices = [] + + def keep(encoded): + if memory_cache.compiled and encoded.is_cuda: + with comfy.model_prefetch.pause_malloc_graph(): + encoded_slices.append(encoded.clone()) + else: + encoded_slices.append(encoded) + try: - encoded_slices = [self._encode(torch.cat((x[:, :, :1], x_slices[0]), dim=2), - memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache)] - for x_slice in x_slices[1:]: - encoded_slices.append(self._encode(x_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)) + with memory_cache.scope(self.device): + memory_cache.next_slice(self.device, self.encoder, x_slices[0].size(2) + 1) + keep(self._encode(torch.cat((x[:, :, :1], x_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache, head_frames=head_frames)) + for x_slice in x_slices[1:]: + memory_cache.next_slice(self.device, self.encoder, x_slice.size(2)) + keep(self._encode(x_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache, head_frames=head_frames)) + memory_cache.next_slice(self.device) finally: memory_cache.free() return torch.cat(encoded_slices, dim=2) @@ -1307,11 +1388,20 @@ def slicing_decode(self, z: torch.Tensor, output_buffer=None) -> torch.Tensor: outs = [self.slicing_decode(z[i:i + 1], None if output_buffer is None else output_buffer[i:i + 1]) for i in range(z.size(0))] return output_buffer if output_buffer is not None else torch.cat(outs) - if (z.shape[2] - 1) <= self.slicing_latent_min_size: - decoded = self._decode(z) + area = z.shape[-2] * z.shape[-1] * self.spatial_downsample_factor ** 2 + factor = _eager_factor(z.dtype, SEEDVR2_EAGER_DECODE_FACTOR) + slice_latents, memory_cache = _plan_pass(self.device, self.current_patcher.get_free_memory(self.device), area, z.shape[2] - 1, (area * SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + SEEDVR2_DECODE_FIXED_BYTES) * factor, + area * SEEDVR2_DECODE_SLICE_BYTES_PER_FRAME_PIXEL * factor, + area * SEEDVR2_DECODE_TAIL_BYTES_PER_FRAME_PIXEL + SEEDVR2_DECODE_TAIL_FIXED_BYTES, z.dtype) + tail_frames = None if slice_latents else 1 + split_size = max(self.slicing_latent_min_size, slice_latents) + if (z.shape[2] - 1) <= split_size: + memory_cache.free() + decoded = self._decode(z, tail_frames=tail_frames) return decoded if output_buffer is None else output_buffer.copy_(decoded) - memory_cache = CausalMemoryCache(offload=_offload_caches_for(z.shape[-2] * z.shape[-1] * self.spatial_downsample_factor ** 2)) - z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2) + z_slices = z[:, :, 1:].split(split_size=split_size, dim=2) + if memory_cache.compiled and output_buffer is None: # the frames outlive their slice + output_buffer = torch.empty(self.decode_output_shape(z.shape), device=z.device, dtype=z.dtype) decoded_slices = [] write_pos = 0 @@ -1324,10 +1414,14 @@ def emit(decoded): write_pos += decoded.size(2) try: - emit(self._decode(torch.cat((z[:, :, :1], z_slices[0]), dim=2), - memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache)) - for z_slice in z_slices[1:]: - emit(self._decode(z_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)) + with memory_cache.scope(self.device): + memory_cache.next_slice(self.device, self.decoder, z_slices[0].size(2) + 1) + emit(self._decode(torch.cat((z[:, :, :1], z_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache, tail_frames=tail_frames)) + for z_slice in z_slices[1:]: + memory_cache.next_slice(self.device, self.decoder, z_slice.size(2)) + emit(self._decode(z_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache, tail_frames=tail_frames)) + memory_cache.next_slice(self.device) finally: memory_cache.free() return output_buffer if output_buffer is not None else torch.cat(decoded_slices, dim=2) diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 75dfaf786b3..679789f8143 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -62,6 +62,9 @@ def __exit__(self, exc_type, *args): def malloc_graph_scope(device): return _MallocGraphScope(device) +def malloc_graph_recording(): + return threading.get_ident() in MALLOC_GRAPHS + def malloc_graph_begin(device): global MALLOC_GRAPH_USED if not malloc_graph_enabled(device): diff --git a/comfy/sd.py b/comfy/sd.py index ca61c01f776..fca0c757682 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -1159,6 +1159,7 @@ def _no_generic_io(*args, **kwargs): if self.disable_offload: mp = comfy.model_patcher.ModelPatcher self.patcher = mp(self.first_stage_model, load_device=self.device, offload_device=offload_device, fast_disk=fast_disk) + self.first_stage_model.current_patcher = self.patcher m, u = self.first_stage_model.load_state_dict(sd, strict=False, assign=self.patcher.is_dynamic()) if not self.patcher.is_dynamic(): @@ -1177,6 +1178,7 @@ def clone(self): n = VAE(no_init=True) n.patcher = self.patcher.clone() n.first_stage_model = self.first_stage_model + n.first_stage_model.current_patcher = n.patcher n.memory_used_encode = self.memory_used_encode n.memory_used_decode = self.memory_used_decode n.downscale_ratio = self.downscale_ratio diff --git a/tests-unit/comfy_test/test_seedvr2_internals.py b/tests-unit/comfy_test/test_seedvr2_internals.py index 431eec28192..e21f410fc12 100644 --- a/tests-unit/comfy_test/test_seedvr2_internals.py +++ b/tests-unit/comfy_test/test_seedvr2_internals.py @@ -3,6 +3,7 @@ from __future__ import annotations import inspect +from types import SimpleNamespace from unittest.mock import patch import pytest @@ -355,6 +356,67 @@ def test_causal_memory_cache_offloads_small_tails_and_parked_frames_exactly(): cache.free() +def _plan(free_gib, tail_gib=1, compiler=True, room_on_host=True, dtype=torch.float16, vram_state="NORMAL_VRAM", latents=80): + mm = vae_mod.comfy.model_management + gib = 1024 ** 3 + with patch.object(vae_mod.comfy.model_prefetch, "malloc_graph_enabled", lambda dev: compiler), \ + patch.object(mm, "vram_state", mm.VRAMState[vram_state]), \ + patch.object(mm, "is_device_cuda", lambda dev: True), \ + patch.object(mm, "extra_reserved_memory", lambda: 0), \ + patch.object(vae_mod, "SEEDVR2_SLICE_MEM_HEADROOM", 1.0), \ + patch.object(vae_mod, "_offload_caches_for", lambda frame_pixels: room_on_host): + return vae_mod._plan_pass(torch.device("cpu"), free_gib * gib, 1, latents, 1 * gib, 1 * gib, tail_gib * gib, dtype) + + +@pytest.mark.parametrize("plan, slices, compiled, offload", [ + (dict(free_gib=0.5), 0, False, True), # below the floor: as it always ran + (dict(free_gib=2.5), 0, True, False), # the tails first, nothing left for a slice + (dict(free_gib=5.0), 3, True, False), + (dict(free_gib=30.0), vae_mod.SEEDVR2_MAX_SLICE_LATENTS, True, False), + (dict(free_gib=30.0, latents=2), 2, False, False), # fits in one run: no cache + (dict(free_gib=30.0, dtype=torch.bfloat16), vae_mod.SEEDVR2_MAX_SLICE_LATENTS, False, True), # only fp16 tails stay on the device + (dict(free_gib=5.0, compiler=False), vae_mod.SEEDVR2_MAX_SLICE_LATENTS, False, True), # tails in host memory + (dict(free_gib=3.0, tail_gib=3, room_on_host=False), 0, False, False), # tails have no home: the floor + (dict(free_gib=30.0, vram_state="LOW_VRAM"), 0, False, True), +]) +def test_seedvr2_plan(plan, slices, compiled, offload): + planned, cache = _plan(**plan) + assert (planned, cache.compiled, cache.offload) == (slices, compiled, offload) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="the buffered convs run fp16 on CUDA") +def test_seedvr2_slice_length_does_not_change_the_result(): + """The caches carry a causal stage across slices and GroupNorm is per frame, so how many frames run + at once leaves only fp16 rounding: three summed taps round differently from one conv.""" + class Unpacked(vae_mod.CausalMemoryCache): + def _packable(self, value): + return False + + torch.manual_seed(0) + vae = vae_mod.VideoAutoencoderKLWrapper() + for name, p in vae.named_parameters(): # small enough that nothing overflows fp16 through the whole network + if p.dim() >= 2: + torch.nn.init.normal_(p, std=0.01) + else: + p.data.fill_(1.0 if "norm" in name else 0.0) + vae = vae.to("cuda", torch.float16) + vae.device = torch.device("cuda") + vae.current_patcher = SimpleNamespace(get_free_memory=lambda device: 0) + z = torch.randn(1, 16, 7, 16, 24, device="cuda", dtype=torch.float16) + x = torch.randn(1, 3, 13, 64, 96, device="cuda", dtype=torch.float16) + + def run(fn, data, slices, **cache): + with patch.object(vae_mod, "_plan_pass", lambda *a: (slices, Unpacked(**cache))): + return fn(data).float() + + for fn, data in ((vae.slicing_decode, z), (vae.slicing_encode, x)): + reference = run(fn, data, 0) + assert torch.isfinite(reference).all() + for slices in (1, 2, 4): + torch.testing.assert_close(run(fn, data, slices), reference, rtol=0, atol=1e-2) + torch.testing.assert_close(run(fn, data, slices, compiled=True), reference, rtol=0, atol=1e-2) + + def _ring(cache, key): frames = [] cache.for_each_frame(key, lambda i, frame: frames.append(frame.clone())) @@ -659,9 +721,12 @@ def test_seedvr2_batch_runs_a_video_at_a_time(): slices land back in their video's place (in the preallocated output too).""" vae = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper) vae.slicing_sample_min_size = vae.slicing_latent_min_size = 2 + vae.device = torch.device("cpu") # encode() and decode() set it before slicing + vae.encoder = vae.decoder = None + vae.current_patcher = SimpleNamespace(get_free_memory=lambda device: 0) seen = [] - def run(x, memory_state=vae_mod.MemoryState.DISABLED, memory_cache=None): + def run(x, memory_state=vae_mod.MemoryState.DISABLED, memory_cache=None, **_): seen.append(x.size(0)) return x[:, :3].clone() vae._decode = vae._encode = run From bd12362ea953851842dcb3551bd7f713e2a3db3a Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:38:18 +0400 Subject: [PATCH 4/8] [Partner Nodes] feat(Pika): add Pika H3 Flex Video and Reference to Video nodes (#17509) Signed-off-by: bigcat88 --- comfy_api_nodes/apis/pika.py | 38 ++++ comfy_api_nodes/nodes_pika.py | 384 ++++++++++++++++++++++++++++++++++ 2 files changed, 422 insertions(+) create mode 100644 comfy_api_nodes/apis/pika.py create mode 100644 comfy_api_nodes/nodes_pika.py diff --git a/comfy_api_nodes/apis/pika.py b/comfy_api_nodes/apis/pika.py new file mode 100644 index 00000000000..e080b3abf65 --- /dev/null +++ b/comfy_api_nodes/apis/pika.py @@ -0,0 +1,38 @@ +from pydantic import BaseModel, Field + + +class PikaH3FlexRequest(BaseModel): + prompt: str = Field(...) + duration: int = Field(...) + aspect_ratio: str = Field(...) + resolution: str = Field(...) + seed: int = Field(...) + rewrite: bool = Field(...) + + +class PikaH3FlexImageRequest(PikaH3FlexRequest): + image: str | None = Field(None) + last_frame_image: str | None = Field(None) + + +class PikaH3FlexReference(BaseModel): + type: str = Field(...) + url: str = Field(...) + + +class PikaH3FlexReferenceRequest(PikaH3FlexRequest): + references: list[PikaH3FlexReference] = Field(...) + + +class PikaMediaFile(BaseModel): + url: str = Field(...) + + +class PikaJobOutput(BaseModel): + video: PikaMediaFile | None = Field(None) + + +class PikaJob(BaseModel): + id: str = Field(...) + status: str = Field(...) + output: PikaJobOutput | None = Field(None) diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py new file mode 100644 index 00000000000..87f88fba249 --- /dev/null +++ b/comfy_api_nodes/nodes_pika.py @@ -0,0 +1,384 @@ +import re +import uuid +from io import BytesIO + +import av +from pydantic import BaseModel +from typing_extensions import override + +from comfy_api.latest import IO, ComfyExtension, Input, Types +from comfy_api_nodes.apis.pika import ( + PikaH3FlexImageRequest, + PikaH3FlexReference, + PikaH3FlexReferenceRequest, + PikaH3FlexRequest, + PikaJob, + PikaJobOutput, +) +from comfy_api_nodes.util import ( + ApiEndpoint, + download_url_to_video_output, + poll_op, + sync_op, + upload_audio_to_comfyapi, + upload_file_to_comfyapi, + upload_image_to_comfyapi, + upload_video_to_comfyapi, + validate_audio_duration, + validate_string, + validate_video_duration, +) + +PIKA_H3_FLEX_ASPECT_RATIOS = ["auto", "21:9", "16:9", "3:2", "4:3", "5:4", "1:1", "4:5", "3:4", "2:3", "9:16", "9:21"] +PIKA_H3_FLEX_REF_TAG_RE = re.compile(r"@(?Pimage|video|audio)(?P\d*)(?!\w)", re.IGNORECASE | re.ASCII) +PIKA_H3_FLEX_REF_LABELS = {"image": "Picture", "video": "Video", "audio": "Audio"} + + +def _seed_input() -> IO.Int.Input: + return IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="With rewrite_prompt off, the same seed repeats the video exactly. " + "A local run with the same seed gives a different result.", + ) + + +def _h3_flex_settings_inputs(aspect_ratio_tooltip: str) -> list[IO.Input]: + return [ + IO.Combo.Input( + "resolution", + options=["480p", "768p", "1080p", "2k", "4k"], + default="768p", + tooltip="1080p and above render at 768p and upscale once. The same seed at 480p gives a different video.", + ), + IO.Combo.Input( + "aspect_ratio", + options=PIKA_H3_FLEX_ASPECT_RATIOS, + default="auto", + tooltip=aspect_ratio_tooltip, + ), + IO.Int.Input( + "duration", + default=5, + min=5, + max=15, + tooltip="The output length follows a frame grid: 5 seconds gives 124 frames, about 5.17 seconds.", + ), + IO.Boolean.Input( + "rewrite_prompt", + default=False, + tooltip="Let Pika expand the prompt first. The same seed then gives a different video on each run.", + ), + _seed_input(), + ] + + +def _resolve_ref_tags(prompt: str, counts: dict[str, int]) -> str: + parts = [] + pos = 0 + prev_end = -1 + for match in PIKA_H3_FLEX_REF_TAG_RE.finditer(prompt): + start = match.start() + if ( + start > 0 + and start != prev_end + and prompt[start - 1].isascii() + and (prompt[start - 1].isalnum() or prompt[start - 1] == "_") + ): + continue + kind = match.group("kind").lower() + idx = int(match.group("idx") or 1) + if not 1 <= idx <= counts[kind]: + raise ValueError( + f"The prompt references @{kind.capitalize()}{idx}, but {counts[kind]} {kind} input(s) are available." + ) + parts.append(prompt[pos:start]) + parts.append(f"<{PIKA_H3_FLEX_REF_LABELS[kind]} {idx}>") + pos = match.end() + prev_end = match.end() + parts.append(prompt[pos:]) + return "".join(parts) + + +async def _upload_reference_video(cls: type[IO.ComfyNode], video: Input.Video, keep_audio: bool, label: str) -> str: + if keep_audio: + return await upload_video_to_comfyapi(cls, video, wait_label=label) + source = BytesIO() + video.save_to(source, format=Types.VideoContainer.MP4, codec=Types.VideoCodec.H264) + source.seek(0) + silent = BytesIO() + with av.open(source) as src, av.open(silent, mode="w", format="mp4") as dst: + src_stream = src.streams.video[0] + dst_stream = dst.add_stream_from_template(src_stream) + for packet in src.demux(src_stream): + if packet.dts is not None: + packet.stream = dst_stream + dst.mux(packet) + silent.seek(0) + return await upload_file_to_comfyapi(cls, silent, f"{uuid.uuid4()}.mp4", "video/mp4", label) + + +async def _pika_job(cls: type[IO.ComfyNode], path: str, request: BaseModel) -> PikaJobOutput: + job = await sync_op( + cls, + ApiEndpoint(path=f"/proxy/pika/v1/media/pika/{path}", method="POST"), + response_model=PikaJob, + data=request, + ) + job = await poll_op( + cls, + ApiEndpoint(path=f"/proxy/pika/v1/media/jobs/{job.id}"), + response_model=PikaJob, + status_extractor=lambda r: r.status, + completed_statuses=["completed"], + failed_statuses=["failed"], + queued_statuses=["queued"], + ) + if job.output is None: + raise Exception(f"Pika returned no output for job {job.id}.") + return job.output + + +async def _pika_video(cls: type[IO.ComfyNode], path: str, request: BaseModel) -> IO.NodeOutput: + output = await _pika_job(cls, path, request) + return IO.NodeOutput(await download_url_to_video_output(output.video.url, cls=cls)) + + +class PikaH3FlexVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PikaH3FlexVideoNode", + display_name="Pika H3 Flex Video", + category="partner/video/Pika", + description="Pika H3 Flex, Pika's post-trained MiniMax H3. " + "Generate video with synchronized audio from text and optional first and last frames.", + search_aliases=["minimax h3", "h3 flex", "pika h3", "text to video", "image to video", "first last frame"], + inputs=[ + IO.String.Input("prompt", multiline=True, default=""), + IO.Image.Input("first_frame", optional=True), + IO.Image.Input("last_frame", optional=True), + *_h3_flex_settings_inputs("auto follows the first connected frame, otherwise the prompt."), + ], + outputs=[IO.Video.Output()], + hidden=[IO.Hidden.auth_token_comfy_org, IO.Hidden.api_key_comfy_org, IO.Hidden.unique_id], + is_api_node=True, + price_badge=IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution", "duration"]), + expr=""" + ( + $res := widgets.resolution; + $d := widgets.duration; + $rate := $lookup({"480p": 0.0143, "768p": 0.0286, "1080p": 0.05005, "2k": 0.0572, "4k": 0.1144}, $res); + $type($d) = "number" + ? {"type":"usd","usd": $rate * (($res = "480p" or $res = "768p") + ? $d : ($ceil((24 * $d - 5) / 17) * 17 + 5) / 24)} + : {"type":"usd","usd": $rate,"format":{"suffix":"/second"}} + ) + """, + ), + ) + + @classmethod + async def execute( + cls, + prompt: str, + resolution: str, + aspect_ratio: str, + duration: int, + rewrite_prompt: bool, + seed: int, + first_frame: Input.Image | None = None, + last_frame: Input.Image | None = None, + ) -> IO.NodeOutput: + validate_string(prompt, min_length=1, max_length=8000) + settings = { + "prompt": prompt, + "duration": duration, + "aspect_ratio": aspect_ratio, + "resolution": resolution, + "seed": seed, + "rewrite": rewrite_prompt, + } + if first_frame is None and last_frame is None: + return await _pika_video(cls, "pika-h3-flex/text-to-video", PikaH3FlexRequest(**settings)) + return await _pika_video( + cls, + "pika-h3-flex/image-to-video", + PikaH3FlexImageRequest( + **settings, + image=None + if first_frame is None + else await upload_image_to_comfyapi(cls, first_frame, wait_label="Uploading first frame"), + last_frame_image=None + if last_frame is None + else await upload_image_to_comfyapi(cls, last_frame, wait_label="Uploading last frame"), + ), + ) + + +class PikaH3FlexReferenceVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PikaH3FlexReferenceVideoNode", + display_name="Pika H3 Flex Reference to Video", + category="partner/video/Pika", + description="Pika H3 Flex, Pika's post-trained MiniMax H3. " + "Generate or edit video with synchronized audio guided by reference images, videos and audio.", + search_aliases=["minimax h3", "h3 flex", "pika h3", "video editing", "motion transfer", "reference video"], + inputs=[ + IO.Autogrow.Input( + "reference_images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("reference_image"), + names=[f"image_{i}" for i in range(1, 10)], + min=0, + ), + optional=True, + tooltip="Up to 9 images, referred to in the prompt as @Image1..@Image9 in connection order. " + "At least one image or video is required.", + ), + IO.Autogrow.Input( + "reference_videos", + template=IO.Autogrow.TemplateNames( + IO.Video.Input("reference_video"), + names=[f"video_{i}" for i in range(1, 4)], + min=0, + ), + optional=True, + tooltip="Up to 3 videos, referred to in the prompt as @Video1..@Video3 in connection order. " + "2-15 seconds each, 15 seconds in total. Billed per second.", + ), + IO.Autogrow.Input( + "reference_audios", + template=IO.Autogrow.TemplateNames( + IO.Audio.Input("reference_audio"), + names=[f"audio_{i}" for i in range(1, 4)], + min=0, + ), + optional=True, + tooltip="Up to 3 clips, referred to in the prompt as @Audio1..@Audio3. " + "2-15 seconds each, 15 seconds in total. When copied to the output, clips play in connection order, " + "ahead of the sound of connected videos, whichever number the prompt names.", + ), + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Refer to references as @Image1, @Video1, @Audio1, or as ,