diff --git a/.ci/windows_nvidia_base_files/advanced/run_cpu.bat b/.ci/windows_nvidia_base_files/advanced/run_cpu.bat new file mode 100755 index 00000000000..f060f93418c --- /dev/null +++ b/.ci/windows_nvidia_base_files/advanced/run_cpu.bat @@ -0,0 +1,2 @@ +..\python_embeded\python.exe -s ..\ComfyUI\main.py --cpu --windows-standalone-build +pause diff --git a/.ci/windows_nvidia_base_files/run_cpu.bat b/.ci/windows_nvidia_base_files/run_cpu.bat deleted file mode 100755 index c3ba4172161..00000000000 --- a/.ci/windows_nvidia_base_files/run_cpu.bat +++ /dev/null @@ -1,2 +0,0 @@ -.\python_embeded\python.exe -s ComfyUI\main.py --cpu --windows-standalone-build -pause diff --git a/.github/PULL_REQUEST_TEMPLATE/api-node.md b/.github/PULL_REQUEST_TEMPLATE/api-node.md index c1f1bafb158..90fa712fea4 100644 --- a/.github/PULL_REQUEST_TEMPLATE/api-node.md +++ b/.github/PULL_REQUEST_TEMPLATE/api-node.md @@ -16,6 +16,3 @@ If **Need pricing update**: ### QA - [ ] **QA done** - [ ] **QA not required** - -### Comms -- [ ] Informed **Kosinkadink** diff --git a/CODEOWNERS b/CODEOWNERS index ec1c6f14138..e3ae1720c98 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,4 +1,5 @@ * @comfyanonymous @guill @alexisrolland @rattus128 @kijai +/comfy_api_nodes/ @comfyanonymous @guill @alexisrolland @rattus128 @kijai @comfyui-wiki @purzbeats /CODEOWNERS @comfyanonymous /AGENTS.md @comfyanonymous diff --git a/agent_requirements.txt b/agent_requirements.txt index a4cca7397f6..90cae1ac64f 100644 --- a/agent_requirements.txt +++ b/agent_requirements.txt @@ -1 +1 @@ -comfy-agent==0.0.0.dev0 +comfy-agent diff --git a/app/agent_host.py b/app/agent_host.py index 9db531b0f69..726db94bf11 100644 --- a/app/agent_host.py +++ b/app/agent_host.py @@ -48,14 +48,16 @@ def load() -> bool: entry_points = {ep.name: ep for ep in importlib.metadata.entry_points(group=ENTRY_POINT_GROUP)} if PROTOCOL not in entry_points: # Some comfy-agent is installed (other protocols present) -> "update"; - # nothing there -> "install". + # nothing there -> "install". The file names the package without a + # version, so only `-U` moves an old or placeholder install. + req = os.path.join(os.path.dirname(os.path.abspath(folder_paths.__file__)), "agent_requirements.txt") + install = f'pip install -U -r "{req}"' if entry_points: - logging.warning(f"The installed comfy-agent offers agent protocols {sorted(entry_points)}; this ComfyUI needs {PROTOCOL}. Update comfy-agent to use the agent.") + logging.warning(f"The installed comfy-agent offers agent protocols {sorted(entry_points)}; this ComfyUI needs {PROTOCOL}. Update it with `{install}` to use the agent.") _failure = {"state": "failed", "reason": "outdated"} emit_event("agent_error", reason="outdated") else: - req = os.path.join(os.path.dirname(os.path.abspath(folder_paths.__file__)), "agent_requirements.txt") - logging.warning(f"`--enable-agent` was passed but the comfy-agent package is not installed, so the agent is disabled. Its requirements are in {req}.") + logging.warning(f"`--enable-agent` was passed but the comfy-agent package is not installed, so the agent is disabled. Install it with `{install}`.") _failure = {"state": "failed", "reason": "package_missing"} emit_event("package_missing") _publish_flag() diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py index 0e7fba91230..1367c521445 100644 --- a/comfy/latent_formats.py +++ b/comfy/latent_formats.py @@ -880,6 +880,14 @@ def __init__(self): 0.3971, 1.0600, 0.3943, 0.5537, 0.5444, 0.4089, 0.7468, 0.7744 ]).view(1, self.latent_channels, 1, 1, 1) +class RelayVSR(Wan22): + # the keyframe Wan and the Flash decoder both work on the normalized latent + def process_in(self, latent): + return latent + + def process_out(self, latent): + return latent + class HunyuanImage21(LatentFormat): latent_channels = 64 latent_dimensions = 2 diff --git a/comfy/ldm/seedvr/constants.py b/comfy/ldm/seedvr/constants.py index 98fb5057b5a..facefe69285 100644 --- a/comfy/ldm/seedvr/constants.py +++ b/comfy/ldm/seedvr/constants.py @@ -21,6 +21,16 @@ SEEDVR2_VAE_CACHE_QUANT_BYTES = 64 * 1024 ** 2 # tails at or above this size are packed. SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL = 6200 # host-held caches and parked frames, per output-frame pixel. +# Free memory above the floor goes to the caches on the device first, then to longer slices. +SEEDVR2_MAX_SLICE_LATENTS = 4 +SEEDVR2_SLICE_MEM_HEADROOM = 0.8 +SEEDVR2_DECODE_SLICE_BYTES_PER_FRAME_PIXEL = 5300 # per latent frame of a slice, per output-frame pixel. +SEEDVR2_ENCODE_SLICE_BYTES_PER_PIXEL = 3150 # per input-frame pixel. +SEEDVR2_DECODE_TAIL_BYTES_PER_FRAME_PIXEL = 5550 # caches on the device; the fixed part covers tiles too small to pack. +SEEDVR2_DECODE_TAIL_FIXED_BYTES = 768 * 1024 ** 2 +SEEDVR2_ENCODE_TAIL_BYTES_PER_PIXEL = 2700 +SEEDVR2_ENCODE_TAIL_FIXED_BYTES = 512 * 1024 ** 2 + # VAE decode estimate: the free VRAM an untiled decode needs under cudaMallocAsync (ComfyUI's default), # fitted 2-7% above the measured floor from 480p to 4K. Flat in clip length: the decoded frames go # to sd.py's output buffer off the GPU slice by slice (1088p: 5.33 GiB allocated at 61 and 121 frames). diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.py index 6c8032855f6..c73ee1c9709 100644 --- a/comfy/ldm/seedvr/vae.py +++ b/comfy/ldm/seedvr/vae.py @@ -21,6 +21,14 @@ SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL, SEEDVR2_DECODE_FIXED_BYTES, SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL, + SEEDVR2_DECODE_SLICE_BYTES_PER_FRAME_PIXEL, + SEEDVR2_DECODE_TAIL_BYTES_PER_FRAME_PIXEL, + SEEDVR2_DECODE_TAIL_FIXED_BYTES, + SEEDVR2_ENCODE_SLICE_BYTES_PER_PIXEL, + SEEDVR2_ENCODE_TAIL_BYTES_PER_PIXEL, + SEEDVR2_ENCODE_TAIL_FIXED_BYTES, + SEEDVR2_MAX_SLICE_LATENTS, + SEEDVR2_SLICE_MEM_HEADROOM, SEEDVR2_DECODE_LAB_BYTES_PER_OUTPUT_PIXEL, SEEDVR2_ENCODE_BYTES_PER_PIXEL, SEEDVR2_ENCODE_FIXED_BYTES, @@ -33,11 +41,13 @@ from comfy.ldm.modules.diffusionmodules.model import vae_attention import collections +import contextlib import dataclasses import math from enum import Enum import comfy.model_management +import comfy.model_prefetch import comfy.ops import comfy.quant_ops import comfy_kitchen @@ -72,7 +82,8 @@ def _match_layout(t, ref): def _ndhwc_filter(conv): """Store a resident filter NDHWC once, so cast_bias_weight hands it back without a per-call copy.""" if conv.weight.is_cuda and not conv.weight.is_contiguous(memory_format=_CL): - conv.weight.data = conv.weight.data.contiguous(memory_format=_CL) + with comfy.model_prefetch.pause_malloc_graph(): + conv.weight.data = conv.weight.data.contiguous(memory_format=_CL) ops = comfy.ops.manual_cast @@ -107,7 +118,10 @@ def tiled_vae(x, vae_model, tile_size=(512, 512), tile_overlap=(64, 64), encode= def run(tile): tile = tile.contiguous() - out = vae_model.encode(tile) if encode else vae_model.slicing_decode(tile) + if encode: + out = vae_model.encode(tile) + else: + out = vae_model.slicing_decode(tile, torch.empty(vae_model.decode_output_shape(tile.shape), device=storage_device, dtype=tile.dtype)) if out.ndim == 4: # encode drops the time axis of a single-frame latent out = out.unsqueeze(2) return out.to(storage_device).float() @@ -194,12 +208,15 @@ class CausalMemoryCache: Large channels_last CUDA tails are int8-packed (ConvRot, per-token scales); with ``offload`` the packed tails live in host memory and the next reader's tail is prefetched while this one runs. A tail can be a ring of per-frame entries, so extending it packs one frame. free() ends the pass. + A ``compiled`` pass runs its slices under the memory compiler: an activation dies with its slice, and + the tails, which outlive it, stay on the device outside the compiler's memory, so neither fragments the other. """ CONVROT_GROUPSIZE = 256 - def __init__(self, offload=False): + def __init__(self, offload=False, compiled=False): self.offload = offload + self.compiled = compiled self.plain = {} self.packed = {} # key -> (qdata, params, shape), on the host when offloaded self._order = [] # keys in packing order, which is the order the next slice reads them @@ -216,7 +233,7 @@ def _transfer_stream(self, device): """The one stream this cache copies on, so copies reusing a host buffer stay ordered; it starts after the compute queued so far. None copies on the compute stream.""" if self._stream is None: - self._stream = comfy.model_management.get_offload_stream(device) + self._stream = comfy.model_management.get_data_stream(device) if self._stream is not None: self._stream.wait_stream(comfy.model_management.current_stream(device)) return self._stream @@ -319,6 +336,13 @@ def _packable(self, value): ) def __setitem__(self, key, value): + if self.compiled: + with comfy.model_prefetch.pause_malloc_graph(): + self._store(key, value if self._packable(value) else value.clone()) + else: + self._store(key, value) + + def _store(self, key, value): self.discard(key) # a whole tail replaces a ring too if self._packable(value): c = value.shape[1] @@ -393,6 +417,26 @@ def discard(self, key): for t in (entry[0],) if entry[1] is None else (entry[0], entry[1].scale): self._spare.setdefault((t.shape, t.dtype), []).append(t) + @contextlib.contextmanager + def scope(self, device): + """The compiler scope of a pass; a graph of its own is released at the end so the next plan sees + the memory, another model's parked graph is left as found.""" + if not self.compiled: + yield + return + parked = comfy.model_prefetch.malloc_graph_recording() + with comfy.model_prefetch.malloc_graph_scope(device): + yield + if not parked: + comfy.model_prefetch.cleanup_malloc_graph() + + def next_slice(self, device, stage=None, frames=None): + """Close the open slice scope and open one for ``frames`` latents through ``stage``; none to only close.""" + if self.compiled: + comfy.model_prefetch.prefetch_queue_pop(None, device, None, malloc_scope="slice") + if stage is not None: + comfy.model_prefetch.prefetch_queue_pop(None, device, stage, malloc_scope=f"slice{frames}") + def free(self): """Unpin the host buffers at the end of the pass; the cache is empty afterwards.""" if self._pinned: @@ -417,6 +461,26 @@ def _offload_caches_for(frame_pixels): return comfy.model_management.ensure_pin_budget(frame_pixels * SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL, evict_active=False) +def _plan_pass(device, free_memory, frame_pixels, latents, working_bytes, slice_bytes, tail_bytes, dtype): + """Latent frames per slice (0: a frame at a time) and the cache for a pass. A clip that fits runs + whole. Otherwise fp16 keeps the caches on the device under the compiler and the rest of the free + memory sets the slice length; without room for them, they go to host memory, or the pass stays at + its floor.""" + mm = comfy.model_management + slice_latents, compiled = 0, False + if mm.vram_state not in (mm.VRAMState.LOW_VRAM, mm.VRAMState.NO_VRAM) and mm.is_device_cuda(device): + spare = (free_memory - mm.extra_reserved_memory()) * SEEDVR2_SLICE_MEM_HEADROOM - working_bytes + if latents <= SEEDVR2_MAX_SLICE_LATENTS and spare >= latents * slice_bytes: + return latents, CausalMemoryCache() + # the other dtypes' tails are not stored this way + compiled = dtype == torch.float16 and spare >= tail_bytes and comfy.model_prefetch.malloc_graph_enabled(device) + if compiled: + spare -= tail_bytes + slice_latents = max(0, min(SEEDVR2_MAX_SLICE_LATENTS, int(spare / slice_bytes))) + offload = not compiled and _offload_caches_for(frame_pixels) + return slice_latents if compiled or offload else 0, CausalMemoryCache(offload=offload, compiled=compiled) + + class MemoryState(Enum): DISABLED = 0 INITIALIZING = 1 @@ -1165,20 +1229,18 @@ def __init__(self, in_channels: int, out_channels: int, block_out_channels, laye self.conv_norm_out = ops.GroupNorm(num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6) self.conv_out = InflatedCausalConv3d(block_out_channels[-1], 2 * out_channels, 3, padding=1) - # Frames per pass through the full-resolution, spatial-only head; None runs a slice's frames at once. - head_frames = 1 - def _head(self, source, memory_state, memory_cache): box = [self.conv_in(source.pop(), memory_state=memory_state, memory_cache=memory_cache)] for down_block in self.down_blocks[:self.head_blocks]: box = [down_block.run(box, memory_state, memory_cache)] return box.pop() - def forward(self, sample, memory_state, memory_cache): + def forward(self, sample, memory_state, memory_cache, head_frames): + """``head_frames`` per run through the spatial-only head; None runs the slice at once.""" frame_pixels = sample.size(-2) * sample.size(-1) source = [sample] del sample - box = [_frame_chunked(self, self._head, source, self.head_frames, memory_state, memory_cache, frame_pixels)] + box = [_frame_chunked(self, self._head, source, head_frames, memory_state, memory_cache, frame_pixels)] for down_block in self.down_blocks[self.head_blocks:]: box = [down_block.run(box, memory_state, memory_cache)] sample = self.mid_block(box.pop(), memory_state, memory_cache) @@ -1210,9 +1272,6 @@ def __init__(self, in_channels: int, out_channels: int, block_out_channels, laye self.conv_norm_out = ops.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6) self.conv_out = InflatedCausalConv3d(block_out_channels[0], out_channels, 3, padding=1) - # Frames per pass through the spatial-only tail; None runs a slice's frames at once. - tail_frames = 1 - def _tail(self, source, memory_state, memory_cache): for up_block in self.up_blocks[self.temporal_up_num:]: source = [up_block.run(source, memory_state, memory_cache)] @@ -1222,7 +1281,8 @@ def _tail(self, source, memory_state, memory_cache): out = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache) return out - def forward(self, sample, memory_state, memory_cache): + def forward(self, sample, memory_state, memory_cache, tail_frames): + """``tail_frames`` per run through the spatial-only tail; None runs the slice at once.""" sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache) sample = self.mid_block(sample, memory_state, memory_cache) @@ -1247,11 +1307,10 @@ def upsample_and_tail(source, state, cache): for n, box in enumerate(up.frames(source, state, cache))] return torch.cat(outs, dim=2) box = [up.run(source, state, cache)] - return _frame_chunked(self, self._tail, box, self.tail_frames, state, cache, frame_pixels) + return _frame_chunked(self, self._tail, box, tail_frames, state, cache, frame_pixels) source = [sample] del sample - return _frame_chunked(last, upsample_and_tail, source, self.tail_frames and max(1, self.tail_frames // up.temporal_ratio), - memory_state, memory_cache, frame_pixels) + return _frame_chunked(last, upsample_and_tail, source, tail_frames, memory_state, memory_cache, frame_pixels) class VideoAutoencoderKLWrapper(nn.Module): @@ -1266,6 +1325,7 @@ def __init__(self): super().__init__() self.encoder = Encoder3D(3, SEEDVR2_LATENT_CHANNELS, BYTEDANCE_BLOCK_OUT_CHANNELS, layers_per_block=2, norm_num_groups=32, temporal_down_num=2) + self.current_patcher = None # set by the VAE wrapper self.decoder = Decoder3D(SEEDVR2_LATENT_CHANNELS, 3, BYTEDANCE_BLOCK_OUT_CHANNELS, layers_per_block=2, norm_num_groups=32, temporal_up_num=2) set_norm_limit(BYTEDANCE_VAE_NORM_MEM_GIB) @@ -1273,30 +1333,51 @@ def __init__(self): if isinstance(m, InflatedCausalConv3d): m.memory_limit = BYTEDANCE_VAE_CONV_MEM_GIB - def _encode(self, x, memory_state=MemoryState.DISABLED, memory_cache=None): - h = self.encoder(x.to(self.device), memory_state=memory_state, memory_cache=memory_cache) + def _encode(self, x, head_frames, memory_state=MemoryState.DISABLED, memory_cache=None): + h = self.encoder(x.to(self.device), memory_state=memory_state, memory_cache=memory_cache, head_frames=head_frames) return h.to(x.device) - def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None): - output = self.decoder(z.to(self.device), memory_state=memory_state, memory_cache=memory_cache) + def _decode(self, z, tail_frames, memory_state=MemoryState.DISABLED, memory_cache=None): + output = self.decoder(z.to(self.device), memory_state=memory_state, memory_cache=memory_cache, tail_frames=tail_frames) return output.to(z.device) def slicing_encode(self, x: torch.Tensor) -> torch.Tensor: if x.size(0) > 1: # a video at a time: the buffered path and the memory estimate are per video return torch.cat([self.slicing_encode(x[i:i + 1]) for i in range(x.size(0))]) - if (x.shape[2] - 1) <= self.slicing_sample_min_size: - return self._encode(x.to(self.device)) - memory_cache = CausalMemoryCache(offload=_offload_caches_for(x.shape[-2] * x.shape[-1])) - split_size = max(self.slicing_sample_min_size, self.temporal_downsample_factor) + area = x.shape[-2] * x.shape[-1] + factor = _eager_factor(x.dtype, SEEDVR2_EAGER_ENCODE_FACTOR) + latents = -(-(x.shape[2] - 1) // self.temporal_downsample_factor) + slice_bytes = area * SEEDVR2_ENCODE_SLICE_BYTES_PER_PIXEL * factor + short_tail = (x.shape[2] - 1) % self.temporal_downsample_factor / self.temporal_downsample_factor # joins the slice before it + slice_latents, memory_cache = _plan_pass(self.device, self.current_patcher.get_free_memory(self.device), area, latents, self.comfy_memory_used_encode(x.shape, x.dtype) + short_tail * slice_bytes, + slice_bytes, area * SEEDVR2_ENCODE_TAIL_BYTES_PER_PIXEL + SEEDVR2_ENCODE_TAIL_FIXED_BYTES, x.dtype) + head_frames = None if slice_latents else 1 + split_size = max(self.slicing_sample_min_size, self.temporal_downsample_factor, slice_latents * self.temporal_downsample_factor) + if (x.shape[2] - 1) <= split_size: + memory_cache.free() + return self._encode(x.to(self.device), head_frames=head_frames) x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2)) # _encode moves each to the device if len(x_slices) > 1 and x_slices[-1].shape[2] < self.temporal_downsample_factor: x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2) x_slices.pop() + encoded_slices = [] + + def keep(encoded): + if memory_cache.compiled and encoded.is_cuda: + with comfy.model_prefetch.pause_malloc_graph(): + encoded_slices.append(encoded.clone()) + else: + encoded_slices.append(encoded) + try: - encoded_slices = [self._encode(torch.cat((x[:, :, :1], x_slices[0]), dim=2), - memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache)] - for x_slice in x_slices[1:]: - encoded_slices.append(self._encode(x_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)) + with memory_cache.scope(self.device): + memory_cache.next_slice(self.device, self.encoder, x_slices[0].size(2) + 1) + keep(self._encode(torch.cat((x[:, :, :1], x_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache, head_frames=head_frames)) + for x_slice in x_slices[1:]: + memory_cache.next_slice(self.device, self.encoder, x_slice.size(2)) + keep(self._encode(x_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache, head_frames=head_frames)) + memory_cache.next_slice(self.device) finally: memory_cache.free() return torch.cat(encoded_slices, dim=2) @@ -1307,11 +1388,20 @@ def slicing_decode(self, z: torch.Tensor, output_buffer=None) -> torch.Tensor: outs = [self.slicing_decode(z[i:i + 1], None if output_buffer is None else output_buffer[i:i + 1]) for i in range(z.size(0))] return output_buffer if output_buffer is not None else torch.cat(outs) - if (z.shape[2] - 1) <= self.slicing_latent_min_size: - decoded = self._decode(z) + area = z.shape[-2] * z.shape[-1] * self.spatial_downsample_factor ** 2 + factor = _eager_factor(z.dtype, SEEDVR2_EAGER_DECODE_FACTOR) + slice_latents, memory_cache = _plan_pass(self.device, self.current_patcher.get_free_memory(self.device), area, z.shape[2] - 1, (area * SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + SEEDVR2_DECODE_FIXED_BYTES) * factor, + area * SEEDVR2_DECODE_SLICE_BYTES_PER_FRAME_PIXEL * factor, + area * SEEDVR2_DECODE_TAIL_BYTES_PER_FRAME_PIXEL + SEEDVR2_DECODE_TAIL_FIXED_BYTES, z.dtype) + tail_frames = None if slice_latents else 1 + split_size = max(self.slicing_latent_min_size, slice_latents) + if (z.shape[2] - 1) <= split_size: + memory_cache.free() + decoded = self._decode(z, tail_frames=tail_frames) return decoded if output_buffer is None else output_buffer.copy_(decoded) - memory_cache = CausalMemoryCache(offload=_offload_caches_for(z.shape[-2] * z.shape[-1] * self.spatial_downsample_factor ** 2)) - z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2) + z_slices = z[:, :, 1:].split(split_size=split_size, dim=2) + if memory_cache.compiled and output_buffer is None: # the frames outlive their slice + output_buffer = torch.empty(self.decode_output_shape(z.shape), device=z.device, dtype=z.dtype) decoded_slices = [] write_pos = 0 @@ -1324,10 +1414,14 @@ def emit(decoded): write_pos += decoded.size(2) try: - emit(self._decode(torch.cat((z[:, :, :1], z_slices[0]), dim=2), - memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache)) - for z_slice in z_slices[1:]: - emit(self._decode(z_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)) + with memory_cache.scope(self.device): + memory_cache.next_slice(self.device, self.decoder, z_slices[0].size(2) + 1) + emit(self._decode(torch.cat((z[:, :, :1], z_slices[0]), dim=2), + memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache, tail_frames=tail_frames)) + for z_slice in z_slices[1:]: + memory_cache.next_slice(self.device, self.decoder, z_slice.size(2)) + emit(self._decode(z_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache, tail_frames=tail_frames)) + memory_cache.next_slice(self.device) finally: memory_cache.free() return output_buffer if output_buffer is not None else torch.cat(decoded_slices, dim=2) diff --git a/comfy/ldm/wan/model_animate2.py b/comfy/ldm/wan/model_animate2.py index 45629af4dc2..744401ecb40 100644 --- a/comfy/ldm/wan/model_animate2.py +++ b/comfy/ldm/wan/model_animate2.py @@ -210,7 +210,7 @@ def prefetch(self, i, device, dtype): with comfy.model_prefetch.pause_malloc_graph(): self._staging[buf_key] = [torch.empty(t.shape, dtype=buf_key[1], device=device) for _ in range(2)] r = self._staging[buf_key][i % 2] - stream = comfy.model_management.get_offload_stream(device) + stream = comfy.model_management.get_data_stream(device) cs = comfy.model_management.current_stream(device) if stream is not None and cs is not None: # Wait for staging allocation and the previous consumer's reads before overwriting the buffer. diff --git a/comfy/ldm/wan/model_vsr.py b/comfy/ldm/wan/model_vsr.py new file mode 100644 index 00000000000..9fd4a72bcae --- /dev/null +++ b/comfy/ldm/wan/model_vsr.py @@ -0,0 +1,372 @@ +# Wan video super-resolution +# RelayVSR https://github.com/kopperx/RelayVSR (Apache-2.0): one-step Wan keyframes guiding a small per-frame transformer +import math + +import comfy_kitchen +import torch +import torch.nn as nn +import torch.nn.functional as F + +from comfy.ldm.flux.layers import EmbedND +from comfy.ldm.flux.math import apply_rope1 +from comfy.ldm.wan.model import WanAttentionBlock, WanModel, WanSelfAttention, modulate, sinusoidal_embedding_1d +import comfy.ldm.common_dit +import comfy.model_management +import comfy.model_prefetch +import comfy.ops +import comfy.utils + +ops = comfy.ops.disable_weight_init + +RELAY_KEYFRAME_INTERVAL = 15 +RELAY_WAN_CONTEXT = 4 +RELAY_WAN_WINDOW = (22, 40) +RELAY_WAN_CHUNK_TOKENS = 32768 +# the per-layer cache needs about as much VRAM as this many keyframes of a single pass +RELAY_WAN_CACHE_KEYFRAMES = 7 +RELAY_FLASH_CONDITION_WINDOW = (48, 48) +RELAY_FLASH_VIDEO_WINDOW = (24, 24) + + +def relay_keyframes(frames): + return list(range(0, frames - 1, RELAY_KEYFRAME_INTERVAL)) + [frames - 1] + + +def relay_latent_size(height, width, latent_stride): + # whole 2x2 patches of the model's pixel stride + stride = latent_stride * 2 + return math.ceil(height / stride) * 2, math.ceil(width / stride) * 2 + + +class RelayLQProjection(nn.Module): + def __init__(self, upscale_factor, hidden_dim=768, hidden_dim2=3072, out_dim=3072, eps=1e-6, device=None, dtype=None, operations=None): + super().__init__() + self.upscale_factor = upscale_factor + self.conv1 = operations.Conv2d(3 * (4 * upscale_factor) ** 2, hidden_dim, 1, device=device, dtype=dtype) + self.norm1 = operations.RMSNorm(hidden_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + self.conv2 = operations.Conv2d(hidden_dim, hidden_dim2, 3, stride=2, padding=1, device=device, dtype=dtype) + self.norm2 = operations.RMSNorm(hidden_dim2, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + self.linear = operations.Linear(hidden_dim2, out_dim, device=device, dtype=dtype) + + def forward(self, video): + b, c, f, h, w = video.shape + x = video.movedim(2, 1).reshape(b * f, c, h, w) + if self.upscale_factor == 4: + x = F.interpolate(x, scale_factor=4, mode="bilinear", align_corners=False) + x = self.conv1(F.pixel_unshuffle(x, 4 * self.upscale_factor)) + x = F.silu(self.norm1(x.movedim(1, -1))).movedim(-1, 1) + x = F.silu(self.norm2(self.conv2(x).flatten(2).transpose(1, 2))) + return self.linear(x).view(b, -1, self.linear.out_features) + + +class RelayVSRSelfAttention(WanSelfAttention): + def forward(self, x, freqs, cache, start, store, transformer_options={}): + # causal over the last keyframes, per-query windows slid to stay inside the frame + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + frames, height, width = transformer_options["grid_sizes"] + grid = (b, frames, height, width, n, d) + q = apply_rope1(self.norm_q(self.q(x)).view(b, s, n, d), freqs).view(grid) + k = apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs).view(grid) + v = self.v(x).view(grid) + # keyframes before this chunk come from the cache, which holds the last ones in order as int8 rows with a scale each + past = min(start, RELAY_WAN_CONTEXT - 1) + keys, values = k, v + if past > 0: + cached, scales = cache + k_past, v_past = (cached[:, :, :past].to(k.dtype) * scales[:, :, :past]).unbind(0) + keys, values = torch.cat((k_past, k), dim=1), torch.cat((v_past, v), dim=1) + x = comfy_kitchen.window_attention(q, keys, values, RELAY_WAN_WINDOW, temporal_window=RELAY_WAN_CONTEXT) + if store: + # the cache moves on to the last keyframes for the next chunk, older cached ones first when this one is short + cached, scales = cache + new = min(frames, RELAY_WAN_CONTEXT - 1) + old = min(past, RELAY_WAN_CONTEXT - 1 - new) + if old > 0: + cached[:, :, :old] = cached[:, :, past - old:past].clone() + scales[:, :, :old] = scales[:, :, past - old:past].clone() + for i, rows in enumerate((k[:, frames - new:], v[:, frames - new:])): + scale = (rows.abs().amax(dim=-1, keepdim=True) / 127.0).clamp_(min=torch.finfo(rows.dtype).tiny) + scales[i, :, old:old + new] = scale + cached[i, :, old:old + new] = (rows.float() / scale.float()).round_().clamp_(-127.0, 127.0) + return self.o(x.view(b, s, n * d)) + + +class RelayVSRAttentionBlock(WanAttentionBlock): + def forward(self, x, e, freqs, context, cache, start, store, transformer_options={}): + # residual updated in place, nothing allocated in the block outlives its memory compiler scope + e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e).unbind(2) + x.addcmul_(self.self_attn(modulate(x, self.norm1, e[0], e[1]), freqs, cache, start, store, transformer_options=transformer_options), e[2]) + x.add_(self.cross_attn(self.norm3(x), context, transformer_options=transformer_options)) + x.addcmul_(self.ffn(modulate(x, self.norm2, e[3], e[4])), e[5]) + return x + + +class RelayVSRWanModel(WanModel): + def __init__(self, model_type=None, upscale_factor=4, operations=None, device=None, dtype=None, **kwargs): + super().__init__(model_type="t2v", wan_attn_block_class=RelayVSRAttentionBlock, operations=operations, device=device, dtype=dtype, **kwargs) + self.register_buffer("positive_conditioning", torch.empty((self.text_len, self.text_dim), device=device, dtype=dtype)) + operation_settings = {"operations": operations, "device": device, "dtype": dtype} + for block in self.blocks: + block.self_attn = RelayVSRSelfAttention(self.dim, self.num_heads, eps=self.eps, operation_settings=operation_settings) + self.lq_proj_in = RelayLQProjection(upscale_factor, out_dim=self.dim, device=device, dtype=dtype, operations=operations) + self.latent_stride = 16 // upscale_factor + + def forward_orig(self, x, t, context, freqs=None, lq_pixels=None, transformer_options={}, **kwargs): + b, frames = x.shape[0], x.shape[2] + height, width = x.shape[3] // self.patch_size[1], x.shape[4] // self.patch_size[2] + tokens = height * width + e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(dtype=x.dtype)) + e = e.reshape(t.shape[0], -1, e.shape[-1]) + e0 = self.time_projection(e).unflatten(2, (6, self.dim)) + context = self.text_embedding(context) + + # keyframes go through in chunks, every layer's cache carries the keys and values of the last ones across + chunk = max(1, RELAY_WAN_CHUNK_TOKENS // tokens) + if frames <= chunk + RELAY_WAN_CACHE_KEYFRAMES: + chunk = frames + shape = (2, b, RELAY_WAN_CONTEXT - 1, height, width, self.num_heads, self.dim // self.num_heads) + caches = [(torch.empty(shape, device=x.device, dtype=torch.int8), torch.empty((*shape[:-1], 1), device=x.device, dtype=x.dtype)) if frames > chunk else None for _ in self.blocks] + out = torch.empty((b, self.out_dim, *x.shape[2:]), device=x.device, dtype=x.dtype) + blocks_replace = transformer_options.get("patches_replace", {}).get("dit", {}) + transformer_options["total_blocks"] = len(self.blocks) + transformer_options["block_type"] = "double" + starts = range(0, frames, chunk) + # the sampler's bars only count the one step + progress = comfy.utils.ProgressBar(len(starts) * len(self.blocks)) + with comfy.utils.model_trange(len(starts) * len(self.blocks), desc="RelayVSR", unit="block") as pbar: + for start in starts: + end = min(start + chunk, frames) + # only a following chunk reads the cache + store = end < frames + grid_sizes = (end - start, height, width) + transformer_options["grid_sizes"] = grid_sizes + # the projected LQ frames go first so the sum is row-major, the blocks update it in place + hidden = self.lq_proj_in(lq_pixels[:, :, start:end]) + self.patch_embedding(x[:, :, start:end].float()).to(x.dtype).flatten(2).transpose(1, 2) + chunk_freqs = freqs[:, start * tokens:end * tokens] + prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.blocks), x.device, transformer_options) + comfy.model_prefetch.malloc_graph_begin(x.device) + for i, block in enumerate(self.blocks): + comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, block, x.dtype, malloc_scope="block") + transformer_options["block_index"] = i + if ("double_block", i) in blocks_replace: + def block_wrap(args): + return {"img": block(args["img"], args["vec"], args["pe"], args["txt"], caches[i], start, store, transformer_options=args["transformer_options"])} + hidden = blocks_replace[("double_block", i)]({"img": hidden, "txt": context, "vec": e0, "pe": chunk_freqs, "transformer_options": transformer_options}, {"original_block": block_wrap})["img"] + else: + hidden = block(hidden, e0, chunk_freqs, context, caches[i], start, store, transformer_options=transformer_options) + pbar.update(1) + progress.update(1) + comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, None, malloc_scope="block") + comfy.model_prefetch.malloc_graph_end() + out[:, :, start:end] = self.unpatchify(self.head(hidden, e), grid_sizes) + return out + + def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, lq_pixels=None, **kwargs): + if lq_pixels is None: + raise ValueError("RelayVSR needs the low-quality video, connect the model through the RelayVSR Conditioning node.") + t, h, w = x.shape[2:] + x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size, padding_mode="replicate") + tp, hp, wp = x.shape[2:] + lq_height, lq_width = hp * self.latent_stride, wp * self.latent_stride + lq_pixels = lq_pixels[..., :lq_height, :lq_width] + lq_pixels = F.pad(lq_pixels, (0, lq_width - lq_pixels.shape[-1], 0, lq_height - lq_pixels.shape[-2], 0, 0), mode="replicate") + freqs = self.rope_encode(tp, hp, wp, device=x.device, dtype=x.dtype, transformer_options=transformer_options) + return self.forward_orig(x, timestep, context, freqs=freqs, lq_pixels=lq_pixels, transformer_options=transformer_options, **kwargs)[:, :, :t, :h, :w] + + +class RelayFlashAttention(nn.Module): + def __init__(self, dim, num_heads, kv_groups, eps, device=None, dtype=None): + super().__init__() + self.num_heads = num_heads + self.kv_groups = kv_groups + self.head_dim = dim // num_heads + self.query = ops.Linear(dim, dim, device=device, dtype=dtype) + self.key = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype) + self.value = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype) + self.condition_key = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype) + self.condition_value = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype) + self.key_norm = ops.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + self.value_norm = ops.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + self.output = ops.Linear(dim, dim, device=device, dtype=dtype) + + def project(self, x, key, value, freqs): + b, s, _ = x.shape + q = apply_rope1(self.query(x).view(b, s, self.num_heads, self.head_dim), freqs) + k = self.key_norm(key(x).view(b, s, self.kv_groups, self.head_dim)) + v = self.value_norm(value(x).view(b, s, self.kv_groups, self.head_dim)) + return q, k, v + + def condition(self, x, freqs, height, width): + # the rotated condition keys are reused as they are by every frame of the window + b, s, c = x.shape + q, k, v = self.project(x, self.condition_key, self.condition_value, freqs) + k = apply_rope1(k, freqs).view(b, 1, height, width, self.kv_groups, self.head_dim) + v = v.view(k.shape) + x = comfy_kitchen.window_attention(q.view(b, 1, height, width, self.num_heads, self.head_dim), k, v, RELAY_FLASH_CONDITION_WINDOW, clamp=False) + return self.output(x.view(b, s, c)), (k, v) + + def video(self, x, freqs, freqs_past, height, width, condition, past): + b, s, c = x.shape + plane = (b, 1, height, width, self.kv_groups, self.head_dim) + q, k, v = self.project(x, self.key, self.value, freqs) + # both condition keyframes, then the previous frame and this one + (k_start, v_start), (k_end, v_end) = condition + keys, values = [k_start, k_end], [v_start, v_end] + if past is not None: + keys.append(apply_rope1(past[0], freqs_past).view(plane)) + values.append(past[1].view(plane)) + keys.append(apply_rope1(k, freqs).view(plane)) + values.append(v.view(plane)) + windows = [RELAY_FLASH_CONDITION_WINDOW] * 2 + [RELAY_FLASH_VIDEO_WINDOW] * (len(keys) - 2) + x = comfy_kitchen.window_attention(q.view(b, 1, height, width, self.num_heads, self.head_dim), torch.cat(keys, dim=1), torch.cat(values, dim=1), windows, clamp=False) + return self.output(x.view(b, s, c)), (k, v) + + +class RelayFlashFeedForward(nn.Module): + def __init__(self, dim, hidden_dim, device=None, dtype=None): + super().__init__() + self.gate_up_proj = ops.Linear(dim, 2 * hidden_dim, bias=False, device=device, dtype=dtype) + self.down = ops.Linear(hidden_dim, dim, bias=False, device=device, dtype=dtype) + + def forward(self, x): + return comfy.ops.linear_input_act(self.down, self.gate_up_proj(x), "swiglu") + + +class RelayFlashBlock(nn.Module): + def __init__(self, dim, num_heads, kv_groups, hidden_dim, eps, device=None, dtype=None): + super().__init__() + self.attention_norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + self.attention = RelayFlashAttention(dim, num_heads, kv_groups, eps, device=device, dtype=dtype) + self.feed_forward_norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + self.feed_forward = RelayFlashFeedForward(dim, hidden_dim, device=device, dtype=dtype) + + def condition(self, x, *args): + y, kv = self.attention.condition(self.attention_norm(x), *args) + x = x + y + return x + self.feed_forward(self.feed_forward_norm(x)), kv + + def video(self, x, *args): + y, kv = self.attention.video(self.attention_norm(x), *args) + x = x + y + return x + self.feed_forward(self.feed_forward_norm(x)), kv + + +class RelayFlashEncoder(nn.Module): + def __init__(self, last_stride, dim, eps, device=None, dtype=None): + super().__init__() + self.layers = nn.Sequential( + ops.Conv2d(3, 64, 3, stride=2, padding=1, device=device, dtype=dtype), nn.SiLU(), + ops.Conv2d(64, 128, 3, stride=2, padding=1, device=device, dtype=dtype), nn.SiLU(), + ops.Conv2d(128, 128, 3, stride=last_stride, padding=1, device=device, dtype=dtype), nn.SiLU(), + ops.Conv2d(128, dim, 1, device=device, dtype=dtype)) + self.norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + + def forward(self, x): + return self.norm(self.layers(x).flatten(2).transpose(1, 2)) + + +class RelayFlashConditionEncoder(nn.Module): + def __init__(self, in_channels, dim, eps, device=None, dtype=None): + super().__init__() + self.projection = ops.Conv2d(in_channels, dim, 1, device=device, dtype=dtype) + self.norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype) + + def forward(self, x): + return self.norm(self.projection(x).flatten(2).transpose(1, 2)) + + +class RelayFlashBackbone(nn.Module): + def __init__(self, depth, dim, num_heads, kv_groups, hidden_dim, eps, device=None, dtype=None): + super().__init__() + self.blocks = nn.ModuleList([RelayFlashBlock(dim, num_heads, kv_groups, hidden_dim, eps, device=device, dtype=dtype) for _ in range(depth)]) + + +class RelayFlash(nn.Module): + """Per-frame decoder conditioned on the keyframe latents on either side of the frame, VAE slot of RelayVSR.""" + + comfy_has_chunked_io = True + + def __init__(self, upscale_factor, latent_channels=48, dim=512, depth=10, num_heads=8, kv_groups=2, hidden_dim=1408, head_dim=1024, eps=1e-5, device=None, dtype=None): + super().__init__() + self.upscale_factor = upscale_factor + self.latent_stride = 16 // upscale_factor + self.lr_encoder = RelayFlashEncoder(self.latent_stride // 4, dim, eps, device=device, dtype=dtype) + self.condition_encoder = RelayFlashConditionEncoder(latent_channels, dim, eps, device=device, dtype=dtype) + self.video_type_embedding = nn.Parameter(torch.empty(1, 1, dim, device=device, dtype=dtype)) + self.condition_type_embedding = nn.Parameter(torch.empty(1, 1, dim, device=device, dtype=dtype)) + self.backbone = RelayFlashBackbone(depth, dim, num_heads, kv_groups, hidden_dim, eps, device=device, dtype=dtype) + self.spatial_head = nn.Sequential(ops.Linear(dim, head_dim, device=device, dtype=dtype), nn.SiLU()) + self.residual_projection = ops.Linear(head_dim, 768, device=device, dtype=dtype) + self.rope_embedder = EmbedND(dim=dim // num_heads, theta=10000.0, axes_dim=[16, 24, 24]) + + def grid_rope(self, t, height, width, device): + ids = torch.zeros((height, width, 3), device=device, dtype=torch.float32) + ids[..., 0] = t + ids[..., 1] = torch.arange(height, device=device, dtype=torch.float32)[:, None] + ids[..., 2] = torch.arange(width, device=device, dtype=torch.float32)[None] + return self.rope_embedder(ids.reshape(1, -1, 3)).movedim(1, 2) + + def prepare_condition(self, latent, freqs): + height, width = latent.shape[-2:] + x = self.condition_encoder(latent) + comfy.ops.cast_to_input(self.condition_type_embedding, latent) + out = [] + for block in self.backbone.blocks: + x, kv = block.condition(x, freqs, height, width) + out.append(kv) + return out + + def decode_frame(self, lr, dtype, freqs, freqs_past, condition, history, detail): + height, width = lr.shape[-2] // self.latent_stride, lr.shape[-1] // self.latent_stride + x = self.lr_encoder(lr.to(dtype)) + x = x + comfy.ops.cast_to_input(self.video_type_embedding, x) + next_history = [] + for block, cond, past in zip(self.backbone.blocks, condition, history): + x, kv = block.video(x, freqs, freqs_past, height, width, cond, past) + next_history.append(kv) + residual = F.pixel_shuffle(self.residual_projection(self.spatial_head(x)).transpose(1, 2).reshape(lr.shape[0], 768, height, width), 16) + if detail != 1.0: + residual = residual * detail + return F.interpolate(lr, scale_factor=self.upscale_factor, mode="bicubic", align_corners=False) + residual, next_history + + def decode_output_shape(self, input_shape, lq_images=None, detail=1.0): + if lq_images is None: + raise ValueError("The RelayVSR Flash decoder needs the low-quality frames, decode with the VSR Decode node.") + frames, height, width = lq_images.shape[:3] + # the x2 and x4 decoders load from the same weight shapes, a latent of the other factor would decode garbage or fail on the copy + if tuple(input_shape[2:]) != (len(relay_keyframes(frames)), *relay_latent_size(height, width, self.latent_stride)): + raise ValueError(f"The latent {tuple(input_shape[2:])} doesn't fit the x{self.upscale_factor} RelayVSR Flash decoder and these {frames} {width}x{height} frames: use the RelayVSR model and Flash decoder of the same factor, and the same video in RelayVSR Conditioning and VSR Decode.") + return (input_shape[0], 3, frames, height * self.upscale_factor, width * self.upscale_factor) + + def decode(self, x, lq_images=None, output_buffer=None, detail=1.0): + # detail scales what the model adds to the bicubic upscale of each frame + out = output_buffer + if out is None: + out = torch.empty(self.decode_output_shape(x.shape, lq_images), device=comfy.model_management.intermediate_device()) + b, _, _, height, width = x.shape + frames, lq_height, lq_width = lq_images.shape[:3] + depth = len(self.backbone.blocks) + keyframes = relay_keyframes(frames) + device = x.device + freqs, freqs_past = self.grid_rope(0, height, width, device), self.grid_rope(-1, height, width, device) + pbar = comfy.utils.ProgressBar(frames) + # finished frames go to the output without the GPU waiting on each copy + with comfy.model_management.HostWriter(device, out, (b, 3, *out.shape[-2:])) as writer, comfy.utils.model_trange(frames, desc="RelayVSR Flash decode", unit="frame") as console: + start = self.prepare_condition(x[:, :, 0], freqs) + for window, (first, last) in enumerate(zip(keyframes, keyframes[1:] or keyframes)): + comfy.model_management.throw_exception_if_processing_interrupted() + end = self.prepare_condition(x[:, :, min(window + 1, len(keyframes) - 1)], freqs) + condition = list(zip(start, end)) + history = [None] * depth + # one upload per window, an upload from pageable memory waits for the whole stream + lr_window = lq_images[first:last + 1].to(device=device)[..., :3].movedim(-1, 1) * 2.0 - 1.0 + lr_window = F.pad(lr_window, (0, width * self.latent_stride - lq_width, 0, height * self.latent_stride - lq_height), mode="replicate") + for i, frame in enumerate(range(first, last + 1)): + decoded, history = self.decode_frame(lr_window[i:i + 1].expand(b, -1, -1, -1), x.dtype, freqs, freqs_past, condition, history, detail) + # the shared keyframe was already output by the previous window, here it only seeds the history + if window == 0 or frame > first: + # finished [0, 1] pixels on the GPU, the VAE's process_output is the identity + writer.write(out[:, :, frame], decoded[..., :out.shape[-2], :out.shape[-1]].add_(1.0).div_(2.0).clamp_(0.0, 1.0).to(out.dtype)) + pbar.update(1) + console.update(1) + start = end + return out diff --git a/comfy/memory_management.py b/comfy/memory_management.py index abb387239f8..0a40b58e35a 100644 --- a/comfy/memory_management.py +++ b/comfy/memory_management.py @@ -5,7 +5,7 @@ from typing import NamedTuple import comfy_aimdo.host_buffer -from comfy.quant_ops import QuantizedTensor +from comfy.quant_ops import QuantizedTensor, ck class TensorFileSlice(NamedTuple): @@ -190,6 +190,50 @@ def interpret_gathered_like(tensors, gathered): return dest_views +def cat_gathered(tensors, gathered): + """Concatenate quantized tensors laid out in order in gathered into one, in place. + + Per-row fields are concatenated; every other field and parameter must match and is + shared. The codes are moved down over the earlier tensors' metadata, which is saved + first and rewritten after the last codes, so the result is laid out exactly like + interpret_gathered_like would lay out one tensor of the summed rows. Returns None + when the tensors cannot be concatenated.""" + inner_tensors, ctx = tensors[0].__tensor_flatten__() + rows = [t._params.orig_shape[0] for t in tensors] + orig_shape = (sum(rows), *tensors[0]._params.orig_shape[1:]) + for t in tensors[1:]: + other_inner, other_ctx = t.__tensor_flatten__() + other_ctx["non_tensor_fields"]["orig_shape"] = (rows[0], *t._params.orig_shape[1:]) + if (other_inner != inner_tensors or other_ctx["layout_cls"] != ctx["layout_cls"] + or other_ctx["non_tensor_fields"] != ctx["non_tensor_fields"]): + return None + fields = {} + for attr in inner_tensors[1:]: + values = [getattr(t, attr) for t in tensors] + if all(v.ndim > 0 and v.shape[0] == n and v.shape[1:] == values[0].shape[1:] and v.dtype == values[0].dtype for v, n in zip(values, rows)): + fields[attr] = torch.cat(values) + elif all(torch.equal(v, values[0]) for v in values[1:]): + fields[attr] = values[0].clone() + else: + return None + + offset = 0 + for t in tensors: + qdata = t._qdata + if qdata.data_ptr() < gathered.data_ptr() + offset: + raise RuntimeError("Tensors must sit in order within gathered") + ck.memmove(gathered[offset:offset + qdata.nbytes], qdata.view(torch.uint8).view(-1)) + offset += qdata.nbytes + qdata = tensors[0]._qdata + actuals = {"_qdata": gathered[:offset].view(qdata.dtype).view(orig_shape[0], *qdata.shape[1:])} + offset = vram_aligned_size(actuals["_qdata"]) + for attr, value in fields.items(): + actuals[attr] = gathered[offset:offset + value.nbytes].view(value.dtype).view(value.shape) + actuals[attr].copy_(value) + offset += vram_aligned_size(value) + ctx["non_tensor_fields"]["orig_shape"] = orig_shape + return QuantizedTensor.__tensor_unflatten__(actuals, ctx, 0, 0) + aimdo_enabled = False extra_ram_release_callback = None diff --git a/comfy/model_base.py b/comfy/model_base.py index 42ccd5dbfbc..39db511fc38 100644 --- a/comfy/model_base.py +++ b/comfy/model_base.py @@ -52,6 +52,7 @@ import comfy.ldm.wan.model_animate2 import comfy.ldm.wan.ar_model import comfy.ldm.wan.model_wandancer +import comfy.ldm.wan.model_vsr import comfy.ldm.hunyuan3d.model import comfy.ldm.triposplat.model import comfy.ldm.hidream.model @@ -2045,6 +2046,19 @@ def extra_conds_shapes(self, **kwargs): return out +class WAN22_RelayVSR(WAN21): + def __init__(self, model_config, model_type=ModelType.FLOW, device=None): + super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model_vsr.RelayVSRWanModel) + self.image_to_video = False + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + lq_images = kwargs.get("lq_images", None) + if lq_images is not None: + out['lq_pixels'] = comfy.conds.CONDRegular(lq_images[..., :3].movedim(-1, 0).unsqueeze(0) * 2.0 - 1.0) + return out + + class WAN21_SCAIL2(WAN21_SCAIL): """SCAIL-2: SCAIL-Preview + an additive binary multi-identity mask stream.""" @@ -2200,7 +2214,7 @@ def audio_scale(self): """Scale the sampler carries the audio stream at, 1.0 when not sampling the packed latent.""" if self.latent_shapes is None or len(self.latent_shapes) < 2: return 1.0 - return self.model_sampling.audio_scale + return getattr(self.model_sampling, "audio_scale", 1.0) def _scale_audio_slice(self, latent, scale): # the sampler carries the audio stream scaled onto the video schedule diff --git a/comfy/model_detection.py b/comfy/model_detection.py index 04d76db2a31..f7e412a2b48 100644 --- a/comfy/model_detection.py +++ b/comfy/model_detection.py @@ -787,6 +787,10 @@ def detect_unet_config(state_dict, key_prefix, metadata=None): dit_config["model_type"] = "scail" elif '{}patch_embedding_global.weight'.format(key_prefix) in state_dict_keys: dit_config["model_type"] = "wandancer" + elif '{}lq_proj_in.conv1.weight'.format(key_prefix) in state_dict_keys: + dit_config["model_type"] = "relayvsr" + # the LQ frames go in pixel-unshuffled by 4 * upscale_factor + dit_config["upscale_factor"] = math.isqrt(state_dict['{}lq_proj_in.conv1.weight'.format(key_prefix)].shape[1] // 3) // 4 else: if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys: dit_config["model_type"] = "i2v" diff --git a/comfy/model_management.py b/comfy/model_management.py index e33147e74b9..405f1501256 100644 --- a/comfy/model_management.py +++ b/comfy/model_management.py @@ -1490,6 +1490,21 @@ def reset_cast_buffers(): STREAM_AIMDO_CAST_BUFFERS.clear() soft_empty_cache() +def new_stream(device): + if is_device_cuda(device): + s = torch.cuda.Stream(device=device, priority=0) + s.as_context = torch.cuda.stream + elif is_device_xpu(device): + s = torch.xpu.Stream(device=device, priority=0) + s.as_context = torch.xpu.stream + elif is_device_npu(device): + # torch.npu.stream provides the context manager for torch-npu streams. + s = torch.npu.Stream(device=device, priority=0) + s.as_context = torch.npu.stream + else: + return None + return s + def get_offload_stream(device): stream_counter = stream_counters.get(device, 0) if NUM_STREAMS == 0: @@ -1505,39 +1520,24 @@ def get_offload_stream(device): stream_counter = (stream_counter + 1) % len(ss) stream_counters[device] = stream_counter return ss[stream_counter] - elif is_device_cuda(device): - ss = [] - for k in range(NUM_STREAMS): - s1 = torch.cuda.Stream(device=device, priority=0) - s1.as_context = torch.cuda.stream - ss.append(s1) - STREAMS[device] = ss - s = ss[stream_counter] - stream_counters[device] = stream_counter - return s - elif is_device_xpu(device): - ss = [] - for k in range(NUM_STREAMS): - s1 = torch.xpu.Stream(device=device, priority=0) - s1.as_context = torch.xpu.stream - ss.append(s1) - STREAMS[device] = ss - s = ss[stream_counter] - stream_counters[device] = stream_counter - return s - elif is_device_npu(device): - ss = [] - # Match the CUDA/XPU stream interface used by the shared offload path. - # torch.npu.stream provides the context manager for torch-npu streams. - for k in range(NUM_STREAMS): - s1 = torch.npu.Stream(device=device, priority=0) - s1.as_context = torch.npu.stream - ss.append(s1) - STREAMS[device] = ss - s = ss[stream_counter] - stream_counters[device] = stream_counter - return s - return None + s = new_stream(device) + if s is None: + return None + ss = [s] + [new_stream(device) for _ in range(NUM_STREAMS - 1)] + STREAMS[device] = ss + stream_counters[device] = stream_counter + return ss[stream_counter] + +DATA_STREAMS = {} + +def get_data_stream(device): + """One stream per device for data copies such as cache staging and decoder output, kept off the + offload streams so they don't disturb the weight prefetch. Callers order on it themselves.""" + if NUM_STREAMS == 0 or torch.compiler.is_compiling(): + return None + if device not in DATA_STREAMS: + DATA_STREAMS[device] = new_stream(device) + return DATA_STREAMS[device] def sync_all_offload_streams(): if NUM_STREAMS == 0: @@ -1737,6 +1737,59 @@ def unpin_memory(tensor): return False +class HostWriter: + """Copies device tensors of one shape into slices of a CPU tensor without the device waiting on each copy: + every write goes through one of two pinned staging buffers on the data stream and lands in its slice one + write later. Writes go straight through when the target is on the device, the device has no streams or the + buffers can't be pinned.""" + + def __init__(self, device, target, shape): + self.device = device + self.staging = [] + self.pending = [None, None] + self.slot = 0 + self.stream = get_data_stream(device) if is_device_cpu(target.device) else None + if self.stream is not None: + # inactive model pins make room for the staging buffers, active ones stay + if ensure_pin_budget(2 * torch.Size(shape).numel() * target.element_size(), evict_active=False): + for _ in range(2): + buffer = torch.empty(shape, dtype=target.dtype) + if pin_memory(buffer, evict_active=False): + self.staging.append(buffer) + if len(self.staging) < 2: + self.stream = None + + def __enter__(self): + return self + + def write(self, dest, src): + if self.stream is None: + dest.copy_(src) + return + if self.pending[self.slot] is not None: + self._collect(self.slot) + self.stream.wait_stream(current_stream(self.device)) + cast_to(src, None, torch.device("cpu"), non_blocking=True, stream=self.stream, r=self.staging[self.slot]) + # src stays referenced until its copy is done + self.pending[self.slot] = (self.stream.record_event(), dest, src) + self.slot ^= 1 + + def _collect(self, slot): + event, dest, _ = self.pending[slot] + event.synchronize() + dest.copy_(self.staging[slot]) + self.pending[slot] = None + + def __exit__(self, exc_type, exc, tb): + if self.stream is not None: + if exc_type is None: + for slot in (self.slot, self.slot ^ 1): + if self.pending[slot] is not None: + self._collect(slot) + self.stream.synchronize() + for buffer in self.staging: + unpin_memory(buffer) + def sage_attention_enabled(): return args.use_sage_attention diff --git a/comfy/model_patcher.py b/comfy/model_patcher.py index 83b92edb563..9105b5a4b92 100644 --- a/comfy/model_patcher.py +++ b/comfy/model_patcher.py @@ -1986,18 +1986,26 @@ def load(self, device_to=None, lowvram_model_memory=0, force_patch_weights=False dynamic_units = list(dynamic_units) last_dynamic_units = list(last_dynamic_units) loading_by_module = {entry[-2]: entry for entry in loading} + + def unit_loading(units, loading): + for unit in units: + unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,) + # VBAR is a bump allocator, so loading a CombinedLinear's members back to + # back in member order is what makes their weights consecutive. + modules = [] + for root in unit_modules: + for module in root.modules(): + if module not in loading_by_module or module in modules: + continue + group = getattr(module, "_combined_linear", None) + modules.extend(group.linears if group is not None else (module,)) + for index, module in enumerate(modules): + loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None)) + loading = [] - for unit in dynamic_units: - unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,) - modules = [module for root in unit_modules for module in root.modules() if module in loading_by_module] - for index, module in enumerate(modules): - loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None)) + unit_loading(dynamic_units, loading) last_loading = [] - for unit in last_dynamic_units: - unit_modules = unit if isinstance(unit, (list, tuple)) else (unit,) - modules = [module for root in unit_modules for module in root.modules() if module in loading_by_module] - for index, module in enumerate(modules): - last_loading.append((*loading_by_module.pop(module), unit if index == len(modules) - 1 else None)) + unit_loading(last_dynamic_units, last_loading) loading.extend((*entry, None) for entry in loading_by_module.values()) loading.extend(last_loading) v_block = None diff --git a/comfy/model_prefetch.py b/comfy/model_prefetch.py index 75dfaf786b3..c8a9bbcdf95 100644 --- a/comfy/model_prefetch.py +++ b/comfy/model_prefetch.py @@ -8,10 +8,12 @@ import comfy_aimdo.malloc_graph import comfy_aimdo.model_vbar +import comfy_aimdo.torch from comfy.cli_args import args import comfy.memory_management import comfy.model_management import comfy.ops +from comfy.quant_ops import QuantizedTensor PREFETCH_QUEUES = [] GRAPH_WARMED_MODULES = weakref.WeakSet() @@ -62,6 +64,9 @@ def __exit__(self, exc_type, *args): def malloc_graph_scope(device): return _MallocGraphScope(device) +def malloc_graph_recording(): + return threading.get_ident() in MALLOC_GRAPHS + def malloc_graph_begin(device): global MALLOC_GRAPH_USED if not malloc_graph_enabled(device): @@ -120,6 +125,28 @@ def pin_modules(comfy_modules, device, dtype=None): if fully_faulted and dtype is not None: for comfy_module in comfy_modules: comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False) + # A miss whose modules all have VBAR slots and no runtime weight functions is resolved + # here rather than in the forward, so the block is combined and relaid out on its first + # residency and the eager warm pass runs the kernels the graph captures on the next hit. + elif dtype is not None and all(s._prefetch["signature"] is not None and not getattr(s, param_key + "_function", []) for s in comfy_modules for param_key in ("weight", "bias")): + for comfy_module in comfy_modules: + comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False) + else: + return offload_stream, fully_faulted + relayout = [] + for s in comfy_modules: + if s._v_relayout: + continue + s._v_relayout = True + group = s._combined_linear + if group is not None and s is group.linears[0]: + group.prepare(device) + relayout.append(group) + relayout.append(s) + for s in relayout: + # FIXME: Make this non_gemv_specific. + if isinstance(s, (comfy.ops.MixedPrecisionOp, comfy.ops.CombinedLinear)) and s.comfy_gemv_optimized and isinstance(s._v_weight, QuantizedTensor) and not s._full_precision_mm: + s._v_weight = s._v_weight.optimize_for_gemv(comfy_aimdo.torch.aimdo_to_tensor(s._v, device)) return offload_stream, fully_faulted def cleanup_prefetched_modules(module, comfy_modules): diff --git a/comfy/ops.py b/comfy/ops.py index d450860fe34..5eecb865c07 100644 --- a/comfy/ops.py +++ b/comfy/ops.py @@ -180,9 +180,30 @@ def get_cast_buffer(buffer_size): cast_buffer_offset += buffer_size return buffer + group_faults = {} + dropped_groups = [] for s in comfy_modules: - signature = comfy_aimdo.model_vbar.vbar_fault(s._v) - resident = comfy_aimdo.model_vbar.vbar_signature_compare(signature, s._v_signature) + group = s._combined_linear + if group is None or group._v_weight is None: + signature = comfy_aimdo.model_vbar.vbar_fault(s._v) + resident = comfy_aimdo.model_vbar.vbar_signature_compare(signature, s._v_signature) + else: + # A combined group's members hold each other's bytes, so one member's miss + # is a miss for all of them. Fault the peers now; each keeps that fault's + # pin as its own, so every member must be in comfy_modules. An uncombined + # group's members fault individually above, which also keeps the + # single-module cast path from pinning peers it never releases. + if s not in group_faults: + for m in group.linears: + signature = comfy_aimdo.model_vbar.vbar_fault(m._v) + group_faults[m] = (signature, comfy_aimdo.model_vbar.vbar_signature_compare(signature, m._v_signature)) + if not all(group_faults[m][1] for m in group.linears): + # Dropped after the loop: a member that already holds its group fault + # must not fall into the uncombined branch and take a second pin. + dropped_groups.append(group) + for m in group.linears: + group_faults[m] = (group_faults[m][0], False) + signature, resident = group_faults.pop(s) if return_faulted and (signature is None or not resident): fully_faulted = False prefetch = { @@ -279,6 +300,8 @@ def handle_pin(m, pin, source, dest, subset="weights", size=None): prefetch["needs_cast"] = needs_cast s._prefetch = prefetch + for group in dropped_groups: + group._v_weight = None if return_faulted: return offload_stream, fully_faulted return offload_stream @@ -305,12 +328,9 @@ def resolve_cast_module_with_vbar(s, dtype, device, bias_dtype, compute_dtype, w weight = params[0] bias = params[1] if prefetch["signature"] is not None: - # FIXME: Make this non_gemv_specific. - if (isinstance(s, MixedPrecisionOp) and s.comfy_gemv_optimized and isinstance(weight, QuantizedTensor) and not s._full_precision_mm - and s.weight_lowvram_function is None and len(s.weight_function) == 0): - weight = weight.optimize_for_gemv(xfer_dest) s._v_weight = weight s._v_bias = bias + s._v_relayout = False s._v_signature = prefetch["signature"] def post_cast(s, param_key, x, dtype, resident, update_weight): @@ -508,6 +528,7 @@ class CastWeightBiasOp: comfy_force_forward = False weight_function = [] bias_function = [] + _combined_linear = None class WritableWeight(torch.nn.Module, CastWeightBiasOp): """One runtime tensor, either realized or cast from writable file backing.""" @@ -1079,8 +1100,68 @@ def _try_quantized_linear_input_act(x, weight, bias, input_act, act_weight, act_ ) +class CombinedLinear: + """A residency-scoped combination of consecutive VBAR linears, not a parameter-owning module. + + All members must belong to one prefetch unit, so they fault, resolve and miss together. + Once combined, the members' weights are gone and the model is committed to the + combined weight until a miss reloads them. The model owns the split path. + """ + # The model opts the combined weight into small-M layout, as it does per MixedPrecisionOp. + comfy_gemv_optimized = False + + @property + def _full_precision_mm(self): + # The members' flag is live: a text encoder lifts it for generate (use_quantized_matmul) + # and restores it for encode, and the combined weight must follow the same contract. + return self.linears[0]._full_precision_mm + + def __init__(self, linears): + self.linears = linears + self._v = None + self._v_weight = None + for linear in self.linears: + linear._combined_linear = self + + def prepare(self, device): + """Concatenate the members' resident weights in place; they are gone afterwards.""" + weights = [] + for s in self.linears: + # FIXME: add bias support if ever needed + if (not isinstance(s, MixedPrecisionOp) or not isinstance(s._v_weight, QuantizedTensor) + or s._full_precision_mm or s.comfy_force_forward or s.bias is not None): + return + weights.append(s._v_weight) + # Members are loaded back to back, so their VBAR allocations are consecutive in member order. + v = (self.linears[0]._v[0], min(s._v[1] for s in self.linears), sum(s._v[2] for s in self.linears)) + weight = comfy.memory_management.cat_gathered(weights, comfy_aimdo.torch.aimdo_to_tensor(v, device)) + if weight is None: + return + self._v = v + self._v_weight = weight + for s in self.linears: + s._v_weight = None + + def forward(self, x): + return self.linear_input_act(x, None) + + def linear_input_act(self, x, input_act, act_weight=None, act_eps=0.0, + residual=None, residual_scale=None): + run_every_op() + if self._v_weight is None: + return NotImplemented + if self._full_precision_mm: + x = _eager_input_act(x, input_act, act_weight, act_eps) + out = torch.nn.functional.linear(x, self._v_weight.to(dtype=x.dtype).dequantize()) + return _linear_residual(out, residual, residual_scale) + return _try_quantized_linear_input_act(x, self._v_weight, None, input_act, act_weight, act_eps, + residual, residual_scale) + + def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0, residual=None, residual_scale=None): + if isinstance(linear, CombinedLinear): + return linear.linear_input_act(x, input_act, act_weight, act_eps, residual, residual_scale) run_every_op() weight = linear.weight quantized = isinstance(weight, QuantizedTensor) diff --git a/comfy/sd.py b/comfy/sd.py index ca61c01f776..0c6168c0ed3 100644 --- a/comfy/sd.py +++ b/comfy/sd.py @@ -17,6 +17,7 @@ import comfy.ldm.wan.vae import comfy.ldm.trellis2.vae import comfy.ldm.wan.vae2_2 +import comfy.ldm.wan.model_vsr import comfy.ldm.hunyuan3d.vae import comfy.ldm.seedvr.vae import comfy.ldm.mage_flow.vae @@ -987,6 +988,18 @@ def estimate_memory(shape, dtype, num_layers = 16, kv_cache_multiplier = 2): self.downscale_index_formula = (4, 8, 8) self.memory_used_encode = lambda shape, dtype: (700 * (max(1, (shape[-3] ** 0.66 * 0.11)) * shape[-2] * shape[-1]) * model_management.dtype_size(dtype)) self.memory_used_decode = lambda shape, dtype: (50 * (max(1, (shape[-3] ** 0.65 * 0.26)) * shape[-2] * shape[-1] * 32 * 32) * model_management.dtype_size(dtype)) + elif "lr_encoder.layers.0.weight" in sd and "condition_encoder.projection.weight" in sd: # RelayVSR Flash + self.latent_channels = 48 + self.latent_dim = 3 + self.upscale_ratio = (lambda a: max(0, (a - 1) * comfy.ldm.wan.model_vsr.RELAY_KEYFRAME_INTERVAL + 1), 16, 16) + # x2 and x4 decoders have the same weight shapes, only the metadata tells them apart + config = json.loads((metadata or {}).get("config", "{}")) + if "upscale_factor" not in config: + raise ValueError("RelayVSR Flash decoder without its upscale_factor metadata, use the released relayvsr_flash_x2/x4 file.") + self.first_stage_model = comfy.ldm.wan.model_vsr.RelayFlash(config["upscale_factor"]) + self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32] + self.process_input = self.process_output = lambda image: image + self.memory_used_decode = lambda shape, dtype: (40000 * shape[-2] * shape[-1] * model_management.dtype_size(dtype)) elif "vocoder.resblocks.0.convs1.0.weight" in sd or "vocoder.vocoder.resblocks.0.convs1.0.weight" in sd: # LTX Audio sd = comfy.utils.state_dict_prefix_replace(sd, {"audio_vae.": "autoencoder."}) self.first_stage_model = comfy.ldm.lightricks.vae.audio_vae.AudioVAE(metadata=metadata) @@ -1159,6 +1172,7 @@ def _no_generic_io(*args, **kwargs): if self.disable_offload: mp = comfy.model_patcher.ModelPatcher self.patcher = mp(self.first_stage_model, load_device=self.device, offload_device=offload_device, fast_disk=fast_disk) + self.first_stage_model.current_patcher = self.patcher m, u = self.first_stage_model.load_state_dict(sd, strict=False, assign=self.patcher.is_dynamic()) if not self.patcher.is_dynamic(): @@ -1177,6 +1191,7 @@ def clone(self): n = VAE(no_init=True) n.patcher = self.patcher.clone() n.first_stage_model = self.first_stage_model + n.first_stage_model.current_patcher = n.patcher n.memory_used_encode = self.memory_used_encode n.memory_used_decode = self.memory_used_decode n.downscale_ratio = self.downscale_ratio @@ -1364,7 +1379,7 @@ def decode(self, samples_in, vae_options={}): preallocated = False if getattr(self.first_stage_model, 'comfy_has_chunked_io', False): with comfy.model_prefetch.pause_malloc_graph(): - pixel_samples = torch.empty(self.first_stage_model.decode_output_shape(samples_in.shape), device=self.output_device, dtype=self.vae_output_dtype()) + pixel_samples = torch.empty(self.first_stage_model.decode_output_shape(samples_in.shape, **vae_options), device=self.output_device, dtype=self.vae_output_dtype()) preallocated = True for x in range(0, samples_in.shape[0], batch_number): diff --git a/comfy/supported_models.py b/comfy/supported_models.py index 3d2015bbd3c..66c823e6c0f 100644 --- a/comfy/supported_models.py +++ b/comfy/supported_models.py @@ -1578,6 +1578,19 @@ def get_model(self, state_dict, prefix="", device=None): out = model_base.WAN21_SCAIL2(self, image_to_video=False, device=device) return out +class WAN22_RelayVSR(WAN21_T2V): + unet_config = { + "image_model": "wan2.1", + "model_type": "relayvsr", + } + + latent_format = latent_formats.RelayVSR + + def get_model(self, state_dict, prefix="", device=None): + out = model_base.WAN22_RelayVSR(self, device=device) + return out + + class WAN22_WanDancer(WAN21_T2V): unet_config = { "image_model": "wan2.1", @@ -2724,6 +2737,7 @@ def get_model(self, state_dict, prefix="", device=None): WAN21_FlowRVS, WAN21_SCAIL, WAN21_SCAIL2, + WAN22_RelayVSR, WAN22_WanDancer, Hunyuan3Dv2mini, Hunyuan3Dv2, diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py index d84d7b7cd11..36770190498 100644 --- a/comfy/text_encoders/llama.py +++ b/comfy/text_encoders/llama.py @@ -677,6 +677,7 @@ def __init__(self, config: Llama2Config, device=None, dtype=None, ops: Any = Non self.q_proj = ops.Linear(config.hidden_size, self.inner_size, bias=config.qkv_bias, device=device, dtype=dtype) self.k_proj = ops.Linear(config.hidden_size, self.kv_size, bias=config.qkv_bias, device=device, dtype=dtype) self.v_proj = ops.Linear(config.hidden_size, self.kv_size, bias=config.qkv_bias, device=device, dtype=dtype) + self.combined_qkv = comfy.ops.CombinedLinear((self.q_proj, self.k_proj, self.v_proj)) self.o_proj = ops.Linear(self.inner_size, config.hidden_size, bias=False, device=device, dtype=dtype) self.q_norm = None @@ -701,9 +702,13 @@ def forward( if self.merged_qkv: xq, xk, xv = self.qkv_proj(hidden_states).split((self.inner_size, self.kv_size, self.kv_size), dim=-1) else: - xq = self.q_proj(hidden_states) - xk = self.k_proj(hidden_states) - xv = self.v_proj(hidden_states) + qkv = self.combined_qkv.forward(hidden_states) + if qkv is NotImplemented: + xq = self.q_proj(hidden_states) + xk = self.k_proj(hidden_states) + xv = self.v_proj(hidden_states) + else: + xq, xk, xv = qkv.split((self.inner_size, self.kv_size, self.kv_size), dim=-1) xq = xq.view(batch_size, seq_length, self.num_heads, self.head_dim).transpose(1, 2) xk = xk.view(batch_size, seq_length, self.num_kv_heads, self.head_dim).transpose(1, 2) @@ -783,6 +788,7 @@ def __init__(self, config: Llama2Config, device=None, dtype=None, ops: Any = Non else: self.gate_proj = ops.Linear(config.hidden_size, intermediate_size, bias=False, device=device, dtype=dtype) self.up_proj = ops.Linear(config.hidden_size, intermediate_size, bias=False, device=device, dtype=dtype) + self.combined_gate_up = comfy.ops.CombinedLinear((self.gate_proj, self.up_proj)) self.down_proj = ops.Linear(intermediate_size, config.hidden_size, bias=False, device=device, dtype=dtype) if config.mlp_activation == "silu": self.activation = torch.nn.functional.silu @@ -793,14 +799,15 @@ def __init__(self, config: Llama2Config, device=None, dtype=None, ops: Any = Non def forward(self, x, norm=None): # norm: the block's pre-norm, folded into the merged projection's input quantizer where supported - if self.merged_mlp: - if norm is not None: - x = comfy.ops.linear_input_act(self.gate_up_proj, x, "rms_norm", norm.scale(), norm.eps) - else: - x = self.gate_up_proj(x) + gate_up = self.gate_up_proj if self.merged_mlp else self.combined_gate_up + if norm is not None: + combined = comfy.ops.linear_input_act(gate_up, x, "rms_norm", norm.scale(), norm.eps) + else: + combined = comfy.ops.linear_input_act(gate_up, x, None) + if combined is not NotImplemented: if self.merged_input_act is not None: - return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act) - gate, up = x.chunk(2, dim=-1) + return comfy.ops.linear_input_act(self.down_proj, combined, self.merged_input_act) + gate, up = combined.chunk(2, dim=-1) return self.down_proj(self.activation(gate) * up) if norm is not None: x = norm(x) diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py index d4f241c5f20..196f5327b69 100644 --- a/comfy/text_encoders/qwen35.py +++ b/comfy/text_encoders/qwen35.py @@ -350,6 +350,7 @@ def __init__(self, config, device=None, dtype=None, ops=None): self.in_proj_qkv = ops.Linear(hidden, conv_dim, bias=False, device=device, dtype=dtype) self.in_proj_z = ops.Linear(hidden, value_dim, bias=False, device=device, dtype=dtype) + self.combined_qkvz = comfy.ops.CombinedLinear((self.in_proj_qkv, self.in_proj_z)) self.in_proj_b = ops.Linear(hidden, self.num_value_heads, bias=False, device=device, dtype=dtype) self.in_proj_a = ops.Linear(hidden, self.num_value_heads, bias=False, device=device, dtype=dtype) self.out_proj = ops.Linear(value_dim, hidden, bias=False, device=device, dtype=dtype) @@ -366,8 +367,12 @@ def forward(self, x, past_key_value=None, **kwargs): batch_size, seq_len, _ = x.shape # Projections (shared) - proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim] - z = self.in_proj_z(x) + qkvz = self.combined_qkvz.forward(x) + if qkvz is NotImplemented: + proj = self.in_proj_qkv(x) # [B, seq_len, conv_dim] + z = self.in_proj_z(x) + else: + proj, z = qkvz.split((self.key_dim * 2 + self.value_dim, self.value_dim), dim=-1) if past_key_value is not None and past_key_value.decode_step(seq_len): # decode: conv step into the side buffer, then gates + delta rule + gated norm in one @@ -467,6 +472,7 @@ def __init__(self, config, device=None, dtype=None, ops=None): self.q_proj = ops.Linear(config.hidden_size, self.inner_size * 2, bias=config.qkv_bias, device=device, dtype=dtype) self.k_proj = ops.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.qkv_bias, device=device, dtype=dtype) self.v_proj = ops.Linear(config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.qkv_bias, device=device, dtype=dtype) + self.combined_qkv = comfy.ops.CombinedLinear((self.q_proj, self.k_proj, self.v_proj)) self.o_proj = ops.Linear(self.inner_size, config.hidden_size, bias=False, device=device, dtype=dtype) # QK norms with (1+weight) scaling @@ -491,15 +497,19 @@ def forward(self, x, attention_mask=None, freqs_cis=None, optimized_attention=No batch_size, seq_length, _ = x.shape # Project Q (with gate), K, V - qg = self.q_proj(x) + qkv = self.combined_qkv.forward(x) + if qkv is NotImplemented: + qg = self.q_proj(x) + xk = self.k_proj(x) + xv = self.v_proj(x) + else: + kv_size = self.num_kv_heads * self.head_dim + qg, xk, xv = qkv.split((self.inner_size * 2, kv_size, kv_size), dim=-1) # Split into query and gate: each is [B, seq, inner_size] qg = qg.view(batch_size, seq_length, self.num_heads, self.head_dim * 2) xq, gate = qg[..., :self.head_dim], qg[..., self.head_dim:] gate = gate.reshape(batch_size, seq_length, -1) # [B, seq, inner_size] - xk = self.k_proj(x) - xv = self.v_proj(x) - xq = self.q_norm(xq).transpose(1, 2) # [B, heads, seq, head_dim] xk = self.k_norm(xk.view(batch_size, seq_length, self.num_kv_heads, self.head_dim)).transpose(1, 2) xv = xv.view(batch_size, seq_length, self.num_kv_heads, self.head_dim).transpose(1, 2) @@ -859,6 +869,8 @@ def __init__(self, config_dict, dtype, device, operations): for m in unit.modules(): if isinstance(m, comfy.ops.MixedPrecisionOp): m.comfy_gemv_optimized = True + if m._combined_linear is not None: + m._combined_linear.comfy_gemv_optimized = True vision_overrides = QWEN35_MODELS.get(self.model_type, {}).get("vision", {}) vision_config = {**QWEN35_VISION_DEFAULTS, **vision_overrides, "out_hidden_size": config.hidden_size} self.visual = Qwen35VisionModel(vision_config, device=device, dtype=dtype, ops=operations) diff --git a/comfy_api/feature_flags.py b/comfy_api/feature_flags.py index 6bc29ba9c03..569a77c9977 100644 --- a/comfy_api/feature_flags.py +++ b/comfy_api/feature_flags.py @@ -37,12 +37,12 @@ class FeatureFlagInfo(TypedDict): }, "partner_node_token": { "type": "bool", - "default": False, + "default": True, "description": "Queue Local API-node prompts with a renewable partner-node token instead of the workspace token", }, "local_web_sso": { "type": "bool", - "default": False, + "default": True, "description": "Offer Sign in with browser (Cloud OAuth + PKCE) on Local ComfyUI opened at localhost or 127.0.0.1", }, } @@ -126,13 +126,17 @@ def _parse_cli_feature_flags() -> dict[str, Any]: # Owned by app.deploy_to_cloud, which sets it via set_server_feature. _SERVER_SET_FEATURE_FLAGS = ("deploy_to_cloud",) -# CLI-provided flags cannot overwrite core or server-set flags. Startup code may: main.py sets "assets". -_cli_flags = { - k: v for k, v in _parse_cli_feature_flags().items() - if k not in _CORE_FEATURE_FLAGS and k not in _SERVER_SET_FEATURE_FLAGS -} +def _server_feature_flags() -> dict[str, Any]: + """Registry defaults, then core flags, then CLI flags. CLI flags cannot overwrite core or server-set flags; main.py may still set "assets".""" + registry_defaults = {k: info["default"] for k, info in CLI_FEATURE_FLAG_REGISTRY.items()} + cli_flags = { + k: v for k, v in _parse_cli_feature_flags().items() + if k not in _CORE_FEATURE_FLAGS and k not in _SERVER_SET_FEATURE_FLAGS + } + return {**registry_defaults, **_CORE_FEATURE_FLAGS, **cli_flags} + -SERVER_FEATURE_FLAGS: dict[str, Any] = {**_CORE_FEATURE_FLAGS, **_cli_flags} +SERVER_FEATURE_FLAGS: dict[str, Any] = _server_feature_flags() def get_connection_feature( diff --git a/comfy_api_nodes/apis/bytedance.py b/comfy_api_nodes/apis/bytedance.py index 1d830511ba9..12e5985dea5 100644 --- a/comfy_api_nodes/apis/bytedance.py +++ b/comfy_api_nodes/apis/bytedance.py @@ -308,6 +308,7 @@ class SeedanceVirtualLibraryCreateAssetRequest(BaseModel): "480p": {"min": 409_600, "max": 8_295_044}, "720p": {"min": 409_600, "max": 8_295_044}, "1080p": {"min": 409_600, "max": 8_295_044}, + "4k": {"min": 409_600, "max": 8_295_044}, }, } diff --git a/comfy_api_nodes/apis/hunyuan_llm.py b/comfy_api_nodes/apis/hunyuan_llm.py new file mode 100644 index 00000000000..a47cf7d7c50 --- /dev/null +++ b/comfy_api_nodes/apis/hunyuan_llm.py @@ -0,0 +1,49 @@ +from typing import Literal + +from pydantic import BaseModel, Field + + +class TencentHyTextPart(BaseModel): + type: Literal["text"] = "text" + text: str = Field(...) + + +class TencentHyImageUrl(BaseModel): + url: str = Field(...) + + +class TencentHyImagePart(BaseModel): + type: Literal["image_url"] = "image_url" + image_url: TencentHyImageUrl = Field(...) + + +class TencentHyMessage(BaseModel): + role: str = Field(...) + content: list[TencentHyTextPart | TencentHyImagePart] = Field(...) + + +class TencentHyThinking(BaseModel): + type: str = Field(...) + + +class TencentHyChatRequest(BaseModel): + model: str = Field(...) + messages: list[TencentHyMessage] = Field(...) + thinking: TencentHyThinking = Field(...) + temperature: float = Field(...) + max_tokens: int = Field(...) + seed: int = Field(...) + stream: bool = Field(...) + + +class TencentHyResponseMessage(BaseModel): + content: str | None = Field(None) + + +class TencentHyChoice(BaseModel): + message: TencentHyResponseMessage | None = Field(None) + finish_reason: str | None = Field(None) + + +class TencentHyChatResponse(BaseModel): + choices: list[TencentHyChoice] | None = Field(None) diff --git a/comfy_api_nodes/apis/kling.py b/comfy_api_nodes/apis/kling.py index 748c0c4b5a7..831b9f67332 100644 --- a/comfy_api_nodes/apis/kling.py +++ b/comfy_api_nodes/apis/kling.py @@ -174,9 +174,10 @@ class Kling3TurboImage2VideoRequest(BaseModel): class Kling4Settings(BaseModel): - resolution: str = Field(..., description="'720p' or '1080p'") + resolution: str = Field(..., description="'720p' or '1080p'; Flash is 720p only") aspect_ratio: str | None = Field(None, description="'16:9', '9:16', '1:1', '21:9'; omni also 'adaptive'") - duration: int | None = Field(None, description="3-30 seconds; omitted for edits to keep the input length") + duration: int | None = Field(None, description="3-30 seconds, 3-20 on Flash; omitted for edits to keep the input length") + audio: str | None = Field(None, description="'original' keeps the input video's audio; omitted generates new audio") class Kling4Content(BaseModel): diff --git a/comfy_api_nodes/apis/openai.py b/comfy_api_nodes/apis/openai.py index c73d87b0f86..4d3e4d7c6db 100644 --- a/comfy_api_nodes/apis/openai.py +++ b/comfy_api_nodes/apis/openai.py @@ -26,6 +26,7 @@ class OpenAIImageGenerationResponse(BaseModel): class OpenAIImageEditRequest(BaseModel): background: str | None = Field(None, description="Background transparency") + complexity: str | None = Field(None) model: str = Field(...) moderation: str | None = Field(None) n: int | None = Field(None, description="The number of images to generate") @@ -38,6 +39,7 @@ class OpenAIImageEditRequest(BaseModel): class OpenAIImageGenerationRequest(BaseModel): background: str | None = Field(None, description="Background transparency") + complexity: str | None = Field(None) model: str | None = Field(None) moderation: str | None = Field(None) n: int | None = Field( diff --git a/comfy_api_nodes/apis/pika.py b/comfy_api_nodes/apis/pika.py new file mode 100644 index 00000000000..446ba610e43 --- /dev/null +++ b/comfy_api_nodes/apis/pika.py @@ -0,0 +1,61 @@ +from pydantic import BaseModel, Field + + +class PikaH3FlexRequest(BaseModel): + prompt: str = Field(...) + duration: int = Field(...) + aspect_ratio: str = Field(...) + resolution: str = Field(...) + seed: int = Field(...) + rewrite: bool = Field(...) + + +class PikaH3FlexImageRequest(PikaH3FlexRequest): + image: str | None = Field(None) + last_frame_image: str | None = Field(None) + + +class PikaH3FlexReference(BaseModel): + type: str = Field(...) + url: str = Field(...) + + +class PikaH3FlexReferenceRequest(PikaH3FlexRequest): + references: list[PikaH3FlexReference] = Field(...) + + +class PikaSoundtrackRequest(BaseModel): + video: str = Field(...) + instruction: str | None = Field(None) + seed: int = Field(...) + + +class PikaSFXRequest(BaseModel): + prompt: str = Field(...) + duration: float = Field(...) + seed: int = Field(...) + + +class PikaMusicRequest(BaseModel): + mode: str = Field(...) + prompt: str = Field(...) + lyrics: str | None = Field(None) + source_audio: str | None = Field(None) + rewrite_prompt: bool = Field(...) + duration: float | None = Field(None) + seed: int = Field(...) + + +class PikaMediaFile(BaseModel): + url: str = Field(...) + + +class PikaJobOutput(BaseModel): + video: PikaMediaFile | None = Field(None) + audio: PikaMediaFile | None = Field(None) + + +class PikaJob(BaseModel): + id: str = Field(...) + status: str = Field(...) + output: PikaJobOutput | None = Field(None) diff --git a/comfy_api_nodes/nodes_bytedance.py b/comfy_api_nodes/nodes_bytedance.py index b29a93c4c38..48dd56cb64f 100644 --- a/comfy_api_nodes/nodes_bytedance.py +++ b/comfy_api_nodes/nodes_bytedance.py @@ -122,8 +122,8 @@ } SEEDANCE_MODEL_TOOLTIP = ( - "Seedance 2.5 for the newest model, videos up to 30 seconds and mp4/mov output; " - "Seedance 2.5 Draft for a quick 480p preview whose draft_task_id renders the 1080p final " + "Seedance 2.5 for the newest model, videos up to 30 seconds, 4k for enterprise accounts and mp4/mov output; " + "Seedance 2.5 Draft for a quick 480p preview whose draft_task_id renders the 1080p or 4k final " "in the Seedance 2.5 Draft to Final Video node; " "Seedance 2.0 for maximum quality and 4k; Fast for speed optimization; " "Mini for the fastest, lowest-cost generation." @@ -2136,9 +2136,11 @@ def _seedance25_text_inputs( ), IO.Combo.Input( "resolution", - options=["480p"] if draft else ["480p", "720p", "1080p"], + options=["480p"] if draft else ["480p", "720p", "1080p", "4k"], default="480p" if draft else "720p", - tooltip="Resolution of the output video.", + tooltip="Resolution of the output video." + if draft + else "Resolution of the output video. 4k is available to enterprise accounts only.", ), *( [ @@ -2319,14 +2321,15 @@ def _seedance2_validate_draft_output(cls: type[IO.ComfyNode], model: dict) -> No $contains($m, "2.5") ? ( $is480 := $res = "480p"; $is1080 := $res = "1080p"; - $perFrame := $ratio = "1:1" ? ($is480 ? 400 : $is1080 ? 2025 : 900) : - $ratio = "4:3" ? ($is480 ? 411.25 : $is1080 ? 2028 : 905.6719) : - $ratio = "3:4" ? ($is480 ? 411.25 : $is1080 ? 2028 : 905.6719) : - $ratio = "21:9" ? ($is480 ? 418.5 : $is1080 ? 2037.9648 : 904.3945) : - ($is480 ? 400.3125 : $is1080 ? 2025 : 900); - $price := $is1080 - ? ($hasVideo ? 0.01001 : 0.016731) - : ($hasVideo ? 0.009152 : 0.015301); + $is4k := $res = "4k"; + $perFrame := $ratio = "1:1" ? ($is480 ? 400 : $is1080 ? 2025 : $is4k ? 8100 : 900) : + $ratio = "4:3" ? ($is480 ? 411.25 : $is1080 ? 2028 : $is4k ? 8100.6289 : 905.6719) : + $ratio = "3:4" ? ($is480 ? 411.25 : $is1080 ? 2028 : $is4k ? 8100.6289 : 905.6719) : + $ratio = "21:9" ? ($is480 ? 418.5 : $is1080 ? 2037.9648 : $is4k ? 8100.2227 : 904.3945) : + ($is480 ? 400.3125 : $is1080 ? 2025 : $is4k ? 8100 : 900); + $price := $is4k ? ($hasVideo ? 0.004147 : 0.006721) : + $is1080 ? ($hasVideo ? 0.01001 : 0.016731) : + ($hasVideo ? 0.009152 : 0.015301); $costFor := function($d) { $floor($perFrame * (24 * $d + 1)) / 1000 * $price }; $lo := $costFor($auto ? 4 : $dur); $hi := $costFor(($auto ? 30 : $dur) + ($hasVideo ? 30 : 0)); @@ -2361,6 +2364,31 @@ def _seedance2_validate_draft_output(cls: type[IO.ComfyNode], model: dict) -> No _SEEDANCE_AUDIO_POLICY_CODE = "OutputAudioSensitiveContentDetected.PolicyViolation" _SEEDANCE_TASK_TYPE_CONSTRAINT_CODE = "InvalidParameter.TaskTypeConstraint" _SEEDANCE_TASK_TYPE_MISMATCH_CODE = "InvalidParameter.TaskTypeMismatch" +_SEEDANCE_MODEL_UNAVAILABLE_CODE = "PARTNER_MODEL_UNAVAILABLE" + + +async def _seedance2_create_task( + cls: type[IO.ComfyNode], + request: Seedance2TaskCreationRequest, +) -> TaskCreationResponse: + try: + return await sync_op( + cls, + ApiEndpoint(path=BYTEPLUS_TASK_ENDPOINT, method="POST"), + data=request, + response_model=TaskCreationResponse, + ) + except Exception as exc: + if ( + request.model == SEEDANCE_MODELS["Seedance 2.5"] + and request.resolution == "4k" + and _SEEDANCE_MODEL_UNAVAILABLE_CODE in str(exc) + ): + raise ValueError( + "Seedance 2.5 at 4k is available to enterprise accounts only. " + "Contact us at https://comfy.org/contact to get access." + ) from exc + raise async def _seedance2_poll_video_task( @@ -2484,7 +2512,7 @@ def define_schema(cls): IO.String.Output( "draft_task_id", tooltip="Task ID of a Seedance 2.5 Draft run. Connect it to the Seedance 2.5 Draft to " - "Final Video node to render the 1080p final.", + "Final Video node to render the 1080p or 4k final.", ), ], hidden=[ @@ -2507,10 +2535,9 @@ async def execute( validate_string(model["prompt"], strip_whitespace=True, min_length=1) _seedance2_validate_draft_output(cls, model) model_id = SEEDANCE_MODELS[model["model"]] - initial_response = await sync_op( + initial_response = await _seedance2_create_task( cls, - ApiEndpoint(path=BYTEPLUS_TASK_ENDPOINT, method="POST"), - data=_seedance2_build_request( + _seedance2_build_request( model, model_id, [TaskTextContent(text=model["prompt"])], @@ -2518,7 +2545,6 @@ async def execute( watermark, ratio=model["ratio"], ), - response_model=TaskCreationResponse, ) response = await _seedance2_poll_video_task(cls, initial_response.id) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url), initial_response.id) @@ -2604,7 +2630,7 @@ def define_schema(cls): IO.String.Output( "draft_task_id", tooltip="Task ID of a Seedance 2.5 Draft run. Connect it to the Seedance 2.5 Draft to " - "Final Video node to render the 1080p final.", + "Final Video node to render the 1080p or 4k final.", ), ], hidden=[ @@ -2715,11 +2741,9 @@ async def execute( ), ) - initial_response = await sync_op( + initial_response = await _seedance2_create_task( cls, - ApiEndpoint(path=BYTEPLUS_TASK_ENDPOINT, method="POST"), - data=_seedance2_build_request(model, model_id, content, seed, watermark, ratio=request_ratio), - response_model=TaskCreationResponse, + _seedance2_build_request(model, model_id, content, seed, watermark, ratio=request_ratio), ) response = await _seedance2_poll_video_task(cls, initial_response.id) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url), initial_response.id) @@ -2856,7 +2880,7 @@ def define_schema(cls): IO.String.Output( "draft_task_id", tooltip="Task ID of a Seedance 2.5 Draft run. Connect it to the Seedance 2.5 Draft to " - "Final Video node to render the 1080p final.", + "Final Video node to render the 1080p or 4k final.", ), ], hidden=[ @@ -3039,11 +3063,9 @@ async def execute( content.append( TaskAudioContent(audio_url=TaskAudioContentUrl(url=url)), ) - initial_response = await sync_op( + initial_response = await _seedance2_create_task( cls, - ApiEndpoint(path=BYTEPLUS_TASK_ENDPOINT, method="POST"), - data=_seedance2_build_request(model, model_id, content, seed, watermark, ratio=model["ratio"]), - response_model=TaskCreationResponse, + _seedance2_build_request(model, model_id, content, seed, watermark, ratio=model["ratio"]), ) response = await _seedance2_poll_video_task( cls, @@ -3123,7 +3145,7 @@ def define_schema(cls): node_id="ByteDance2DraftToFinalVideoNode", display_name="ByteDance Seedance 2.5 Draft to Final Video", category="partner/video/ByteDance", - description="Render the 1080p final video of a Seedance 2.5 Draft. The final keeps the draft's " + description="Render the 1080p or 4k final video of a Seedance 2.5 Draft. The final keeps the draft's " "scene and motion, and reuses its prompt, references, duration, aspect ratio, and audio setting.", inputs=[ IO.String.Input( @@ -3140,6 +3162,13 @@ def define_schema(cls): tooltip="Whether to add a watermark to the video.", advanced=True, ), + IO.Combo.Input( + "resolution", + options=["1080p", "4k"], + default="1080p", + tooltip="Resolution of the final video. 4k is available to enterprise accounts only.", + optional=True, + ), ], outputs=[ IO.Video.Output(), @@ -3151,23 +3180,26 @@ def define_schema(cls): ], is_api_node=True, price_badge=IO.PriceBadge( - expr='{"type":"range_usd","min_usd":3.2864,"max_usd":29.3964,"format":{"approximate":true}}', + depends_on=IO.PriceBadgeDepends(widgets=["resolution"]), + expr=""" + widgets.resolution = "4k" + ? {"type":"range_usd","min_usd":5.2807,"max_usd":48.408,"format":{"approximate":true}} + : {"type":"range_usd","min_usd":3.2864,"max_usd":29.3964,"format":{"approximate":true}} + """, ), ) @classmethod - async def execute(cls, draft_task_id: str, watermark: bool) -> IO.NodeOutput: + async def execute(cls, draft_task_id: str, watermark: bool, resolution: str = "1080p") -> IO.NodeOutput: validate_string(draft_task_id, strip_whitespace=True, field_name="draft_task_id", min_length=1) - initial_response = await sync_op( + initial_response = await _seedance2_create_task( cls, - ApiEndpoint(path=BYTEPLUS_TASK_ENDPOINT, method="POST"), - data=Seedance2TaskCreationRequest( + Seedance2TaskCreationRequest( model=SEEDANCE_MODELS["Seedance 2.5"], content=[TaskDraftTaskContent(draft_task=TaskDraftTaskContentTask(id=draft_task_id.strip()))], - resolution="1080p", + resolution=resolution, watermark=watermark, ), - response_model=TaskCreationResponse, ) response = await _seedance2_poll_video_task(cls, initial_response.id) return IO.NodeOutput(await download_url_to_video_output(response.content.video_url)) diff --git a/comfy_api_nodes/nodes_gemini.py b/comfy_api_nodes/nodes_gemini.py index 1862fb35ccd..e3850af4276 100644 --- a/comfy_api_nodes/nodes_gemini.py +++ b/comfy_api_nodes/nodes_gemini.py @@ -1819,6 +1819,7 @@ async def execute( OMNI_MODELS: dict[str, str] = { "Omni Flash": "gemini-omni-flash-preview", "Omni Flash 1.1": "gemini-omni-1.1-flash", + "Omni Flash 1.2": "gemini-omni-1.2-flash", } @@ -2106,6 +2107,56 @@ def _omni_v2_flash_1_1_inputs() -> list[Input]: ] +def _omni_v2_flash_1_2_inputs() -> list[Input]: + return [ + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Describe the video to generate, or the edit to apply to an attached video. Specify the " + 'length directly in the prompt, e.g. "a 6-second clip" or, for the \'extend\' task, "extend by 5 seconds"; ' + "the generated length may be 3-15 seconds, and 'extend' adds at most 10 seconds.", + ), + IO.Combo.Input( + "resolution", + options=["360p", "720p", "1080p", "4k"], + default="720p", + tooltip="Output resolution.", + ), + IO.Combo.Input( + "aspect_ratio", + options=["16:9", "9:16"], + default="16:9", + tooltip="Output aspect ratio: 16:9 (landscape) or 9:16 (portrait). " + "The 'edit' and 'extend' tasks keep the aspect ratio of the input video instead.", + ), + _omni_task_input(with_extend=True), + IO.Autogrow.Input( + "images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("image"), + names=[f"image_{i}" for i in range(1, OMNI_MAX_IMAGES + 1)], + min=0, + ), + tooltip=f"Optional reference image(s) to guide or animate the video. Up to {OMNI_MAX_IMAGES} images; " + "with the 'image_to_video' task the first one is the starting frame and an optional second one " + "is the ending frame.", + ), + IO.Autogrow.Input( + "videos", + template=IO.Autogrow.TemplateNames( + IO.Video.Input("video"), + names=[f"video_{i}" for i in range(1, OMNI_MAX_VIDEOS + 1)], + min=0, + ), + tooltip=f"Optional reference video(s) to guide or edit. Up to {OMNI_MAX_VIDEOS} videos, " + "each up to 15 seconds long. The 'extend' task accepts up to 25 seconds, " + "and its result can be at most 30 seconds.", + ), + _omni_seed_input(), + ] + + class GeminiVideoOmniV2(IO.ComfyNode): @@ -2118,11 +2169,12 @@ def define_schema(cls): essentials_category="Video Generation", description="Generate a video with audio from a text prompt using Google's Gemini Omni Flash models. " "Optionally provide reference images and/or videos to guide or edit the result. Describe the desired " - "length (3-10s) directly in the prompt.", + "length directly in the prompt (3-15s for Omni Flash 1.2, 3-10s for older models).", inputs=[ IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option("Omni Flash 1.2", _omni_v2_flash_1_2_inputs()), IO.DynamicCombo.Option("Omni Flash 1.1", _omni_v2_flash_1_1_inputs()), IO.DynamicCombo.Option("Omni Flash", _omni_v2_flash_inputs()), ], @@ -2143,9 +2195,11 @@ def define_schema(cls): depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution"]), expr=""" ( - $prices := {"360p": 0.0483, "720p": 0.1449, "1080p": 0.2174, "4k": 0.4349}; + $prices := widgets.model = "omni flash 1.2" + ? {"360p": 0.0331, "720p": 0.0994, "1080p": 0.1491, "4k": 0.2982} + : {"360p": 0.0483, "720p": 0.1449, "1080p": 0.2174, "4k": 0.4349}; $r := $lookup(widgets, "model.resolution"); - {"type":"usd","usd": $r ? $lookup($prices, $r) : 0.1449, + {"type":"usd","usd": $lookup($prices, $r ? $r : "720p"), "format":{"suffix":"/second","approximate":true}} ) """, @@ -2166,8 +2220,11 @@ async def execute(cls, model: dict) -> IO.NodeOutput: raise ValueError(f"The current maximum number of supported images is {OMNI_MAX_IMAGES}.") if len(videos) > OMNI_MAX_VIDEOS: raise ValueError(f"The current maximum number of supported videos is {OMNI_MAX_VIDEOS}.") + max_video_duration = 10.1 + if model["model"] == "Omni Flash 1.2": + max_video_duration = 25.1 if task == "extend" else 15.1 for video in videos: - validate_video_duration(video, max_duration=10.1) + validate_video_duration(video, max_duration=max_video_duration) if task == "text_to_video" and (images or videos): raise ValueError("The 'text_to_video' task generates from the prompt alone; detach the reference media.") if task == "image_to_video" and videos: diff --git a/comfy_api_nodes/nodes_hunyuan_llm.py b/comfy_api_nodes/nodes_hunyuan_llm.py new file mode 100644 index 00000000000..8cb9b2d40d5 --- /dev/null +++ b/comfy_api_nodes/nodes_hunyuan_llm.py @@ -0,0 +1,181 @@ +from typing_extensions import override + +from comfy_api.latest import IO, ComfyExtension +from comfy_api_nodes.apis.hunyuan_llm import ( + TencentHyChatRequest, + TencentHyChatResponse, + TencentHyImagePart, + TencentHyImageUrl, + TencentHyMessage, + TencentHyTextPart, + TencentHyThinking, +) +from comfy_api_nodes.util import ( + ApiEndpoint, + get_number_of_images, + sync_op, + upload_images_to_comfyapi, + validate_string, +) + +CHAT_COMPLETIONS_PATH = "/proxy/tencent/v1/chat/completions" +HY_MODELS = {"Hy4": "hy4"} +HY_MAX_IMAGES = 20 + + +def _hy_model_option(label: str) -> IO.DynamicCombo.Option: + return IO.DynamicCombo.Option( + label, + [ + IO.String.Input("prompt", multiline=True, default=""), + IO.Autogrow.Input( + "images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("image"), + names=[f"image_{i}" for i in range(1, HY_MAX_IMAGES + 1)], + min=0, + ), + optional=True, + tooltip=f"Up to {HY_MAX_IMAGES} images. Each image in a batch counts.", + ), + IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="Seed controls whether the node should re-run; " + "results are non-deterministic regardless of seed.", + ), + IO.String.Input( + "system_prompt", + multiline=True, + default="", + optional=True, + advanced=True, + tooltip="Foundational instructions that dictate the model's behavior.", + ), + IO.Boolean.Input( + "reasoning", + default=False, + advanced=True, + tooltip="The model thinks before answering. Answers take longer and use more tokens, " + "and very hard tasks can time out.", + ), + IO.Float.Input( + "temperature", + default=1.0, + min=0.0, + max=1.4, + step=0.01, + advanced=True, + tooltip="Controls randomness. Results still vary at 0.", + ), + IO.Int.Input( + "max_tokens", + default=32000, + min=1, + max=64000, + advanced=True, + tooltip="Maximum number of tokens to generate, including reasoning tokens when reasoning is on. " + "Very long answers can time out.", + ), + ], + ) + + +def _get_text_from_response(response: TencentHyChatResponse) -> str: + choice = response.choices[0] if response.choices else None + if choice is None: + raise ValueError("Empty response from Hy4.") + if choice.finish_reason == "content_filter": + raise ValueError("Tencent blocked this request with its content filter.") + text = choice.message.content if choice.message else None + if text: + return text + if choice.finish_reason == "length": + raise ValueError("Hy4 used all max_tokens before answering. Raise max_tokens or turn reasoning off.") + raise ValueError("Empty response from Hy4.") + + +class TencentHyNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="TencentHyNode", + display_name="Tencent HY", + category="partner/text/Tencent", + essentials_category="Text Generation", + search_aliases=["HY4"], + description="Generate text responses with Tencent's Hy4 model. " + "Provide a text prompt and optionally one or more images for context.", + inputs=[ + IO.DynamicCombo.Input("model", options=[_hy_model_option(label) for label in HY_MODELS]), + ], + outputs=[IO.String.Output()], + hidden=[ + IO.Hidden.auth_token_comfy_org, + IO.Hidden.api_key_comfy_org, + IO.Hidden.unique_id, + ], + is_api_node=True, + price_badge=IO.PriceBadge( + expr=""" + { + "type": "list_usd", + "usd": [0.00119262, 0.00357643], + "format": { "approximate": true, "separator": "-", "suffix": " per 1K tokens" } + } + """, + ), + ) + + @classmethod + async def execute(cls, model: dict) -> IO.NodeOutput: + validate_string(model["prompt"], min_length=1) + images = [image for image in model["images"].values() if image is not None] + if sum(get_number_of_images(image) for image in images) > HY_MAX_IMAGES: + raise ValueError(f"Up to {HY_MAX_IMAGES} images are supported per request.") + + messages = [] + if model.get("system_prompt"): + messages.append(TencentHyMessage(role="system", content=[TencentHyTextPart(text=model["system_prompt"])])) + content = [] + if images: + urls = await upload_images_to_comfyapi( + cls, + images, + max_images=HY_MAX_IMAGES, + mime_type="image/webp", + wait_label="Uploading images", + ) + content.extend(TencentHyImagePart(image_url=TencentHyImageUrl(url=url)) for url in urls) + content.append(TencentHyTextPart(text=model["prompt"])) + messages.append(TencentHyMessage(role="user", content=content)) + + response = await sync_op( + cls, + ApiEndpoint(path=CHAT_COMPLETIONS_PATH, method="POST"), + response_model=TencentHyChatResponse, + data=TencentHyChatRequest( + model=HY_MODELS[model["model"]], + messages=messages, + thinking=TencentHyThinking(type="enabled" if model["reasoning"] else "disabled"), + temperature=model["temperature"], + max_tokens=model["max_tokens"], + seed=model["seed"], + stream=False, + ), + max_retries=1, + ) + return IO.NodeOutput(_get_text_from_response(response)) + + +class TencentHyExtension(ComfyExtension): + @override + async def get_node_list(self) -> list[type[IO.ComfyNode]]: + return [TencentHyNode] + + +async def comfy_entrypoint() -> TencentHyExtension: + return TencentHyExtension() diff --git a/comfy_api_nodes/nodes_kling.py b/comfy_api_nodes/nodes_kling.py index cd05affdf10..58c167458cb 100644 --- a/comfy_api_nodes/nodes_kling.py +++ b/comfy_api_nodes/nodes_kling.py @@ -2757,12 +2757,15 @@ async def execute( return IO.NodeOutput(await download_url_to_image_tensor(final_response.data[0].outputs[0].url)) -def _kling4_resolution() -> IO.Combo.Input: - return IO.Combo.Input("resolution", options=["720p", "1080p"]) +KLING4_VIDEO_SECONDS_LIMIT = {"kling-4.0": 30, "kling-4.0-flash": 20.4} -def _kling4_duration() -> IO.Int.Input: - return IO.Int.Input("duration", default=5, min=3, max=30, display_mode=IO.NumberDisplay.slider) +def _kling4_resolution(options: list[str]) -> IO.Combo.Input: + return IO.Combo.Input("resolution", options=options) + + +def _kling4_duration(max_duration: int) -> IO.Int.Input: + return IO.Int.Input("duration", default=5, min=3, max=max_duration, display_mode=IO.NumberDisplay.slider) def _kling4_seed() -> IO.Int.Input: @@ -2792,7 +2795,7 @@ def _kling4_reference_images() -> IO.Autogrow.Input: ) -def _kling4_reference_videos(names: list[str]) -> IO.Autogrow.Input: +def _kling4_reference_videos(names: list[str], max_seconds: int) -> IO.Autogrow.Input: return IO.Autogrow.Input( "reference_videos", template=IO.Autogrow.TemplateNames( @@ -2800,16 +2803,18 @@ def _kling4_reference_videos(names: list[str]) -> IO.Autogrow.Input: names=names, min=0, ), - tooltip="Each video and all videos together must be 3-30 seconds long.", + tooltip=f"Each video and all videos together must be 3-{max_seconds} seconds long.", ) def _kling4_price_badge() -> IO.PriceBadge: return IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["model.resolution", "model.duration"]), + depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution", "model.duration"]), expr=""" ( - $rate := $lookup(widgets, "model.resolution") = "1080p" ? 0.33605 : 0.22451; + $rate := $contains(widgets.model, "flash") + ? 0.13442 + : ($lookup(widgets, "model.resolution") = "1080p" ? 0.33605 : 0.22451); {"type":"usd","usd": $rate * $lookup(widgets, "model.duration")} ) """, @@ -2836,9 +2841,9 @@ def _validate_kling4_images(images: list[torch.Tensor]) -> None: validate_image_aspect_ratio(image, (2, 5), (5, 2), strict=False) -def _validate_kling4_videos(videos: list[Input.Video]) -> None: +def _validate_kling4_videos(videos: list[Input.Video], max_seconds: float) -> None: for video in videos: - validate_video_duration(video, min_duration=3, max_duration=30) + validate_video_duration(video, min_duration=3, max_duration=max_seconds) validate_video_dimensions(video, min_width=700, min_height=700, max_width=4553, max_height=4553) width, height = video.get_dimensions() if width * height > 8294400: @@ -2848,8 +2853,8 @@ def _validate_kling4_videos(videos: list[Input.Video]) -> None: if video.get_frame_rate() < 23: raise ValueError(f"Video frame rate must be at least 24 fps, got {float(video.get_frame_rate()):.2f} fps.") total = sum(video.get_duration() for video in videos) - if total > 30.05: - raise ValueError(f"All videos together must be at most 30 seconds long, got {total:.2f}s.") + if total > max_seconds + 0.05: + raise ValueError(f"All videos together must be at most {max_seconds} seconds long, got {total:.2f}s.") async def _kling4_media_contents( @@ -2865,6 +2870,71 @@ async def _kling4_media_contents( return contents +def _kling4_text_to_video_inputs(resolutions: list[str], max_duration: int) -> list[IO.Input]: + return [ + IO.String.Input("prompt", multiline=True, default=""), + _kling4_resolution(resolutions), + IO.Combo.Input("aspect_ratio", options=["16:9", "9:16", "1:1", "21:9"]), + _kling4_duration(max_duration), + _kling4_seed(), + ] + + +def _kling4_reference_to_video_inputs(resolutions: list[str], max_duration: int, frames: list[str]) -> list[IO.Input]: + return [ + _kling4_reference_images(), + _kling4_reference_videos([f"video_{i}" for i in range(1, 6)], max_duration), + *[ + IO.Image.Input( + frame, + optional=True, + tooltip="Counts toward the 10-image limit together with the reference images.", + ) + for frame in frames + ], + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Refer to the inputs as @Image1, @Image2, ... and @Video1, @Video2, ...", + ), + _kling4_resolution(resolutions), + _kling4_omni_aspect_ratio( + "adaptive follows the prompt, then the first frame or the reference videos, otherwise 16:9." + ), + _kling4_duration(max_duration), + _kling4_seed(), + ] + + +def _kling4_edit_inputs(resolutions: list[str], max_seconds: int) -> list[IO.Input]: + return [ + IO.Video.Input( + "video_1", + tooltip=f"The video to edit, 3-{max_seconds} seconds. The output keeps its length.", + ), + _kling4_reference_videos([f"video_{i}" for i in range(2, 6)], max_seconds), + _kling4_reference_images(), + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Describe what to change and what to keep. Refer to the input video as " + "@Video1, the reference videos as @Video2, ... and the images as @Image1, ...", + ), + _kling4_resolution(resolutions), + _kling4_omni_aspect_ratio("adaptive keeps the aspect ratio of the input video."), + _kling4_seed(), + IO.Combo.Input( + "audio", + options=["native", "original"], + default="native", + optional=True, + tooltip="native generates new audio. original keeps the audio of the input video.", + ), + ] + + class Kling4TextToVideoNode(IO.ComfyNode): @classmethod @@ -2874,20 +2944,13 @@ def define_schema(cls) -> IO.Schema: display_name="Kling 4.0 Text to Video", category="partner/video/Kling", description="Generate a video with audio from a text prompt.", + search_aliases=["kling 4.0 flash", "kling flash"], inputs=[ IO.DynamicCombo.Input( "model", options=[ - IO.DynamicCombo.Option( - "kling-4.0", - [ - IO.String.Input("prompt", multiline=True, default=""), - _kling4_resolution(), - IO.Combo.Input("aspect_ratio", options=["16:9", "9:16", "1:1", "21:9"]), - _kling4_duration(), - _kling4_seed(), - ], - ), + IO.DynamicCombo.Option("kling-4.0", _kling4_text_to_video_inputs(["720p", "1080p"], 30)), + IO.DynamicCombo.Option("kling-4.0-flash", _kling4_text_to_video_inputs(["720p"], 20)), ], ), ], @@ -2926,8 +2989,10 @@ def define_schema(cls) -> IO.Schema: node_id="Kling4ImageToVideoNode", display_name="Kling 4.0 Image to Video", category="partner/video/Kling", - description="Generate a video with audio through up to 10 keyframes. " + description="Generate a video with audio through up to 10 keyframes on kling-4.0, " + "or from a first frame on kling-4.0-flash. " "Middle frames are spaced evenly; the video takes the aspect ratio of the first keyframe.", + search_aliases=["kling 4.0 flash", "kling flash"], inputs=[ IO.DynamicCombo.Input( "model", @@ -2953,8 +3018,23 @@ def define_schema(cls) -> IO.Schema: tooltip="Optional. Refer to the frames as @Image1, @Image2, ... " "in order: first frame, middle frames, last frame.", ), - _kling4_resolution(), - _kling4_duration(), + _kling4_resolution(["720p", "1080p"]), + _kling4_duration(30), + _kling4_seed(), + ], + ), + IO.DynamicCombo.Option( + "kling-4.0-flash", + [ + IO.Image.Input("first_frame"), + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Optional. Refer to the first frame as @Image1.", + ), + _kling4_resolution(["720p"]), + _kling4_duration(20), _kling4_seed(), ], ), @@ -2976,11 +3056,14 @@ def define_schema(cls) -> IO.Schema: @classmethod async def execute(cls, model: dict) -> IO.NodeOutput: frames = [ - ("middle_frame", image) for batch in model["middle_frames"].values() if batch is not None for image in batch + ("middle_frame", image) + for batch in model.get("middle_frames", {}).values() + if batch is not None + for image in batch ] if model["first_frame"] is not None: frames.insert(0, ("first_frame", model["first_frame"][0])) - if model["last_frame"] is not None: + if model.get("last_frame") is not None: frames.append(("last_frame", model["last_frame"][0])) if not frames: raise ValueError("Connect at least one frame: first_frame, a middle frame or last_frame.") @@ -3016,39 +3099,18 @@ def define_schema(cls) -> IO.Schema: category="partner/video/Kling", description="Generate a video with audio from reference images and videos, " "optionally starting or ending on given frames.", + search_aliases=["kling 4.0 flash", "kling flash"], inputs=[ IO.DynamicCombo.Input( "model", options=[ IO.DynamicCombo.Option( "kling-4.0", - [ - _kling4_reference_images(), - _kling4_reference_videos([f"video_{i}" for i in range(1, 6)]), - IO.Image.Input( - "first_frame", - optional=True, - tooltip="Counts toward the 10-image limit together with the reference images.", - ), - IO.Image.Input( - "last_frame", - optional=True, - tooltip="Counts toward the 10-image limit together with the reference images.", - ), - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="Refer to the inputs as @Image1, @Image2, ... and @Video1, @Video2, ...", - ), - _kling4_resolution(), - _kling4_omni_aspect_ratio( - "adaptive follows the prompt, then the first frame or the reference videos, " - "otherwise 16:9." - ), - _kling4_duration(), - _kling4_seed(), - ], + _kling4_reference_to_video_inputs(["720p", "1080p"], 30, ["first_frame", "last_frame"]), + ), + IO.DynamicCombo.Option( + "kling-4.0-flash", + _kling4_reference_to_video_inputs(["720p"], 20, ["first_frame"]), ), ], ), @@ -3064,19 +3126,21 @@ def define_schema(cls) -> IO.Schema: is_api_node=True, price_badge=IO.PriceBadge( depends_on=IO.PriceBadgeDepends( - widgets=["model.resolution", "model.duration"], + widgets=["model", "model.resolution", "model.duration"], input_groups=["model.reference_videos"], ), expr=""" ( + $flash := $contains(widgets.model, "flash"); $hd := $lookup(widgets, "model.resolution") = "1080p"; $duration := $lookup(widgets, "model.duration"); $exists(inputGroups) and $lookup(inputGroups, "model.reference_videos") > 0 ? ( - $rate := $hd ? 0.20163 : 0.13442; - {"type":"range_usd","min_usd": $rate * 2 * $duration,"max_usd": $rate * (30 + $duration)} + $rate := $flash ? 0.09009 : ($hd ? 0.20163 : 0.13442); + $max_input := $flash ? 20 : 30; + {"type":"range_usd","min_usd": $rate * 2 * $duration,"max_usd": $rate * ($max_input + $duration)} ) - : {"type":"usd","usd": ($hd ? 0.33605 : 0.22451) * $duration} + : {"type":"usd","usd": ($flash ? 0.13442 : ($hd ? 0.33605 : 0.22451)) * $duration} ) """, ), @@ -3086,14 +3150,14 @@ def define_schema(cls) -> IO.Schema: async def execute(cls, model: dict) -> IO.NodeOutput: images = [image for batch in model["reference_images"].values() if batch is not None for image in batch] videos = [video for video in model["reference_videos"].values() if video is not None] - keyframes = [(name, model[name][0]) for name in ("first_frame", "last_frame") if model[name] is not None] + keyframes = [(name, model[name][0]) for name in ("first_frame", "last_frame") if model.get(name) is not None] _validate_kling4_images(images + [image for _, image in keyframes]) - _validate_kling4_videos(videos) + _validate_kling4_videos(videos, KLING4_VIDEO_SECONDS_LIMIT[model["model"]]) prompt = _kling4_prompt(model["prompt"], len(images), len(videos)) validate_string(prompt, min_length=1, max_length=26000) if model["first_frame"] is not None: prompt = f"The video starts exactly on @first_frame. {prompt}" - if model["last_frame"] is not None: + if model.get("last_frame") is not None: prompt = f"{prompt} The video ends exactly on @last_frame." contents = [Kling4Content(type="prompt", text=prompt)] contents.extend(await _kling4_media_contents(cls, images, [("feature_video", video) for video in videos])) @@ -3122,31 +3186,13 @@ def define_schema(cls) -> IO.Schema: category="partner/video/Kling", description="Edit a video with a text prompt and optional reference images and videos. " "The result has the same length as the input video.", + search_aliases=["kling 4.0 flash", "kling flash"], inputs=[ IO.DynamicCombo.Input( "model", options=[ - IO.DynamicCombo.Option( - "kling-4.0", - [ - IO.Video.Input( - "video_1", - tooltip="The video to edit, 3-30 seconds. The output keeps its length.", - ), - _kling4_reference_videos([f"video_{i}" for i in range(2, 6)]), - _kling4_reference_images(), - IO.String.Input( - "prompt", - multiline=True, - default="", - tooltip="Describe what to change and what to keep. Refer to the input video as " - "@Video1, the reference videos as @Video2, ... and the images as @Image1, ...", - ), - _kling4_resolution(), - _kling4_omni_aspect_ratio("adaptive keeps the aspect ratio of the input video."), - _kling4_seed(), - ], - ), + IO.DynamicCombo.Option("kling-4.0", _kling4_edit_inputs(["720p", "1080p"], 30)), + IO.DynamicCombo.Option("kling-4.0-flash", _kling4_edit_inputs(["720p"], 20)), ], ), ], @@ -3160,10 +3206,15 @@ def define_schema(cls) -> IO.Schema: ], is_api_node=True, price_badge=IO.PriceBadge( - depends_on=IO.PriceBadgeDepends(widgets=["model.resolution"], input_groups=["model.reference_videos"]), + depends_on=IO.PriceBadgeDepends( + widgets=["model", "model.resolution"], + input_groups=["model.reference_videos"], + ), expr=""" ( - $rate := $lookup(widgets, "model.resolution") = "1080p" ? 0.20163 : 0.13442; + $rate := $contains(widgets.model, "flash") + ? 0.09009 + : ($lookup(widgets, "model.resolution") = "1080p" ? 0.20163 : 0.13442); $exists(inputGroups) and $lookup(inputGroups, "model.reference_videos") > 0 ? {"type":"range_usd","min_usd": $rate,"max_usd": $rate * 2,"format":{"suffix":"/second of input video"}} : {"type":"usd","usd": $rate * 2,"format":{"suffix":"/second of input video"}} @@ -3172,13 +3223,17 @@ def define_schema(cls) -> IO.Schema: ), ) + @classmethod + def validate_inputs(cls, **kwargs) -> bool: + return True + @classmethod async def execute(cls, model: dict) -> IO.NodeOutput: images = [image for batch in model["reference_images"].values() if batch is not None for image in batch] videos = [("base_video", model["video_1"])] videos += [("feature_video", v) for v in model["reference_videos"].values() if v is not None] _validate_kling4_images(images) - _validate_kling4_videos([v for _, v in videos]) + _validate_kling4_videos([v for _, v in videos], KLING4_VIDEO_SECONDS_LIMIT[model["model"]]) prompt = _kling4_prompt(model["prompt"], len(images), len(videos)) validate_string(prompt, min_length=1, max_length=26000) return await _run_kling_task( @@ -3186,7 +3241,11 @@ async def execute(cls, model: dict) -> IO.NodeOutput: f"/proxy/kling/omni-video/{model['model']}", Kling4ContentsRequest( contents=[Kling4Content(type="prompt", text=prompt), *await _kling4_media_contents(cls, images, videos)], - settings=Kling4Settings(resolution=model["resolution"], aspect_ratio=model["aspect_ratio"]), + settings=Kling4Settings( + resolution=model["resolution"], + aspect_ratio=model["aspect_ratio"], + audio="original" if model.get("audio") == "original" else None, + ), ), ) diff --git a/comfy_api_nodes/nodes_openai.py b/comfy_api_nodes/nodes_openai.py index 53774985f07..758f005aeed 100644 --- a/comfy_api_nodes/nodes_openai.py +++ b/comfy_api_nodes/nodes_openai.py @@ -401,7 +401,14 @@ async def execute( GPT_IMAGE_QUALITIES = ("low", "medium", "high") GPT_IMAGE_25_QUALITIES = ("low", "medium", "high", "xhigh", "max") -GPT_IMAGE_MODELS = ("gpt-image-2.5-flare", "gpt-image-2.5-sunburst", "gpt-image-2", "gpt-image-1.5") +GPT_IMAGE_MODELS = ( + "gpt-image-2.6-flare", + "gpt-image-2.6-sunburst", + "gpt-image-2.5-flare", + "gpt-image-2.5-sunburst", + "gpt-image-2", + "gpt-image-1.5", +) def _gpt_image_shared_inputs(qualities: tuple[str, ...] = GPT_IMAGE_QUALITIES): @@ -495,13 +502,25 @@ def _gpt_image_2_model_inputs(backgrounds: tuple[str, ...], qualities: tuple[str ] +def _gpt_image_26_model_inputs(): + return [ + *_gpt_image_2_model_inputs(("auto", "opaque", "transparent"), GPT_IMAGE_25_QUALITIES), + IO.Combo.Input( + "complexity", + default="high", + options=["low", "medium", "high"], + tooltip="How much the model plans the image before rendering.", + ), + ] + + class OpenAIGPTImageNodeV2(IO.ComfyNode): @classmethod def define_schema(cls): return IO.Schema( node_id="OpenAIGPTImageNodeV2", - display_name="OpenAI GPT Image 2.5", + display_name="OpenAI GPT Image 2.6", category="partner/image/OpenAI", description="Generates images via OpenAI's GPT Image endpoint.", inputs=[ @@ -514,6 +533,8 @@ def define_schema(cls): IO.DynamicCombo.Input( "model", options=[ + IO.DynamicCombo.Option("gpt-image-2.6-flare", _gpt_image_26_model_inputs()), + IO.DynamicCombo.Option("gpt-image-2.6-sunburst", _gpt_image_26_model_inputs()), IO.DynamicCombo.Option( "gpt-image-2.5-flare", _gpt_image_2_model_inputs(("auto", "opaque", "transparent"), GPT_IMAGE_25_QUALITIES), @@ -584,6 +605,20 @@ def define_schema(cls): "high": [0.0222, 0.2544], "xhigh": [0.0388, 0.4523], "max": [0.0887, 1.0175] + }, + "gpt-image-2.6-flare": { + "low": [0.0097, 0.1188], + "medium": [0.0181, 0.2100], + "high": [0.0222, 0.2544], + "xhigh": [0.0388, 0.4523], + "max": [0.0887, 1.0175] + }, + "gpt-image-2.6-sunburst": { + "low": [0.0097, 0.1188], + "medium": [0.0181, 0.2100], + "high": [0.0222, 0.2544], + "xhigh": [0.0388, 0.4523], + "max": [0.0887, 1.0175] } }; $presets := { @@ -598,16 +633,27 @@ def define_schema(cls): "high": {"1024x1024": 0.0753, "1024x1536": 0.0589, "1536x1024": 0.0589, "2048x2048": 0.1531, "2048x1152": 0.0606, "1152x2048": 0.0606, "3840x2160": 0.1431, "2160x3840": 0.1431}, "xhigh": {"1024x1024": 0.1339, "1024x1536": 0.1055, "1536x1024": 0.1055, "2048x2048": 0.2721, "2048x1152": 0.1077, "1152x2048": 0.1077, "3840x2160": 0.2544, "2160x3840": 0.2544}, "max": {"1024x1024": 0.3013, "1024x1536": 0.2354, "1536x1024": 0.2354, "2048x2048": 0.6123, "2048x1152": 0.2424, "1152x2048": 0.2424, "3840x2160": 0.5724, "2160x3840": 0.5724} + }, + "gpt-image-2.6": { + "low": {"1024x1024": 0.0353, "1024x1536": 0.0285, "1536x1024": 0.0285, "2048x2048": 0.0716, "2048x1152": 0.0283, "1152x2048": 0.0283, "3840x2160": 0.0669, "2160x3840": 0.0669}, + "medium": {"1024x1024": 0.0622, "1024x1536": 0.0486, "1536x1024": 0.0486, "2048x2048": 0.1263, "2048x1152": 0.0520, "1152x2048": 0.0520, "3840x2160": 0.1225, "2160x3840": 0.1225}, + "high": {"1024x1024": 0.0753, "1024x1536": 0.0589, "1536x1024": 0.0589, "2048x2048": 0.1531, "2048x1152": 0.0606, "1152x2048": 0.0606, "3840x2160": 0.1431, "2160x3840": 0.1431}, + "xhigh": {"1024x1024": 0.1339, "1024x1536": 0.1055, "1536x1024": 0.1055, "2048x2048": 0.2721, "2048x1152": 0.1077, "1152x2048": 0.1077, "3840x2160": 0.2544, "2160x3840": 0.2544}, + "max": {"1024x1024": 0.3013, "1024x1536": 0.2354, "1536x1024": 0.2354, "2048x2048": 0.6123, "2048x1152": 0.2424, "1152x2048": 0.2424, "3840x2160": 0.5724, "2160x3840": 0.5724} } }; $perImage := { "gpt-image-1.5": [0.0016, 0.0016], "gpt-image-2": [0.0098, 0.0147], "gpt-image-2.5-flare": [0.0117, 0.0176], - "gpt-image-2.5-sunburst": [0.0117, 0.0176] + "gpt-image-2.5-sunburst": [0.0117, 0.0176], + "gpt-image-2.6-flare": [0.0117, 0.0176], + "gpt-image-2.6-sunburst": [0.0117, 0.0176] }; $model := widgets.model; - $family := ($model = "gpt-image-2.5-flare" or $model = "gpt-image-2.5-sunburst") ? "gpt-image-2.5" : $model; + $family := ($model = "gpt-image-2.6-flare" or $model = "gpt-image-2.6-sunburst") ? "gpt-image-2.6" + : ($model = "gpt-image-2.5-flare" or $model = "gpt-image-2.5-sunburst") ? "gpt-image-2.5" + : $model; $qualityRaw := $lookup(widgets, "model.quality"); $quality := ($qualityRaw != null) ? $qualityRaw : ""; $sizeRaw := $lookup(widgets, "model.size"); @@ -647,6 +693,7 @@ async def execute( quality = model["quality"] custom_width = model.get("custom_width", 1024) custom_height = model.get("custom_height", 1024) + complexity = model.get("complexity") images_dict = model.get("images") or {} image_tensors: list[Input.Image] = [t for t in images_dict.values() if t is not None] @@ -734,6 +781,7 @@ async def execute( n=n, size=size, moderation="low", + complexity=complexity, ), content_type="multipart/form-data", files=files, @@ -752,6 +800,7 @@ async def execute( n=n, size=size, moderation="low", + complexity=complexity, ), ) return IO.NodeOutput(await validate_and_cast_response(response, cls=cls)) diff --git a/comfy_api_nodes/nodes_pika.py b/comfy_api_nodes/nodes_pika.py new file mode 100644 index 00000000000..4cec4802cde --- /dev/null +++ b/comfy_api_nodes/nodes_pika.py @@ -0,0 +1,575 @@ +import re +import uuid +from io import BytesIO + +import av +from pydantic import BaseModel +from typing_extensions import override + +from comfy_api.latest import IO, ComfyExtension, Input, Types +from comfy_api_nodes.apis.pika import ( + PikaH3FlexImageRequest, + PikaH3FlexReference, + PikaH3FlexReferenceRequest, + PikaH3FlexRequest, + PikaJob, + PikaJobOutput, + PikaMusicRequest, + PikaSFXRequest, + PikaSoundtrackRequest, +) +from comfy_api_nodes.util import ( + ApiEndpoint, + audio_bytes_to_audio_input, + download_url_as_bytesio, + download_url_to_video_output, + poll_op, + sync_op, + upload_audio_to_comfyapi, + upload_file_to_comfyapi, + upload_image_to_comfyapi, + upload_video_to_comfyapi, + validate_audio_duration, + validate_string, + validate_video_duration, +) + +PIKA_H3_FLEX_ASPECT_RATIOS = ["auto", "21:9", "16:9", "3:2", "4:3", "5:4", "1:1", "4:5", "3:4", "2:3", "9:16", "9:21"] +PIKA_H3_FLEX_REF_TAG_RE = re.compile(r"@(?Pimage|video|audio)(?P\d*)(?!\w)", re.IGNORECASE | re.ASCII) +PIKA_H3_FLEX_REF_LABELS = {"image": "Picture", "video": "Video", "audio": "Audio"} + + +def _seed_input() -> IO.Int.Input: + return IO.Int.Input( + "seed", + default=42, + min=0, + max=2147483647, + control_after_generate=True, + tooltip="With rewrite_prompt off, the same seed repeats the video exactly. " + "A local run with the same seed gives a different result.", + ) + + +def _audio_seed_input() -> IO.Int.Input: + return IO.Int.Input("seed", default=42, min=0, max=2147483647, control_after_generate=True) + + +def _h3_flex_settings_inputs(aspect_ratio_tooltip: str) -> list[IO.Input]: + return [ + IO.Combo.Input( + "resolution", + options=["480p", "768p", "1080p", "2k", "4k"], + default="768p", + tooltip="1080p and above render at 768p and upscale once. The same seed at 480p gives a different video.", + ), + IO.Combo.Input( + "aspect_ratio", + options=PIKA_H3_FLEX_ASPECT_RATIOS, + default="auto", + tooltip=aspect_ratio_tooltip, + ), + IO.Int.Input( + "duration", + default=5, + min=5, + max=15, + tooltip="The output length follows a frame grid: 5 seconds gives 124 frames, about 5.17 seconds.", + ), + IO.Boolean.Input( + "rewrite_prompt", + default=False, + tooltip="Let Pika expand the prompt first. The same seed then gives a different video on each run.", + ), + _seed_input(), + ] + + +def _resolve_ref_tags(prompt: str, counts: dict[str, int]) -> str: + parts = [] + pos = 0 + prev_end = -1 + for match in PIKA_H3_FLEX_REF_TAG_RE.finditer(prompt): + start = match.start() + if ( + start > 0 + and start != prev_end + and prompt[start - 1].isascii() + and (prompt[start - 1].isalnum() or prompt[start - 1] == "_") + ): + continue + kind = match.group("kind").lower() + idx = int(match.group("idx") or 1) + if not 1 <= idx <= counts[kind]: + raise ValueError( + f"The prompt references @{kind.capitalize()}{idx}, but {counts[kind]} {kind} input(s) are available." + ) + parts.append(prompt[pos:start]) + parts.append(f"<{PIKA_H3_FLEX_REF_LABELS[kind]} {idx}>") + pos = match.end() + prev_end = match.end() + parts.append(prompt[pos:]) + return "".join(parts) + + +async def _upload_reference_video(cls: type[IO.ComfyNode], video: Input.Video, keep_audio: bool, label: str) -> str: + if keep_audio: + return await upload_video_to_comfyapi(cls, video, wait_label=label) + source = BytesIO() + video.save_to(source, format=Types.VideoContainer.MP4, codec=Types.VideoCodec.H264) + source.seek(0) + silent = BytesIO() + with av.open(source) as src, av.open(silent, mode="w", format="mp4") as dst: + src_stream = src.streams.video[0] + dst_stream = dst.add_stream_from_template(src_stream) + for packet in src.demux(src_stream): + if packet.dts is not None: + packet.stream = dst_stream + dst.mux(packet) + silent.seek(0) + return await upload_file_to_comfyapi(cls, silent, f"{uuid.uuid4()}.mp4", "video/mp4", label) + + +async def _pika_job(cls: type[IO.ComfyNode], path: str, request: BaseModel) -> PikaJobOutput: + job = await sync_op( + cls, + ApiEndpoint(path=f"/proxy/pika/v1/media/pika/{path}", method="POST"), + response_model=PikaJob, + data=request, + ) + job = await poll_op( + cls, + ApiEndpoint(path=f"/proxy/pika/v1/media/jobs/{job.id}"), + response_model=PikaJob, + status_extractor=lambda r: r.status, + completed_statuses=["completed"], + failed_statuses=["failed"], + queued_statuses=["queued"], + ) + if job.output is None: + raise Exception(f"Pika returned no output for job {job.id}.") + return job.output + + +async def _pika_video(cls: type[IO.ComfyNode], path: str, request: BaseModel) -> IO.NodeOutput: + output = await _pika_job(cls, path, request) + return IO.NodeOutput(await download_url_to_video_output(output.video.url, cls=cls)) + + +async def _pika_audio(cls: type[IO.ComfyNode], path: str, request: BaseModel) -> IO.NodeOutput: + output = await _pika_job(cls, path, request) + audio = await download_url_as_bytesio(output.audio.url, cls=cls) + return IO.NodeOutput(audio_bytes_to_audio_input(audio.getvalue())) + + +class PikaH3FlexVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PikaH3FlexVideoNode", + display_name="Pika H3 Flex Video", + category="partner/video/Pika", + description="Pika H3 Flex, Pika's post-trained MiniMax H3. " + "Generate video with synchronized audio from text and optional first and last frames.", + search_aliases=["minimax h3", "h3 flex", "pika h3", "text to video", "image to video", "first last frame"], + inputs=[ + IO.String.Input("prompt", multiline=True, default=""), + IO.Image.Input("first_frame", optional=True), + IO.Image.Input("last_frame", optional=True), + *_h3_flex_settings_inputs("auto follows the first connected frame, otherwise the prompt."), + ], + outputs=[IO.Video.Output()], + hidden=[IO.Hidden.auth_token_comfy_org, IO.Hidden.api_key_comfy_org, IO.Hidden.unique_id], + is_api_node=True, + price_badge=IO.PriceBadge( + depends_on=IO.PriceBadgeDepends(widgets=["resolution", "duration"]), + expr=""" + ( + $res := widgets.resolution; + $d := widgets.duration; + $rate := $lookup({"480p": 0.0143, "768p": 0.0286, "1080p": 0.05005, "2k": 0.0572, "4k": 0.1144}, $res); + $type($d) = "number" + ? {"type":"usd","usd": $rate * (($res = "480p" or $res = "768p") + ? $d : ($ceil((24 * $d - 5) / 17) * 17 + 5) / 24)} + : {"type":"usd","usd": $rate,"format":{"suffix":"/second"}} + ) + """, + ), + ) + + @classmethod + async def execute( + cls, + prompt: str, + resolution: str, + aspect_ratio: str, + duration: int, + rewrite_prompt: bool, + seed: int, + first_frame: Input.Image | None = None, + last_frame: Input.Image | None = None, + ) -> IO.NodeOutput: + validate_string(prompt, min_length=1, max_length=8000) + settings = { + "prompt": prompt, + "duration": duration, + "aspect_ratio": aspect_ratio, + "resolution": resolution, + "seed": seed, + "rewrite": rewrite_prompt, + } + if first_frame is None and last_frame is None: + return await _pika_video(cls, "pika-h3-flex/text-to-video", PikaH3FlexRequest(**settings)) + return await _pika_video( + cls, + "pika-h3-flex/image-to-video", + PikaH3FlexImageRequest( + **settings, + image=None + if first_frame is None + else await upload_image_to_comfyapi(cls, first_frame, wait_label="Uploading first frame"), + last_frame_image=None + if last_frame is None + else await upload_image_to_comfyapi(cls, last_frame, wait_label="Uploading last frame"), + ), + ) + + +class PikaH3FlexReferenceVideoNode(IO.ComfyNode): + @classmethod + def define_schema(cls): + return IO.Schema( + node_id="PikaH3FlexReferenceVideoNode", + display_name="Pika H3 Flex Reference to Video", + category="partner/video/Pika", + description="Pika H3 Flex, Pika's post-trained MiniMax H3. " + "Generate or edit video with synchronized audio guided by reference images, videos and audio.", + search_aliases=["minimax h3", "h3 flex", "pika h3", "video editing", "motion transfer", "reference video"], + inputs=[ + IO.Autogrow.Input( + "reference_images", + template=IO.Autogrow.TemplateNames( + IO.Image.Input("reference_image"), + names=[f"image_{i}" for i in range(1, 10)], + min=0, + ), + optional=True, + tooltip="Up to 9 images, referred to in the prompt as @Image1..@Image9 in connection order. " + "At least one image or video is required.", + ), + IO.Autogrow.Input( + "reference_videos", + template=IO.Autogrow.TemplateNames( + IO.Video.Input("reference_video"), + names=[f"video_{i}" for i in range(1, 4)], + min=0, + ), + optional=True, + tooltip="Up to 3 videos, referred to in the prompt as @Video1..@Video3 in connection order. " + "2-15 seconds each, 15 seconds in total. Billed per second.", + ), + IO.Autogrow.Input( + "reference_audios", + template=IO.Autogrow.TemplateNames( + IO.Audio.Input("reference_audio"), + names=[f"audio_{i}" for i in range(1, 4)], + min=0, + ), + optional=True, + tooltip="Up to 3 clips, referred to in the prompt as @Audio1..@Audio3. " + "2-15 seconds each, 15 seconds in total. When copied to the output, clips play in connection order, " + "ahead of the sound of connected videos, whichever number the prompt names.", + ), + IO.String.Input( + "prompt", + multiline=True, + default="", + tooltip="Refer to references as @Image1, @Video1, @Audio1, or as ,