diff --git a/assets/example_data/waypoint/example_controls.json b/assets/example_data/waypoint/example_controls.json new file mode 100644 index 000000000..652fec219 --- /dev/null +++ b/assets/example_data/waypoint/example_controls.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "actions": [ + {"mouse_dx": 0.2, "mouse_dy": 0.2}, {"buttons": [32]}, {}, {}, {}, {"buttons": [1]}, {}, {}, {"buttons": [1, 32]}, {}, {}, {}, {}, {}, {}, + {"mouse_dx": 0.2, "mouse_dy": 0.2}, {"buttons": [32]}, {}, {}, {}, {"buttons": [1]}, {}, {}, {"buttons": [1, 32]}, {}, {}, {}, {}, {}, {}, + {"mouse_dx": 0.2, "mouse_dy": 0.2}, {"buttons": [32]}, {}, {}, {}, {"buttons": [1]}, {}, {}, {"buttons": [1, 32]}, {}, {}, {}, {}, {}, {}, + {"mouse_dx": 0.2, "mouse_dy": 0.2}, {"buttons": [32]}, {}, {}, {}, {"buttons": [1]}, {}, {}, {"buttons": [1, 32]}, {}, {}, {}, {}, {}, {}, + {}, {}, {}, {}, {}, {}, {}, {}, + {"buttons": [32]}, {"buttons": [32]}, {"buttons": [32]}, {"buttons": [32]}, {"buttons": [32]}, {"buttons": [32]}, {"buttons": [32]}, {"buttons": [32]}, {"buttons": [32]}, {"buttons": [32]}, + {"buttons": [65]}, {"buttons": [65]}, {"buttons": [65]}, {"buttons": [65]}, {"buttons": [65]}, {"buttons": [65]}, {"buttons": [65]}, {"buttons": [65]}, {"buttons": [65]}, {"buttons": [65]}, + {"buttons": [68]}, {"buttons": [68]}, {"buttons": [68]}, {"buttons": [68]}, {"buttons": [68]}, {"buttons": [68]}, {"buttons": [68]}, {"buttons": [68]}, {"buttons": [68]}, {"buttons": [68]}, + {"buttons": [83]}, {"buttons": [83]}, {"buttons": [83]}, {"buttons": [83]}, {"buttons": [83]}, {"buttons": [83]}, {"buttons": [83]}, {"buttons": [83]}, {"buttons": [83]}, {"buttons": [83]}, + {}, {}, {}, {}, {}, {}, {}, {}, {}, {} + ] +} diff --git a/flashdreams/flashdreams/recipes/taehv/__init__.py b/flashdreams/flashdreams/recipes/taehv/__init__.py index 372a4066a..353dc63e5 100644 --- a/flashdreams/flashdreams/recipes/taehv/__init__.py +++ b/flashdreams/flashdreams/recipes/taehv/__init__.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""TAEHV video decoder.""" +"""TAEHV video decoder and Hunyuan Video 1.5 codec configs.""" from __future__ import annotations @@ -26,16 +26,18 @@ from torch import Tensor from flashdreams.infra.decoder import DecoderConfig, StreamingVideoDecoder +from flashdreams.infra.encoder import EncoderConfig, StreamingVideoEncoder from flashdreams.recipes.taehv.checkpoint import ( StateDictTransform, compose, legacy_to_blocks_keys, truncate_oversize_tgrow_weights, ) -from flashdreams.recipes.taehv.impl import TAEHV, TAEHVCache +from flashdreams.recipes.taehv.impl import TAEHV, TAEHVCache, TAEHVEncoderCache AVAILABLE_TAEHV_CHECKPOINT_PATHS = { "lighttae": "https://huggingface.co/lightx2v/Autoencoders/resolve/main/lighttaew2_1.pth", + "hy1_5": "https://huggingface.co/Overworld-Models/taehv1_5/resolve/main/taehv1_5.pth", } """Checkpoint paths for the TAEHV decoder.""" @@ -230,9 +232,136 @@ def get_input_temporal_size( return output_temporal_size // r -if __name__ == "__main__": - import tyro +@dataclass(kw_only=True) +class Hy15TAEHVDecoderConfig(DecoderConfig): + """Config for the Hunyuan Video 1.5 TAEHV decoder.""" + + _target: Annotated[type, tyro.conf.Suppress] = field( + default_factory=lambda: Hy15TAEHVDecoder + ) + + checkpoint_path: str = AVAILABLE_TAEHV_CHECKPOINT_PATHS["hy1_5"] + """Path or URL for the Hunyuan Video 1.5 TAEHV checkpoint.""" + + state_dict_transform: StateDictTransform | None = legacy_to_blocks_keys + """Pre-load state-dict remap from the published flat key layout.""" + + dtype: torch.dtype = torch.bfloat16 + """Network parameter / activation dtype.""" + + use_cuda_graph: bool = True + """Wrap the decoder forward in a CUDA graph for replay.""" + + use_compile: bool = True + """``torch.compile(mode="max-autotune-no-cudagraphs")``.""" + + +class Hy15TAEHVDecoder(TeahvVAEDecoder): + """Hunyuan Video 1.5 TAEHV decoder with raw 32-channel latents.""" + + TEMPORAL_COMPRESSION_RATIO = 4 + SPATIAL_COMPRESSION_RATIO = 16 + + def __init__(self, config: Hy15TAEHVDecoderConfig) -> None: + StreamingVideoDecoder.__init__(self, config) + self.config: Hy15TAEHVDecoderConfig = config + self.need_scaled = False + self.taehv = TAEHV( + checkpoint_path=config.checkpoint_path, + model_type="hy1_5", + use_cuda_graph=config.use_cuda_graph, + use_compile=config.use_compile, + state_dict_transform=config.state_dict_transform, + ).to(dtype=config.dtype) + + +@dataclass(kw_only=True) +class Hy15TAEHVEncoderConfig(EncoderConfig): + """Config for the Hunyuan Video 1.5 TAEHV encoder.""" + + _target: Annotated[type, tyro.conf.Suppress] = field( + default_factory=lambda: Hy15TAEHVEncoder + ) + + checkpoint_path: str = AVAILABLE_TAEHV_CHECKPOINT_PATHS["hy1_5"] + """Path or URL for the Hunyuan Video 1.5 TAEHV checkpoint.""" + + state_dict_transform: StateDictTransform | None = legacy_to_blocks_keys + """Pre-load state-dict remap from the published flat key layout.""" + + dtype: torch.dtype = torch.bfloat16 + """Network parameter / activation dtype.""" + + +class Hy15TAEHVEncoder(StreamingVideoEncoder[TAEHVEncoderCache]): + """Hunyuan Video 1.5 TAEHV pixel-video encoder.""" + + TEMPORAL_COMPRESSION_RATIO = 4 + SPATIAL_COMPRESSION_RATIO = 16 + def __init__(self, config: Hy15TAEHVEncoderConfig) -> None: + super().__init__(config) + self.config: Hy15TAEHVEncoderConfig = config + self.taehv = TAEHV( + checkpoint_path=config.checkpoint_path, + model_type="hy1_5", + enable_encoder=True, + use_cuda_graph=False, + use_compile=False, + state_dict_transform=config.state_dict_transform, + ).to(dtype=config.dtype) + + def initialize_autoregressive_cache(self) -> TAEHVEncoderCache: + """Return an empty causal encoder cache.""" + return self.taehv.prepare_encoder_cache() + + @torch.no_grad() + def forward( + self, + input: Tensor, + autoregressive_index: int = 0, + cache: TAEHVEncoderCache | None = None, + ) -> Tensor: + """Encode frames in ``[-1, 1]`` to raw Hunyuan Video 1.5 latents.""" + if cache is None: + cache = self.initialize_autoregressive_cache() + assert input.ndim >= 4, "Expected input to have shape [..., T, C, H, W]" + + *batch_shape, T, C, H, W = input.shape + batch_size = math.prod(batch_shape) + x = input.reshape(batch_size, T, C, H, W).add(1).mul_(0.5) + z = self.taehv.encode(x, cache=cache) + return z.reshape(*batch_shape, *z.shape[1:]) + + @property + def temporal_compression_ratio(self) -> int: + """Pixel frames / latent frames for complete causal groups.""" + return self.TEMPORAL_COMPRESSION_RATIO + + @property + def spatial_compression_ratio(self) -> int: + """Pixel side / latent side.""" + return self.SPATIAL_COMPRESSION_RATIO + + def get_output_temporal_size( + self, autoregressive_index: int, input_temporal_size: int + ) -> int: + """Return latent count emitted from complete input frame groups.""" + r = self.temporal_compression_ratio + assert input_temporal_size % r == 0, ( + f"Hy15 TAEHV encoder input_temporal_size={input_temporal_size} must be " + f"divisible by temporal_compression_ratio={r}." + ) + return input_temporal_size // r + + def get_input_temporal_size( + self, autoregressive_index: int, output_temporal_size: int + ) -> int: + """Return pixel frame count needed for ``output_temporal_size`` latents.""" + return output_temporal_size * self.temporal_compression_ratio + + +if __name__ == "__main__": config = tyro.cli(TeahvVAEDecoderConfig) model = config.setup() print(model) diff --git a/flashdreams/flashdreams/recipes/taehv/checkpoint.py b/flashdreams/flashdreams/recipes/taehv/checkpoint.py index e6fd4301f..750f97720 100644 --- a/flashdreams/flashdreams/recipes/taehv/checkpoint.py +++ b/flashdreams/flashdreams/recipes/taehv/checkpoint.py @@ -36,22 +36,22 @@ def legacy_to_blocks_keys( sd: Mapping[str, torch.Tensor], ) -> dict[str, torch.Tensor]: - """Re-key legacy ``decoder..*`` weights to ``decoder.blocks..*``. + """Re-key flat encoder and decoder weights to their ``blocks`` layouts. - The current :class:`~flashdreams.recipes.taehv.impl.Decoder` wraps - its ``Sequential`` in a ``blocks`` attribute, so older checkpoints - whose keys flatten to ``decoder..*`` need rewriting to line up. - Keys already under ``decoder.blocks.`` (and keys outside the - ``decoder.`` subtree) pass through unchanged. + The current :class:`~flashdreams.recipes.taehv.impl.Encoder` and + :class:`~flashdreams.recipes.taehv.impl.Decoder` each wrap their + ``Sequential`` in a ``blocks`` attribute. Published checkpoints use + flat ``encoder..*`` and ``decoder..*`` keys. Keys already + under ``*.blocks.`` (and unrelated keys) pass through unchanged. """ - return { - ( - k.replace("decoder.", "decoder.blocks.", 1) - if k.startswith("decoder.") and not k.startswith("decoder.blocks.") - else k - ): v - for k, v in sd.items() - } + out: dict[str, torch.Tensor] = {} + for key, value in sd.items(): + for prefix in ("encoder.", "decoder."): + if key.startswith(prefix) and not key.startswith(f"{prefix}blocks."): + key = key.replace(prefix, f"{prefix}blocks.", 1) + break + out[key] = value + return out def truncate_oversize_tgrow_weights( diff --git a/flashdreams/flashdreams/recipes/taehv/impl.py b/flashdreams/flashdreams/recipes/taehv/impl.py index 53d335bd5..2423f3ff7 100755 --- a/flashdreams/flashdreams/recipes/taehv/impl.py +++ b/flashdreams/flashdreams/recipes/taehv/impl.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Streaming causal decoder for TAEHV (Tiny AutoEncoder for Hunyuan Video).""" +"""Streaming causal TAEHV encoder and decoder.""" from __future__ import annotations from dataclasses import dataclass, field -from typing import Callable, Dict, Optional +from typing import Callable, Dict, Optional, cast import torch import torch.nn as nn @@ -28,6 +28,7 @@ from flashdreams.infra.compile import compile_module from flashdreams.infra.cuda_graph import CUDAGraphWrapper, set_or_copy from flashdreams.infra.decoder import StreamingDecoderCache +from flashdreams.infra.encoder import StreamingEncoderCache from flashdreams.recipes.taehv.checkpoint import ( StateDictTransform, compose, @@ -48,6 +49,19 @@ class TAEHVCache(StreamingDecoderCache): dec_state: Dict[int, torch.Tensor] = field(default_factory=dict) +@dataclass +class TAEHVEncoderCache(StreamingEncoderCache): + """Streaming encoder work queue and per-layer causal state.""" + + enc_state: list[torch.Tensor | list[torch.Tensor] | None] = field( + default_factory=list + ) + """Causal ``MemBlock`` frames and partial ``TPool`` groups per encoder layer.""" + + work_queue: list[tuple[torch.Tensor, int]] = field(default_factory=list) + """Pending ``(frame, block_index)`` items in causal evaluation order.""" + + def _conv(n_in: int, n_out: int, **kwargs) -> nn.Conv2d: return nn.Conv2d(n_in, n_out, 3, padding=1, **kwargs) @@ -120,6 +134,19 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return self.conv(x).reshape(-1, C, H, W) +class TPool(nn.Module): + """Temporal downsample by ``stride`` (channel-pack + projection).""" + + def __init__(self, n_f: int, stride: int): + super().__init__() + self.stride = stride + self.conv = nn.Conv2d(n_f * stride, n_f, 1, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + _NT, C, H, W = x.shape + return self.conv(x.reshape(-1, self.stride * C, H, W)) + + class Decoder(nn.Module): """TAEHV decoder body. @@ -224,18 +251,119 @@ def initialize_state( x = blk(x) -class TAEHV(nn.Module): - """TAEHV streaming decode-only network. +class Encoder(nn.Module): + """TAEHV encoder body with causal temporal pools and memory blocks.""" + + def __init__( + self, + latent_channels: int, + image_channels: int, + patch_size: int, + act_func: nn.Module, + ): + super().__init__() + self.blocks = nn.Sequential( + _conv(image_channels * patch_size**2, 64), + act_func, + TPool(64, 2), + _conv(64, 64, stride=2, bias=False), + MemBlock(64, 64, act_func), + MemBlock(64, 64, act_func), + MemBlock(64, 64, act_func), + TPool(64, 2), + _conv(64, 64, stride=2, bias=False), + MemBlock(64, 64, act_func), + MemBlock(64, 64, act_func), + MemBlock(64, 64, act_func), + TPool(64, 1), + _conv(64, 64, stride=2, bias=False), + MemBlock(64, 64, act_func), + MemBlock(64, 64, act_func), + MemBlock(64, 64, act_func), + _conv(64, latent_channels), + ) - Loads a TAEHV checkpoint and exposes ``decode``. Encoder weights in the - checkpoint are silently dropped. With ``use_cuda_graph=True``, rollout 1 - drains Inductor autotune on the eager path; rollout 2 warms up and - captures, after which same-shape body chunks replay. + def initialize_state( + self, state: list[torch.Tensor | list[torch.Tensor] | None] + ) -> None: + """Populate empty per-layer causal encoder state.""" + state[:] = [None] * len(self.blocks) + + def _advance_work_queue( + self, + state: list[torch.Tensor | list[torch.Tensor] | None], + work_queue: list[tuple[torch.Tensor, int]], + ) -> torch.Tensor | None: + """Advance queued work until one latent frame is ready.""" + while work_queue: + x, block_index = work_queue.pop(0) + if block_index == len(self.blocks): + return x.unsqueeze(1) + + block = self.blocks[block_index] + if isinstance(block, MemBlock): + past = cast(torch.Tensor | None, state[block_index]) + state[block_index] = x + x = block(x, x * 0 if past is None else past) + work_queue.insert(0, (x, block_index + 1)) + elif isinstance(block, TPool): + group = cast(list[torch.Tensor] | None, state[block_index]) + if group is None: + group = [] + state[block_index] = group + group = cast(list[torch.Tensor], group) + group.append(x) + if len(group) == block.stride: + state[block_index] = [] + batch, channels, height, width = x.shape + x = block( + torch.cat(group, dim=1).view( + batch * block.stride, channels, height, width + ) + ) + work_queue.insert(0, (x, block_index + 1)) + elif len(group) > block.stride: + raise RuntimeError("TAEHV TPool cache overflow.") + else: + work_queue.insert(0, (block(x), block_index + 1)) + return None + + def forward( + self, + x: torch.Tensor, + state: list[torch.Tensor | list[torch.Tensor] | None], + work_queue: list[tuple[torch.Tensor, int]], + ) -> torch.Tensor: + """Encode a frame chunk with causal ``MemBlock`` and ``TPool`` state. - Supported ``model_type``: ``"wan21"`` (default; ReLU, patch_size=1, - latent_channels=16) and ``"wan22"`` (ReLU, patch_size=2, - latent_channels=48). The legacy ``"hy15"`` and ``"taecvx"`` variants are - not ported. + ``x`` has shape ``[B, T, C, H, W]``. Incomplete temporal groups remain + in ``work_queue`` / ``state`` and can therefore yield an empty ``T`` + dimension. + """ + work_queue.extend((frame, 0) for frame in x.unbind(dim=1)) + + outputs: list[torch.Tensor] = [] + while (output := self._advance_work_queue(state, work_queue)) is not None: + outputs.append(output) + if outputs: + return torch.cat(outputs, dim=1) + output_projection = cast(nn.Conv2d, self.blocks[-1]) + return x.new_empty( + x.shape[0], + 0, + output_projection.out_channels, + x.shape[-2] // 8, + x.shape[-1] // 8, + ) + + +class TAEHV(nn.Module): + """TAEHV streaming network with configurable decoder and encoder. + + Loads a TAEHV checkpoint and exposes ``decode`` and ``encode``. With + ``use_cuda_graph=True``, rollout 1 drains Inductor autotune on the eager + path; rollout 2 warms up and captures, after which same-shape body chunks + replay. Examples: @@ -268,10 +396,11 @@ class TAEHV(nn.Module): TEMPORAL_COMPRESSION_RATIO = 4 SPATIAL_COMPRESSION_RATIO = 8 - SUPPORTED_MODEL_TYPES = ("wan21", "wan22") + SUPPORTED_MODEL_TYPES = ("wan21", "wan22", "hy1_5") - # Concrete type so ``self.decoder`` access doesn't go through + # Concrete types so module access doesn't go through # ``nn.Module.__getattr__``'s ``Tensor | Module``. + encoder: "Encoder" decoder: "Decoder" def __init__( @@ -288,14 +417,16 @@ def __init__( use_compile: bool = False, warmup_iters: int = 2, state_dict_transform: StateDictTransform | None = None, + *, + enable_encoder: bool = False, ): super().__init__() if model_type not in self.SUPPORTED_MODEL_TYPES: raise ValueError( f"TAEHV: model_type={model_type!r} is not supported by this slim " f"impl (supported: {self.SUPPORTED_MODEL_TYPES}). The legacy " - f"'hy15' / 'taecvx' branches (different activation, clamp range, " - f"or trim semantics) were dropped in the decode-only refactor." + f"'taecvx' branches (different activation, clamp range, " + f"or trim semantics) were dropped in the refactor." ) if checkpoint_path is not None and "taecvx" in checkpoint_path: raise ValueError( @@ -304,6 +435,8 @@ def __init__( ) if model_type == "wan22": patch_size, latent_channels = 2, 48 + elif model_type == "hy1_5": + patch_size, latent_channels = 2, 32 act_func = nn.ReLU(inplace=True) self.patch_size = patch_size @@ -327,6 +460,10 @@ def __init__( decoder_space_upscale=decoder_space_upscale, act_func=act_func, ) + if enable_encoder: + self.encoder = Encoder( + latent_channels, self.image_channels, patch_size, act_func + ) # Runtime knobs consumed by ``load_from_checkpoint``; stashed here # so subclasses that defer the load (``checkpoint_path=None``) @@ -380,8 +517,8 @@ def load_from_checkpoint( ) sd = load_checkpoint(checkpoint_path) sd = state_dict_transform(sd) - # assign=True: meta params become the checkpoint tensors directly; - # strict=False: silently drop encoder-only weights. + # ``assign=True`` moves checkpoint tensors into meta parameters directly; + # ``strict=False`` lets decoder-only instances ignore encoder weights. self.load_state_dict(sd, strict=False, assign=True) self.eval().requires_grad_(False) @@ -420,6 +557,40 @@ def prepare_cache(self) -> TAEHVCache: self._decoder_wrapper.reset() return TAEHVCache() + def prepare_encoder_cache(self) -> TAEHVEncoderCache: + """Return a fresh cache for a causal streaming encode rollout.""" + cache = TAEHVEncoderCache() + self.encoder.initialize_state(cache.enc_state) + return cache + + @torch.no_grad() + def encode( + self, + x: torch.Tensor, + cache: TAEHVEncoderCache | None = None, + ) -> torch.Tensor: + """Encode pixel frames in ``[0, 1]`` with causal temporal pooling. + + Frames that do not complete the encoder's temporal factor remain in + ``cache`` for the next call. The returned tensor can therefore have zero + time frames when the caller submits an incomplete temporal group. + """ + if cache is None: + cache = self.prepare_encoder_cache() + + batch, _time, _channels, height, width = x.shape + if self.patch_size > 1: + x = F.pixel_unshuffle( + x.reshape(-1, x.shape[2], height, width), self.patch_size + ).reshape( + batch, + -1, + self.image_channels * self.patch_size**2, + height // self.patch_size, + width // self.patch_size, + ) + return self.encoder(x, cache.enc_state, cache.work_queue) + @torch.inference_mode() def decode( self, diff --git a/flashdreams/tests/test_vae.py b/flashdreams/tests/test_vae.py index 57c2121aa..3d8d0c5aa 100644 --- a/flashdreams/tests/test_vae.py +++ b/flashdreams/tests/test_vae.py @@ -21,9 +21,13 @@ from flashdreams.recipes.taehv import ( AVAILABLE_TAEHV_CHECKPOINT_PATHS, + Hy15TAEHVDecoderConfig, + Hy15TAEHVEncoderConfig, TeahvVAEDecoder, TeahvVAEDecoderConfig, ) +from flashdreams.recipes.taehv.checkpoint import legacy_to_blocks_keys +from flashdreams.recipes.taehv.impl import TAEHV from flashdreams.recipes.wan.autoencoder.vae import ( AVAILABLE_WAN_VAE_CHECKPOINT_PATHS, WanVAEDecoder, @@ -33,6 +37,52 @@ ) +@pytest.mark.ci_cpu +def test_hy15_taehv_checkpoint_layout_is_a_full_bijection() -> None: + """The Hunyuan Video 1.5 architecture matches the TAEHV checkpoint layout. + + Builds both branches on ``meta`` and recreates the published flat decoder + naming convention. This guards the 32-channel / patch-2 architecture and + verifies that the generic TAEHV remap covers every parameter without a + checkpoint download. + """ + with torch.device("meta"): + codec = TAEHV( + checkpoint_path=None, + model_type="hy1_5", + enable_encoder=True, + use_cuda_graph=False, + ) + model = codec.state_dict() + raw: dict[str, torch.Tensor] = {} + for key, value in model.items(): + for prefix in ("encoder.blocks.", "decoder.blocks."): + if key.startswith(prefix): + key = key.replace(prefix, prefix.replace(".blocks.", "."), 1) + break + raw[key] = value + transformed = legacy_to_blocks_keys(raw) + + assert len(model) == len(transformed) == 128 + assert set(model) == set(transformed) + assert all(model[key].shape == transformed[key].shape for key in model) + assert tuple(model["encoder.blocks.0.weight"].shape) == (64, 12, 3, 3) + assert tuple(model["decoder.blocks.1.weight"].shape) == (256, 32, 3, 3) + assert tuple(model["decoder.blocks.22.weight"].shape) == (12, 64, 3, 3) + + +@pytest.mark.ci_cpu +def test_hy15_taehv_configs_select_raw_32_channel_latents() -> None: + """The Hunyuan Video 1.5 presets use the published checkpoint unchanged.""" + encoder = Hy15TAEHVEncoderConfig() + decoder = Hy15TAEHVDecoderConfig() + + assert encoder.checkpoint_path == AVAILABLE_TAEHV_CHECKPOINT_PATHS["hy1_5"] + assert decoder.checkpoint_path == AVAILABLE_TAEHV_CHECKPOINT_PATHS["hy1_5"] + assert encoder.state_dict_transform is legacy_to_blocks_keys + assert decoder.state_dict_transform is legacy_to_blocks_keys + + @torch.no_grad() @pytest.mark.manual @pytest.mark.parametrize("tokenizer_choice", ["lightvae", "vae"]) diff --git a/integrations/waypoint/README.md b/integrations/waypoint/README.md new file mode 100644 index 000000000..9467409d1 --- /dev/null +++ b/integrations/waypoint/README.md @@ -0,0 +1,66 @@ + + +# Waypoint for FlashDreams + +This package loads the published [Overworld/Waypoint-1.5-1B](https://huggingface.co/Overworld/Waypoint-1.5-1B) checkpoint through +FlashDreams. + +## Model + +- One action produces one 32-channel latent frame and four presented RGB frames. +- Waypoint 1.5 has no text-conditioning input in its published checkpoint + configuration, so this integration will not expose a prompt encoder. +- User controls are 256 button IDs, mouse delta, and scroll movement. + +## Run a rollout + +Sync the workspace, then invoke the registered runner with a seed image and +repeated control: + +```bash +uv sync +uv run flashdreams-run waypoint-1.5-1b --seed-image .\seed.jpg --actions 45 --buttons 32 +``` + +The runner repeats the supplied button IDs, mouse displacement, and scroll value +for each action. Without ``--actions``, a repeated-control rollout emits four +actions. It writes ``outputs/waypoint-1.5-1b.mp4`` at 60 FPS by default. Pass +``--seed N`` to replay a rollout; otherwise the runner logs its generated seed. +Use ``uv run flashdreams-run waypoint-1.5-1b --help`` to list every override. + +For the pinned public example seed and its 118-action control sequence: + +```bash +uv run flashdreams-run waypoint-1.5-1b --example-data True +``` + +The seed image is cached under +``$FLASHDREAMS_CACHE_DIR/example_data/waypoint/``. The action timeline is the +versioned repository asset +``assets/example_data/waypoint/example_controls.json``. +The CLI requires an explicit Boolean value for this option. + +## Control files + +Pass a JSON action timeline with ``--controls-file``. It uses every listed +action unless ``--actions N`` selects a prefix. The file format is: + +```json +{ + "schema_version": 1, + "actions": [ + {"buttons": [32], "mouse_dx": 0.1, "mouse_dy": 0.0, "scroll_wheel": 0}, + {}, + {"buttons": [1, 32]} + ] +} +``` + +Every field within an action is optional. ``buttons`` is an array of model +button IDs, ``mouse_dx`` / ``mouse_dy`` are finite numbers, and +``scroll_wheel`` is ``-1``, ``0``, or ``1``. A control file takes precedence +over the repeated-control options, so it can be used without ``--buttons``, +``--mouse-dx``, ``--mouse-dy``, or ``--scroll``. diff --git a/integrations/waypoint/pyproject.toml b/integrations/waypoint/pyproject.toml new file mode 100644 index 000000000..894386240 --- /dev/null +++ b/integrations/waypoint/pyproject.toml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "flashdreams-waypoint" +version = "0.1.0" +description = "FlashDreams integration for the Waypoint 1.5 world-model checkpoint." +readme = "README.md" +requires-python = ">=3.10" +dependencies = ["flashdreams", "imageio[pyav]>=2.37", "opencv-python>=4.10"] + +[tool.uv.sources] +flashdreams = { workspace = true } + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[project.entry-points."flashdreams.runner_configs"] +"waypoint-1.5-1b" = "waypoint.config:RUNNER_WAYPOINT_1_5" + +[tool.setuptools.packages.find] +include = ["waypoint*"] +exclude = ["tests"] + +[tool.pytest.ini_options] +addopts = "--import-mode=importlib -p flashdreams._pytest_plugins.marker_enforcement" +markers = [ + "ci_cpu: CPU-safe test, runs on the CPU CI runner", + "ci_gpu: requires GPU or libGL (cv2), runs on the GPU CI runner", + "manual: heavy or environment-specific test, opt-in only", +] diff --git a/integrations/waypoint/tests/test_spec.py b/integrations/waypoint/tests/test_spec.py new file mode 100644 index 000000000..d74def3ee --- /dev/null +++ b/integrations/waypoint/tests/test_spec.py @@ -0,0 +1,607 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CPU-only contract tests for the independently authored Waypoint adapter.""" + +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch +from waypoint import ( + WAYPOINT_1_5, + WaypointControl, + load_controls_from_file, + make_control_context, +) +from waypoint.checkpoint import ( + expected_waypoint_1_5_checkpoint_keys, + expected_waypoint_1_5_checkpoint_shapes, + load_waypoint_state_dict, + validate_waypoint_1_5_checkpoint_keys, + validate_waypoint_1_5_checkpoint_shapes, +) +from waypoint.config import ( + PIPELINE_WAYPOINT_1_5, + RUNNER_CONFIGS, + RUNNER_WAYPOINT_1_5, +) +from waypoint.decoder import WaypointTAEHVDecoder +from waypoint.encoder import WaypointControlEncoderConfig +from waypoint.pipeline import WaypointInferencePipeline, WaypointInferencePipelineConfig +from waypoint.transformer import ( + WaypointAttentionPolicy, + WaypointDiTConfig, + WaypointKVCache, + WaypointOrthoRoPEAngles, + WaypointTransformerConfig, + adaptive_gate, + adaptive_rms_norm, + apply_waypoint_ortho_rope, +) +from waypoint.transformer.impl import WaypointTransformerCache +from waypoint.transformer.network import ( + _ConditionHead, + _ControlFusion, + _WaypointAttention, + _WaypointBlock, + sinusoidal_noise_embedding, +) + +EXAMPLE_CONTROL_FILE = ( + Path(__file__).parents[3] + / "assets" + / "example_data" + / "waypoint" + / "example_controls.json" +) + +pytestmark = pytest.mark.ci_cpu + + +def test_runner_registration_uses_the_image_established_pipeline() -> None: + """The CLI configuration points to the model-specific seed-state pipeline.""" + assert isinstance(PIPELINE_WAYPOINT_1_5, WaypointInferencePipelineConfig) + assert RUNNER_WAYPOINT_1_5.runner_name == PIPELINE_WAYPOINT_1_5.name + assert RUNNER_WAYPOINT_1_5.pipeline is PIPELINE_WAYPOINT_1_5 + assert RUNNER_CONFIGS == {RUNNER_WAYPOINT_1_5.runner_name: RUNNER_WAYPOINT_1_5} + + +def test_waypoint_1_5_static_contract() -> None: + """The published Waypoint configuration has stable model-shape invariants.""" + assert WAYPOINT_1_5.latent_shape() == (1, 1, 32, 32, 64) + assert WAYPOINT_1_5.tokens_per_latent_frame == 512 + assert WAYPOINT_1_5.patch_grid_height == 16 + assert WAYPOINT_1_5.patch_grid_width == 32 + assert WAYPOINT_1_5.frames_per_action == 4 + assert WAYPOINT_1_5.num_denoising_steps == 4 + assert WAYPOINT_1_5.frame_timestamp_stride == 1 + assert WAYPOINT_1_5.head_dim == 64 + assert WAYPOINT_1_5.global_attention_layers == (3, 7, 11, 15, 19, 23) + assert WAYPOINT_1_5.global_pinned_dilation == 8 + assert WAYPOINT_1_5.value_residual + assert WAYPOINT_1_5.noise_conditioning == "wan" + assert WAYPOINT_1_5.rope_theta == 10_000.0 + assert WAYPOINT_1_5.rope_nyquist_fraction == 0.8 + assert not WAYPOINT_1_5.text_conditioning + + +def test_control_context_preserves_waypoint_action_semantics() -> None: + """Buttons, motion, scroll, and latent-frame time use model-ready shapes.""" + context = make_control_context( + WaypointControl(buttons=frozenset({65, 87}), mouse_dx=0.125, mouse_dy=-0.25), + frame_index=7, + dtype=torch.float32, + ) + assert context["button"].shape == (1, 1, 256) + assert context["button"][0, 0, 65].item() == 1.0 + assert context["button"][0, 0, 87].item() == 1.0 + assert context["mouse"].tolist() == [[[0.125, -0.25]]] + assert context["scroll"].tolist() == [[[0.0]]] + assert context["frame_idx"].tolist() == [[7]] + assert context["frame_timestamp"].tolist() == [[7]] + + +def test_control_timeline_file_preserves_per_action_inputs(tmp_path) -> None: + """JSON control files retain neutral defaults and explicit action values.""" + path = tmp_path / "controls.json" + path.write_text( + """{ + "schema_version": 1, + "actions": [ + {"buttons": [32], "mouse_dx": 0.1, "scroll_wheel": -1}, + {} + ] +} +""", + encoding="utf-8", + ) + controls = load_controls_from_file(path) + assert controls == ( + WaypointControl(buttons=frozenset({32}), mouse_dx=0.1, scroll_wheel=-1), + WaypointControl(), + ) + + +def test_example_controls_are_complete() -> None: + """The runner example uses the complete fixed 118-action demonstration.""" + controls = load_controls_from_file(EXAMPLE_CONTROL_FILE) + assert len(controls) == 118 + assert controls[0] == WaypointControl(mouse_dx=0.2, mouse_dy=0.2) + assert controls[1] == WaypointControl(buttons=frozenset({32})) + assert controls[8] == WaypointControl(buttons=frozenset({1, 32})) + assert controls[68] == WaypointControl(buttons=frozenset({32})) + assert controls[78] == WaypointControl(buttons=frozenset({65})) + assert controls[88] == WaypointControl(buttons=frozenset({68})) + assert controls[98] == WaypointControl(buttons=frozenset({83})) + + +@pytest.mark.parametrize("buttons", [frozenset({-1}), frozenset({256})]) +def test_control_context_rejects_invalid_button_ids(buttons: frozenset[int]) -> None: + """The native adapter must never index outside the fixed button vocabulary.""" + with pytest.raises(ValueError, match="button IDs"): + make_control_context(WaypointControl(buttons=buttons), frame_index=0) + + +def test_waypoint_raw_checkpoint_schema_accounts_for_every_tensor() -> None: + """The checkpoint schema captures the published raw 393-tensor layout.""" + keys = expected_waypoint_1_5_checkpoint_keys() + assert len(keys) == 393 + validate_waypoint_1_5_checkpoint_keys(keys) + + +def test_waypoint_raw_checkpoint_schema_rejects_unknown_tensor() -> None: + """Loader work must not silently accept a changed checkpoint format.""" + keys = set(expected_waypoint_1_5_checkpoint_keys()) + keys.add("transformer.blocks.0.unknown.weight") + with pytest.raises(ValueError, match="extra"): + validate_waypoint_1_5_checkpoint_keys(keys) + + +def test_raw_checkpoint_shapes_match_the_published_schema() -> None: + """The checkpoint contract records every raw tensor shape without a fake network.""" + shapes = expected_waypoint_1_5_checkpoint_shapes() + assert set(shapes) == expected_waypoint_1_5_checkpoint_keys() + assert shapes["patchify.weight"] == (2048, 32, 2, 2) + assert shapes["unpatchify.weight"] == (2048, 32, 2, 2) + assert shapes["transformer.blocks.0.attn.k_proj.weight"] == (1024, 2048) + validate_waypoint_1_5_checkpoint_shapes(shapes) + + +def test_checkpoint_loader_requires_the_exact_native_namespace() -> None: + """Validated raw tensors load without a hidden key remap.""" + tiny_spec = replace( + WAYPOINT_1_5, + n_layers=1, + d_model=16, + n_heads=1, + n_kv_heads=1, + mlp_ratio=2, + ) + source = WaypointDiTConfig(spec=tiny_spec).setup() + target = WaypointDiTConfig(spec=tiny_spec).setup() + state_dict = source.state_dict() + load_waypoint_state_dict(target, state_dict, spec=tiny_spec) + assert torch.equal(target.patchify.weight, source.patchify.weight) + + +def test_raw_checkpoint_shape_validation_rejects_a_mismatch() -> None: + """Loader work must not accept a changed raw tensor shape.""" + shapes = expected_waypoint_1_5_checkpoint_shapes() + shapes["patchify.weight"] = (1,) + with pytest.raises(ValueError, match="shape mismatch"): + validate_waypoint_1_5_checkpoint_shapes(shapes) + + +def test_native_dit_topology_matches_the_raw_checkpoint_schema() -> None: + """The native DiT module graph can load every published raw tensor name.""" + with torch.device("meta"): + network = WaypointDiTConfig().setup() + state_shapes = { + key: tuple(value.shape) for key, value in network.state_dict().items() + } + assert state_shapes == expected_waypoint_1_5_checkpoint_shapes() + query, key, value = network.transformer.blocks[0].attn.project_qkv( + torch.empty(2, 512, WAYPOINT_1_5.d_model, device="meta") + ) + assert query.shape == (2, 512, 32, 64) + assert key.shape == value.shape == (2, 512, 16, 64) + + +def test_orthogonal_rope_angles_match_waypoint_geometry() -> None: + """RoPE reserves x, y, and time frequency bands in that exact order.""" + rope = WaypointOrthoRoPEAngles() + cosine, sine = rope( + frame_index=torch.tensor([0, 1]), + row_index=torch.tensor([0, 1]), + column_index=torch.tensor([0, 1]), + ) + assert cosine.shape == (2, 1, 32) + assert sine.shape == (2, 1, 32) + # At x=0, the first frequency is pi / 16 at the centered patch x=-15.5. + assert cosine[0, 0, 0].item() == pytest.approx(-0.9951847, abs=1e-6) + assert sine[0, 0, 0].item() == pytest.approx(-0.0980171, abs=1e-6) + # The temporal band follows the two eight-value spatial bands. + assert cosine[1, 0, 16].item() == pytest.approx(torch.cos(torch.tensor(1.0)).item()) + assert sine[1, 0, 16].item() == pytest.approx(torch.sin(torch.tensor(1.0)).item()) + + +def test_wan_noise_features_use_waypoints_scaled_sine_cosine_order() -> None: + """Noise modulation uses the checkpoint's 1000x, sqrt-two Fourier basis.""" + features = sinusoidal_noise_embedding(512, torch.tensor([1.0])) + assert features.shape == (1, 512) + assert features[0, 0].item() == pytest.approx(1.1693842, abs=1e-6) + assert features[0, 256].item() == pytest.approx(0.7953241, abs=1e-6) + + +def test_orthogonal_rope_rotates_two_half_heads() -> None: + """RoPE rotates each packed half-head pair without mixing attention heads.""" + tokens = torch.zeros(1, 1, 1, 64) + tokens[..., 0] = 1.0 + cosine = torch.full((1, 1, 32), 0.6) + sine = torch.full((1, 1, 32), 0.8) + output = apply_waypoint_ortho_rope(tokens, cosine, sine) + assert output[..., 0].item() == pytest.approx(0.6) + assert output[..., 32].item() == pytest.approx(0.8) + + +def test_adaptive_rms_norm_conditions_each_latent_frame() -> None: + """AdaRMSNorm broadcasts a separate scale and bias over each frame's tokens.""" + tokens = torch.tensor([[[1.0, 2.0], [3.0, 4.0], [2.0, 4.0], [6.0, 8.0]]]) + scale = torch.tensor([[[1.0, 0.0], [0.0, 1.0]]]) + bias = torch.tensor([[[0.1, 0.2], [0.3, 0.4]]]) + output = adaptive_rms_norm(tokens, scale, bias) + rms = torch.rsqrt(torch.mean(tokens.square(), dim=-1, keepdim=True)) + expected = tokens * rms + expected[:, :2] = expected[:, :2] * torch.tensor([2.0, 1.0]) + torch.tensor( + [0.1, 0.2] + ) + expected[:, 2:] = expected[:, 2:] * torch.tensor([1.0, 2.0]) + torch.tensor( + [0.3, 0.4] + ) + assert torch.allclose(output, expected) + + +def test_adaptive_gate_scales_each_latent_frame() -> None: + """AdaGate broadcasts each frame's learned residual multiplier over its tokens.""" + tokens = torch.ones(1, 4, 2) + gate = torch.tensor([[[2.0, 3.0], [5.0, 7.0]]]) + output = adaptive_gate(tokens, gate) + assert torch.equal( + output, + torch.tensor([[[2.0, 3.0], [2.0, 3.0], [5.0, 7.0], [5.0, 7.0]]]), + ) + + +def test_value_residual_blends_with_the_first_block_value_stream() -> None: + """Later blocks linearly blend their V tensor with the retained first V tensor.""" + attention = _WaypointAttention( + replace(WAYPOINT_1_5, d_model=8, n_heads=1, n_kv_heads=1) + ) + current = torch.tensor([[[[1.0, 2.0]]]]) + initial = torch.tensor([[[[10.0, 20.0]]]]) + attention.v_lamb.data.fill_(2.0) + mixed, retained = attention.blend_value_residual(current, initial) + assert torch.equal(mixed, torch.tensor([[[[19.0, 38.0]]]])) + assert retained is initial + + +def test_attention_uses_grouped_query_sparse_kv_view() -> None: + """Attention accepts cached KV heads without expanding them to query heads.""" + tiny_spec = replace( + WAYPOINT_1_5, + n_layers=4, + d_model=16, + n_heads=1, + n_kv_heads=1, + local_window=2, + global_window=6, + global_pinned_dilation=2, + ) + attention = _WaypointAttention(tiny_spec) + for projection in ( + attention.q_proj, + attention.k_proj, + attention.v_proj, + attention.out_proj, + ): + projection.weight.data.copy_(torch.eye(16)) + attention.v_lamb.data.zero_() + cache = WaypointKVCache(policy=WaypointAttentionPolicy(spec=tiny_spec)) + tokens = torch.tensor([[[1.0] * 16, [2.0] * 16]]) + cosine = torch.ones(2, 1, 8) + sine = torch.zeros(2, 1, 8) + output, retained = attention( + tokens, + cosine=cosine, + sine=sine, + layer_index=0, + frame_index=0, + kv_cache=cache, + initial_value=None, + ) + assert output.shape == tokens.shape + assert retained.shape == (1, 2, 1, 16) + + +def test_block_composes_adaptive_attention_control_and_mlp_paths() -> None: + """The block preserves frame-aware conditioning through both residual paths.""" + tiny_spec = replace( + WAYPOINT_1_5, + n_layers=4, + d_model=16, + n_heads=1, + n_kv_heads=1, + mlp_ratio=2, + local_window=2, + global_window=6, + global_pinned_dilation=2, + ) + block = _WaypointBlock(tiny_spec, has_control_fusion=True) + cache = WaypointKVCache(policy=WaypointAttentionPolicy(spec=tiny_spec)) + tokens = torch.randn(1, 2, 16) + conditioning = torch.randn(1, 1, 16) + control = torch.randn(1, 2, 16) + output, initial_value = block( + tokens, + conditioning=conditioning, + control=control, + cosine=torch.ones(2, 1, 8), + sine=torch.zeros(2, 1, 8), + layer_index=0, + frame_index=0, + kv_cache=cache, + initial_value=None, + ) + assert output.shape == tokens.shape + assert initial_value.shape == (1, 2, 1, 16) + + +def test_dit_runs_one_complete_latent_action() -> None: + """The native DiT keeps controller, RoPE, cache, and output layouts aligned.""" + tiny_spec = replace( + WAYPOINT_1_5, + n_layers=4, + d_model=16, + n_heads=1, + n_kv_heads=1, + mlp_ratio=2, + local_window=2, + global_window=6, + global_pinned_dilation=2, + ) + network = WaypointDiTConfig(spec=tiny_spec).setup() + for parameter in network.parameters(): + parameter.data.zero_() + cache = WaypointKVCache(policy=WaypointAttentionPolicy(spec=tiny_spec)) + latent = torch.randn(1, 1, 32, 32, 64) + output = network( + latent, + sigma=torch.ones(1), + frame_index=0, + kv_cache=cache, + button=torch.zeros(1, 1, 256), + mouse=torch.zeros(1, 1, 2), + scroll=torch.zeros(1, 1, 1), + ) + assert output.shape == latent.shape + assert torch.equal(output, torch.zeros_like(latent)) + + +def test_flashdreams_transformer_adapter_owns_action_layout_and_control() -> None: + """The framework adapter passes one public action into the native DiT.""" + tiny_spec = replace( + WAYPOINT_1_5, + n_layers=4, + d_model=16, + n_heads=1, + n_kv_heads=1, + mlp_ratio=2, + local_window=2, + global_window=6, + global_pinned_dilation=2, + ) + transformer = WaypointTransformerConfig( + network=WaypointDiTConfig(spec=tiny_spec), dtype=torch.float32 + ).setup() + for parameter in transformer.parameters(): + parameter.data.zero_() + cache = transformer.initialize_autoregressive_cache(batch_size=1) + cache.start(0) + noisy = torch.randn(1, 1, 32, 32, 64) + output = transformer.predict_flow( + noisy, + torch.tensor(1.0), + cache, + WaypointControl(buttons=frozenset({87})), + ) + assert output.shape == noisy.shape + assert torch.equal(output, torch.zeros_like(noisy)) + external = transformer.unpatchify_and_maybe_gather_cp(output) + assert external.shape == (1, 32, 1, 32, 64) + assert torch.equal(transformer.patchify_and_maybe_split_cp(external), output) + + +def test_pipeline_runs_fixed_euler_steps_for_one_controlled_action() -> None: + """The generic pipeline can drive Waypoint without a custom serving loop.""" + from flashdreams.infra.diffusion.model import DiffusionModelConfig + from flashdreams.infra.diffusion.scheduler import ( + FlowMatchEulerDiscreteSchedulerConfig, + ) + from flashdreams.infra.pipeline import StreamInferencePipelineConfig + + tiny_spec = replace( + WAYPOINT_1_5, + n_layers=1, + d_model=16, + n_heads=1, + n_kv_heads=1, + mlp_ratio=2, + ) + pipeline = StreamInferencePipelineConfig( + name="waypoint-test", + diffusion_model=DiffusionModelConfig( + transformer=WaypointTransformerConfig( + network=WaypointDiTConfig(spec=tiny_spec), dtype=torch.float32 + ), + scheduler=FlowMatchEulerDiscreteSchedulerConfig( + num_inference_steps=4, + num_train_timesteps=1, + fixed_timesteps=tiny_spec.scheduler_sigmas, + ), + ), + encoder=WaypointControlEncoderConfig(), + ).setup() + for parameter in pipeline.parameters(): + parameter.data.zero_() + cache = pipeline.initialize_cache(transformer_context={"batch_size": 1}) + output = pipeline.generate(0, cache, WaypointControl(buttons=frozenset({87}))) + pipeline.finalize(0, cache) + assert output.shape == (1, 32, 1, 32, 64) + + +def test_seed_initialization_advances_decoder_history_once() -> None: + """The seed latent establishes one matching transformer and decoder action.""" + pipeline = WaypointInferencePipeline.__new__(WaypointInferencePipeline) + torch.nn.Module.__init__(pipeline) + + seed_latent = torch.zeros(1, 1, 32, 32, 64) + decoder_cache = object() + decoder = WaypointTAEHVDecoder.__new__(WaypointTAEHVDecoder) + torch.nn.Module.__init__(decoder) + decoder.initialize_autoregressive_cache = Mock(return_value=decoder_cache) + decoder.forward = Mock() + + transformer_cache = WaypointTransformerCache(batch_size=1) + transformer = SimpleNamespace( + initialize_autoregressive_cache=Mock(return_value=transformer_cache), + predict_flow=Mock(), + ) + pipeline.seed_encoder = SimpleNamespace( + taehv=SimpleNamespace(encode=Mock(return_value=seed_latent)) + ) + pipeline.encoder = None + pipeline.decoder = decoder + pipeline.diffusion_model = SimpleNamespace( + device=torch.device("cpu"), transformer=transformer + ) + + cache = pipeline.initialize_cache( + seed_pixels=torch.zeros(1, 4, 3, 512, 1024), + ) + + transformer.predict_flow.assert_called_once() + decoder.forward.assert_called_once() + assert decoder.forward.call_args.args[0].shape == (1, 32, 1, 32, 64) + assert decoder.forward.call_args.kwargs == { + "autoregressive_index": 0, + "cache": decoder_cache, + } + assert cache.autoregressive_index == 0 + + +def test_attention_policy_selects_dense_local_and_pinned_global_history() -> None: + """Waypoint uses a short dense history or a dilated long-range history.""" + policy = WaypointAttentionPolicy() + assert not policy.is_global_layer(2) + assert policy.is_global_layer(3) + assert policy.visible_frame_indices(layer_index=2, frame_index=17) == tuple( + range(2, 18) + ) + assert policy.visible_frame_indices(layer_index=3, frame_index=130) == ( + 8, + 16, + 24, + 32, + 40, + 48, + 56, + 64, + 72, + 80, + 88, + 96, + 104, + 112, + 120, + 128, + 130, + ) + + +def test_kv_cache_replaces_provisional_frame_and_evicts_hidden_history() -> None: + """A cache view contains only the frames the checkpoint can attend to.""" + small_spec = replace( + WAYPOINT_1_5, + n_layers=4, + local_window=2, + global_window=6, + global_pinned_dilation=2, + ) + cache = WaypointKVCache(policy=WaypointAttentionPolicy(spec=small_spec)) + + for frame_index in range(7): + value = torch.full((1, 1, 2, 2), float(frame_index)) + global_view = cache.update( + layer_index=3, + frame_index=frame_index, + key=value, + value=-value, + ) + assert global_view.frame_indices == (2, 4, 6) + assert global_view.key[0, 0, ::2, 0].tolist() == [2.0, 4.0, 6.0] + + replacement = torch.full((1, 1, 2, 2), 99.0) + global_view = cache.update( + layer_index=3, + frame_index=6, + key=replacement, + value=-replacement, + ) + assert global_view.key[0, 0, ::2, 0].tolist() == [2.0, 4.0, 99.0] + + for frame_index in range(4): + value = torch.full((1, 1, 2, 2), float(frame_index)) + local_view = cache.update( + layer_index=0, + frame_index=frame_index, + key=value, + value=value, + ) + assert local_view.frame_indices == (2, 3) + assert local_view.key[0, 0, ::2, 0].tolist() == [2.0, 3.0] + + +def test_condition_and_control_fusion_primitives_use_silu_paths() -> None: + """Checkpoint projection names retain the observed conditioning semantics.""" + condition = _ConditionHead(2) + condition.bias_in.data.copy_(torch.tensor([0.5, -0.5])) + for projection in condition.cond_proj: + assert isinstance(projection, torch.nn.Linear) + projection.weight.data.copy_(torch.eye(2)) + values = condition(torch.tensor([[-1.0, 2.0]])) + expected = torch.nn.functional.silu(torch.tensor([[-0.5, 1.5]])) + assert all(torch.allclose(value, expected) for value in values) + + fusion = _ControlFusion(2) + fusion.fc1_x.weight.data.copy_(torch.eye(2)) + fusion.fc1_c.weight.data.copy_(torch.eye(2)) + fusion.fc2.weight.data.copy_(torch.eye(2)) + output = fusion(torch.tensor([[[1.0, 2.0]]]), torch.tensor([[[3.0, 4.0]]])) + assert torch.allclose( + output, torch.nn.functional.silu(torch.tensor([[[4.0, 6.0]]])) + ) diff --git a/integrations/waypoint/waypoint/__init__.py b/integrations/waypoint/waypoint/__init__.py new file mode 100644 index 000000000..590865688 --- /dev/null +++ b/integrations/waypoint/waypoint/__init__.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Waypoint 1.5 integration contracts.""" + +from waypoint.controls import ( + WaypointControl, + load_controls_from_file, + make_control_context, +) +from waypoint.encoder import WaypointControlEncoder, WaypointControlEncoderConfig +from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec + +__all__ = [ + "WAYPOINT_1_5", + "WaypointControl", + "WaypointControlEncoder", + "WaypointControlEncoderConfig", + "WaypointModelSpec", + "load_controls_from_file", + "make_control_context", +] diff --git a/integrations/waypoint/waypoint/checkpoint.py b/integrations/waypoint/waypoint/checkpoint.py new file mode 100644 index 000000000..9ffb162a1 --- /dev/null +++ b/integrations/waypoint/waypoint/checkpoint.py @@ -0,0 +1,290 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Waypoint checkpoint metadata and key-layout validation.""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping + +import torch +from torch import nn + +from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec + + +@dataclass(frozen=True, kw_only=True) +class CheckpointInventory: + """Small, CPU-only description of a safetensors checkpoint.""" + + tensor_count: int + """Number of tensors in the safetensors artifact.""" + dtypes: dict[str, int] + """Count of tensors by safetensors dtype label.""" + total_elements: int + """Total scalar element count across all tensors.""" + + +def expected_waypoint_1_5_checkpoint_shapes( + spec: WaypointModelSpec = WAYPOINT_1_5, +) -> dict[str, tuple[int, ...]]: + """Return raw checkpoint tensor shapes for the published 1.5 artifact. + + Args: + spec: Static checkpoint contract that defines the block layout. + + Returns: + Raw safetensors names mapped to their expected shapes. + """ + d_model = spec.d_model + hidden_dim = spec.mlp_ratio * d_model + kv_dim = spec.n_kv_heads * spec.head_dim + shapes = { + "ctrl_cfg.null_emb": (1, 1, d_model), + "ctrl_emb.mlp.fc1.weight": (hidden_dim, spec.n_buttons + 3), + "ctrl_emb.mlp.fc2.weight": (d_model, hidden_dim), + "denoise_step_emb.mlp.fc1.weight": (hidden_dim, 512), + "denoise_step_emb.mlp.fc2.weight": (d_model, hidden_dim), + "out_norm.fc.weight": (2 * d_model, d_model), + "patchify.weight": ( + d_model, + spec.channels, + spec.patch_height, + spec.patch_width, + ), + "unpatchify.bias": (spec.channels,), + "unpatchify.weight": ( + d_model, + spec.channels, + spec.patch_height, + spec.patch_width, + ), + } + for layer_index in range(spec.n_layers): + prefix = f"transformer.blocks.{layer_index}." + shapes.update( + { + prefix + "attn.k_proj.weight": (kv_dim, d_model), + prefix + "attn.out_proj.weight": (d_model, d_model), + prefix + "attn.q_proj.weight": (d_model, d_model), + prefix + "attn.v_lamb": (), + prefix + "attn.v_proj.weight": (kv_dim, d_model), + prefix + "attn_cond_head.bias_in": (d_model,), + **{ + prefix + f"attn_cond_head.cond_proj.{index}.weight": ( + d_model, + d_model, + ) + for index in range(3) + }, + prefix + "dit_mlp.fc1.weight": (hidden_dim, d_model), + prefix + "dit_mlp.fc2.weight": (d_model, hidden_dim), + prefix + "mlp_cond_head.bias_in": (d_model,), + **{ + prefix + f"mlp_cond_head.cond_proj.{index}.weight": ( + d_model, + d_model, + ) + for index in range(3) + }, + } + ) + if layer_index % spec.controller_conditioning_period == 0: + shapes.update( + { + prefix + "ctrl_mlpfusion.fc1_c.weight": (d_model, d_model), + prefix + "ctrl_mlpfusion.fc1_x.weight": (d_model, d_model), + prefix + "ctrl_mlpfusion.fc2.weight": (d_model, d_model), + } + ) + return shapes + + +def expected_waypoint_1_5_checkpoint_keys( + spec: WaypointModelSpec = WAYPOINT_1_5, +) -> frozenset[str]: + """Return raw checkpoint names for the published 1.5 artifact. + + Args: + spec: Static checkpoint contract that defines the block layout. + + Returns: + Every raw safetensors key expected from the target checkpoint. + """ + return frozenset(expected_waypoint_1_5_checkpoint_shapes(spec)) + + +def validate_waypoint_1_5_checkpoint_keys( + keys: set[str] | frozenset[str] | tuple[str, ...] | list[str], + *, + spec: WaypointModelSpec = WAYPOINT_1_5, +) -> None: + """Reject a checkpoint whose raw tensor-key layout differs from Waypoint 1.5. + + Args: + keys: Raw key names read from a safetensors checkpoint. + spec: Static checkpoint contract that defines the expected layout. + + Raises: + ValueError: The artifact contains missing or unexpected tensor keys. + """ + actual = set(keys) + expected = expected_waypoint_1_5_checkpoint_keys(spec) + missing = sorted(expected - actual) + extra = sorted(actual - expected) + if missing or extra: + raise ValueError( + "Waypoint 1.5 checkpoint key layout mismatch: " + f"missing={missing[:5]} ({len(missing)} total), " + f"extra={extra[:5]} ({len(extra)} total)" + ) + + +def validate_waypoint_1_5_checkpoint_shapes( + shapes: Mapping[str, tuple[int, ...]], + *, + spec: WaypointModelSpec = WAYPOINT_1_5, +) -> None: + """Reject a checkpoint whose tensor shapes differ from Waypoint 1.5. + + Args: + shapes: Raw tensor names mapped to shapes read from safetensors headers. + spec: Static checkpoint contract that defines expected tensor shapes. + + Raises: + ValueError: The artifact contains missing, unexpected, or mismatched tensors. + """ + expected = expected_waypoint_1_5_checkpoint_shapes(spec) + validate_waypoint_1_5_checkpoint_keys(tuple(shapes), spec=spec) + mismatched = sorted( + key + for key, expected_shape in expected.items() + if tuple(shapes[key]) != expected_shape + ) + if mismatched: + key = mismatched[0] + raise ValueError( + "Waypoint 1.5 checkpoint shape mismatch: " + f"{key} expected={expected[key]}, actual={tuple(shapes[key])}; " + f"{len(mismatched)} tensors differ" + ) + + +def load_waypoint_state_dict( + module: nn.Module, + state_dict: Mapping[str, torch.Tensor], + *, + spec: WaypointModelSpec = WAYPOINT_1_5, +) -> None: + """Validate and strictly load a raw Waypoint checkpoint into ``module``. + + Args: + module: Native module with the published raw state-dict namespace. + state_dict: Checkpoint tensors already materialized on the target device. + spec: Architecture contract used to validate the raw tensor layout. + + Raises: + ValueError: The checkpoint layout or tensor shapes differ from ``spec``. + RuntimeError: The module does not expose exactly the validated namespace. + """ + validate_waypoint_1_5_checkpoint_shapes( + {key: tuple(tensor.shape) for key, tensor in state_dict.items()}, spec=spec + ) + module.load_state_dict(state_dict, strict=True) + + +def inspect_safetensors_checkpoint(path: Path) -> CheckpointInventory: + """Read safetensors headers without materializing tensor payloads. + + Args: + path: Local safetensors artifact to inspect. + + Returns: + Tensor-count, dtype, and element-count metadata. + + Raises: + ImportError: An importable ``safetensors`` build is unavailable. + """ + try: + from safetensors import safe_open + except ImportError as error: # pragma: no cover - import environment dependent. + raise ImportError( + "Inspecting a Waypoint checkpoint requires an importable safetensors build. " + "Install the FlashDreams checkpoint dependencies first." + ) from error + + dtypes: Counter[str] = Counter() + total_elements = 0 + with safe_open(str(path), framework="pt", device="cpu") as checkpoint: + keys = list(checkpoint.keys()) + for key in keys: + tensor_slice = checkpoint.get_slice(key) + dtypes[str(tensor_slice.get_dtype())] += 1 + shape = tensor_slice.get_shape() + elements = 1 + for dimension in shape: + elements *= dimension + total_elements += elements + return CheckpointInventory( + tensor_count=len(keys), dtypes=dict(dtypes), total_elements=total_elements + ) + + +def validate_waypoint_1_5_checkpoint(path: Path) -> CheckpointInventory: + """Validate published Waypoint 1.5 safetensors metadata before native loading. + + This validates raw artifact identity only. The caller must still load it + into a checkpoint-compatible module. + + Args: + path: Local Waypoint 1.5 safetensors artifact. + + Returns: + Validated artifact metadata. + + Raises: + ImportError: An importable ``safetensors`` build is unavailable. + ValueError: The raw key layout or metadata differs from Waypoint 1.5. + """ + try: + from safetensors import safe_open + except ImportError as error: # pragma: no cover - import environment dependent. + raise ImportError( + "Validating a Waypoint checkpoint requires an importable safetensors build." + ) from error + + dtypes: Counter[str] = Counter() + total_elements = 0 + shapes: dict[str, tuple[int, ...]] = {} + with safe_open(str(path), framework="pt", device="cpu") as checkpoint: + for key in checkpoint.keys(): + tensor_slice = checkpoint.get_slice(key) + shape = tuple(tensor_slice.get_shape()) + shapes[key] = shape + dtypes[str(tensor_slice.get_dtype())] += 1 + elements = 1 + for dimension in shape: + elements *= dimension + total_elements += elements + validate_waypoint_1_5_checkpoint_shapes(shapes) + inventory = CheckpointInventory( + tensor_count=len(shapes), dtypes=dict(dtypes), total_elements=total_elements + ) + if inventory.dtypes != {"BF16": 393} or inventory.total_elements != 1_860_823_096: + raise ValueError(f"Waypoint 1.5 checkpoint metadata mismatch: {inventory}") + return inventory diff --git a/integrations/waypoint/waypoint/config.py b/integrations/waypoint/waypoint/config.py new file mode 100644 index 000000000..4ab7fcc82 --- /dev/null +++ b/integrations/waypoint/waypoint/config.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Static FlashDreams configuration for the published Waypoint 1.5 checkpoint.""" + +from __future__ import annotations + +from flashdreams.infra.diffusion.model import DiffusionModelConfig +from flashdreams.infra.pipeline import StreamInferencePipelineConfig +from flashdreams.infra.runner import RunnerConfig +from waypoint.decoder import WaypointTAEHVDecoderConfig +from waypoint.encoder import WaypointControlEncoderConfig +from waypoint.pipeline import WaypointInferencePipelineConfig +from waypoint.runner import WaypointRunnerConfig +from waypoint.scheduler import WaypointEulerSchedulerConfig +from waypoint.spec import WAYPOINT_1_5 +from waypoint.transformer import WaypointTransformerConfig + +WAYPOINT_1_5_CHECKPOINT = ( + "https://huggingface.co/Overworld/Waypoint-1.5-1B/resolve/main/model.safetensors" +) +"""Published raw Waypoint 1.5 DiT checkpoint.""" + +PIPELINE_WAYPOINT_1_5 = WaypointInferencePipelineConfig( + name="waypoint-1.5-1b", + diffusion_model=DiffusionModelConfig( + transformer=WaypointTransformerConfig(checkpoint_path=WAYPOINT_1_5_CHECKPOINT), + scheduler=WaypointEulerSchedulerConfig( + num_inference_steps=WAYPOINT_1_5.num_denoising_steps, + num_train_timesteps=1, + fixed_timesteps=WAYPOINT_1_5.scheduler_sigmas, + ), + context_noise=0, + ), + encoder=WaypointControlEncoderConfig(), + decoder=WaypointTAEHVDecoderConfig( + use_cuda_graph=False, + use_compile=False, + ), +) +"""Waypoint 1.5 DiT, fixed four-step Euler schedule, and matching TAEHV decoder.""" + +WAYPOINT_CONFIGS: dict[str, StreamInferencePipelineConfig] = { + PIPELINE_WAYPOINT_1_5.name: PIPELINE_WAYPOINT_1_5, +} +"""Waypoint pipeline variants keyed by stable slug.""" + +RUNNER_WAYPOINT_1_5 = WaypointRunnerConfig( + runner_name=PIPELINE_WAYPOINT_1_5.name, + description="Waypoint 1.5 1B controlled image-established rollout.", + pipeline=PIPELINE_WAYPOINT_1_5, +) +"""Default command-line runner for the published Waypoint 1.5 checkpoint.""" + +RUNNER_CONFIGS: dict[str, RunnerConfig] = { + cfg.runner_name: cfg for cfg in (RUNNER_WAYPOINT_1_5,) +} +"""Waypoint runners keyed by their ``flashdreams-run`` subcommand.""" diff --git a/integrations/waypoint/waypoint/controls.py b/integrations/waypoint/waypoint/controls.py new file mode 100644 index 000000000..78315b0d2 --- /dev/null +++ b/integrations/waypoint/waypoint/controls.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Waypoint's public per-action control representation.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, cast + +import torch +from torch import Tensor + +from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec + + +def load_controls_from_file(path: Path) -> tuple["WaypointControl", ...]: + """Load a versioned JSON sequence of per-action controller inputs. + + The file format is ``{"schema_version": 1, "actions": [...]}``. + Each action may specify ``buttons`` (integer array), ``mouse_dx``, + ``mouse_dy``, and ``scroll_wheel``; omitted values use the neutral control. + """ + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + raise FileNotFoundError(f"control timeline does not exist: {path}") from None + except json.JSONDecodeError as error: + raise ValueError( + f"control timeline is not valid JSON: {path}: {error}" + ) from error + if not isinstance(payload, dict): + raise ValueError("control timeline root must be an object") + unexpected = set(payload) - {"schema_version", "actions"} + if unexpected: + raise ValueError( + f"control timeline has unsupported fields: {sorted(unexpected)}" + ) + if payload.get("schema_version") != 1: + raise ValueError("control timeline schema_version must be 1") + actions = payload.get("actions") + if not isinstance(actions, list) or not actions: + raise ValueError("control timeline actions must be a non-empty array") + return tuple( + _parse_control_action(action, index) for index, action in enumerate(actions) + ) + + +def _parse_control_action(payload: Any, index: int) -> "WaypointControl": + if not isinstance(payload, dict): + raise ValueError(f"control action {index} must be an object") + allowed = {"buttons", "mouse_dx", "mouse_dy", "scroll_wheel"} + unexpected = set(payload) - allowed + if unexpected: + raise ValueError( + f"control action {index} has unsupported fields: {sorted(unexpected)}" + ) + buttons_value = payload.get("buttons", []) + if not isinstance(buttons_value, list) or any( + not isinstance(button, int) or isinstance(button, bool) + for button in buttons_value + ): + raise ValueError(f"control action {index} buttons must be an integer array") + buttons = cast(list[int], buttons_value) + invalid_buttons = sorted( + button for button in buttons if not 0 <= button < WAYPOINT_1_5.n_buttons + ) + if invalid_buttons: + raise ValueError( + f"control action {index} button IDs must be in " + f"[0, {WAYPOINT_1_5.n_buttons}), got {invalid_buttons}" + ) + mouse_dx = _finite_number(payload.get("mouse_dx", 0.0), "mouse_dx", index) + mouse_dy = _finite_number(payload.get("mouse_dy", 0.0), "mouse_dy", index) + scroll_wheel = payload.get("scroll_wheel", 0) + if ( + not isinstance(scroll_wheel, int) + or isinstance(scroll_wheel, bool) + or scroll_wheel not in (-1, 0, 1) + ): + raise ValueError( + f"control action {index} scroll_wheel must be one of -1, 0, or 1" + ) + return WaypointControl( + buttons=frozenset(buttons), + mouse_dx=mouse_dx, + mouse_dy=mouse_dy, + scroll_wheel=scroll_wheel, + ) + + +def _finite_number(value: Any, name: str, index: int) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"control action {index} {name} must be a finite number") + value = float(value) + if not math.isfinite(value): + raise ValueError(f"control action {index} {name} must be a finite number") + return value + + +@dataclass(frozen=True, kw_only=True) +class WaypointControl: + """Keyboard and mouse state that conditions one autoregressive action.""" + + buttons: frozenset[int] = field(default_factory=frozenset) + """Pressed button identifiers in the fixed 256-entry control vocabulary.""" + mouse_dx: float = 0.0 + """Mouse displacement along the horizontal axis.""" + mouse_dy: float = 0.0 + """Mouse displacement along the vertical axis.""" + scroll_wheel: int = 0 + """Ternary wheel direction: ``-1``, ``0``, or ``1``.""" + + +def make_control_context( + control: WaypointControl, + *, + frame_index: int, + batch_size: int = 1, + dtype: torch.dtype = torch.bfloat16, + device: torch.device | str | None = None, + spec: WaypointModelSpec = WAYPOINT_1_5, +) -> dict[str, Tensor]: + """Convert one public control event into model-ready per-action tensors. + + Args: + control: Keyboard and mouse event for one autoregressive action. + frame_index: Zero-based latent-frame index in the rollout. + batch_size: Number of identical control contexts to construct. + dtype: Floating-point dtype for continuous control tensors. + device: Target device; ``None`` keeps tensors on the current default device. + spec: Static checkpoint contract that defines control dimensions. + + Returns: + Pre-network button, mouse, scroll, and latent-frame timestamp tensors. + + Raises: + ValueError: A frame index, batch size, scroll value, or button ID is invalid. + """ + if frame_index < 0: + raise ValueError(f"frame_index must be non-negative, got {frame_index}") + if batch_size < 1: + raise ValueError(f"batch_size must be positive, got {batch_size}") + if control.scroll_wheel not in (-1, 0, 1): + raise ValueError( + f"scroll_wheel must be one of -1, 0, or 1, got {control.scroll_wheel}" + ) + + invalid_buttons = sorted( + button for button in control.buttons if not 0 <= button < spec.n_buttons + ) + if invalid_buttons: + raise ValueError( + f"button IDs must be in [0, {spec.n_buttons}), got {invalid_buttons}" + ) + + resolved_device = torch.device(device) if device is not None else None + buttons = torch.zeros( + (batch_size, 1, spec.n_buttons), dtype=dtype, device=resolved_device + ) + if control.buttons: + buttons[..., sorted(control.buttons)] = 1 + + mouse = ( + torch.tensor( + (control.mouse_dx, control.mouse_dy), dtype=dtype, device=resolved_device + ) + .view(1, 1, 2) + .expand(batch_size, -1, -1) + .clone() + ) + scroll = torch.full( + (batch_size, 1, 1), control.scroll_wheel, dtype=dtype, device=resolved_device + ) + frame_idx = torch.full( + (batch_size, 1), frame_index, dtype=torch.long, device=resolved_device + ) + frame_timestamp = torch.full( + (batch_size, 1), + frame_index * spec.frame_timestamp_stride, + dtype=torch.long, + device=resolved_device, + ) + return { + "button": buttons, + "mouse": mouse, + "scroll": scroll, + "frame_idx": frame_idx, + "frame_timestamp": frame_timestamp, + } diff --git a/integrations/waypoint/waypoint/decoder.py b/integrations/waypoint/waypoint/decoder.py new file mode 100644 index 000000000..cdfade5de --- /dev/null +++ b/integrations/waypoint/waypoint/decoder.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TAEHV layout adapter for the Waypoint video pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from torch import Tensor + +from flashdreams.recipes.taehv import Hy15TAEHVDecoder, Hy15TAEHVDecoderConfig +from flashdreams.recipes.taehv.impl import TAEHVCache + + +@dataclass(kw_only=True) +class WaypointTAEHVDecoderConfig(Hy15TAEHVDecoderConfig): + """Config for the matching TAEHV decoder behind the Waypoint pipeline.""" + + _target: type["WaypointTAEHVDecoder"] = field( + default_factory=lambda: WaypointTAEHVDecoder + ) + + +class WaypointTAEHVDecoder(Hy15TAEHVDecoder): + """Decode FlashDreams video latents using TAEHV's frame-first layout.""" + + def forward( + self, + input: Tensor, + autoregressive_index: int = 0, + cache: TAEHVCache | None = None, + ) -> Tensor: + """Decode a Waypoint action into ``[B, T, C, H, W]`` RGB frames. + + Args: + input: FlashDreams latent video in ``[B, C, T, H, W]`` layout. + autoregressive_index: Current latent action index. + cache: Long-lived causal TAEHV state. + + Returns: + Decoded RGB video in ``[B, T, C, H, W]`` layout and ``[-1, 1]`` range. + + Raises: + ValueError: The latent does not use FlashDreams' five-dimensional layout. + """ + if input.ndim != 5: + raise ValueError( + "Waypoint TAEHV input must have [B, C, T, H, W] layout, got " + f"{tuple(input.shape)}" + ) + return super().forward( + input.permute(0, 2, 1, 3, 4).contiguous(), + autoregressive_index=autoregressive_index, + cache=cache, + ) diff --git a/integrations/waypoint/waypoint/encoder.py b/integrations/waypoint/waypoint/encoder.py new file mode 100644 index 000000000..0c336fd37 --- /dev/null +++ b/integrations/waypoint/waypoint/encoder.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Per-action control passthrough for the Waypoint pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from flashdreams.infra.encoder import ( + EncoderConfig, + StreamingEncoder, + StreamingEncoderCache, +) +from waypoint.controls import WaypointControl + + +@dataclass(kw_only=True) +class WaypointControlEncoderConfig(EncoderConfig): + """Config for the Waypoint per-action control passthrough.""" + + _target: type["WaypointControlEncoder"] = field( + default_factory=lambda: WaypointControlEncoder + ) + + +class WaypointControlEncoder(StreamingEncoder[StreamingEncoderCache]): + """Carry one public Waypoint control event into the diffusion model. + + The learned control embedding lives inside the checkpoint-compatible DiT. + This streaming encoder exists only because FlashDreams reserves the + pipeline encoder slot for per-action user input. + """ + + def initialize_autoregressive_cache(self) -> StreamingEncoderCache: + """Return empty state because public controls have no temporal preprocessing.""" + return StreamingEncoderCache() + + def forward( + self, + input: WaypointControl, + autoregressive_index: int = 0, + cache: StreamingEncoderCache | None = None, + ) -> WaypointControl: + """Validate and return the control event unchanged. + + Args: + input: Public keyboard, mouse, and wheel input for this action. + autoregressive_index: Action index; accepted for the streaming interface. + cache: Empty passthrough cache; accepted for the streaming interface. + + Returns: + The same :class:`WaypointControl` instance. + + Raises: + TypeError: ``input`` is not a public Waypoint control event. + """ + del autoregressive_index, cache + if not isinstance(input, WaypointControl): + raise TypeError( + f"Waypoint control encoder requires WaypointControl, got {type(input)}" + ) + return input diff --git a/integrations/waypoint/waypoint/pipeline.py b/integrations/waypoint/waypoint/pipeline.py new file mode 100644 index 000000000..de549b8b8 --- /dev/null +++ b/integrations/waypoint/waypoint/pipeline.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Waypoint pipeline initialization for an image-established world state.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, cast + +import torch +from torch import Tensor + +from flashdreams.infra.pipeline import ( + StreamInferencePipeline, + StreamInferencePipelineCache, + StreamInferencePipelineConfig, +) +from flashdreams.recipes.taehv import Hy15TAEHVEncoder, Hy15TAEHVEncoderConfig +from waypoint.controls import WaypointControl +from waypoint.decoder import WaypointTAEHVDecoder +from waypoint.transformer.impl import WaypointTransformerCache + + +@dataclass(kw_only=True) +class WaypointInferencePipelineConfig(StreamInferencePipelineConfig): + """Configuration for an image-established Waypoint rollout.""" + + _target: type["WaypointInferencePipeline"] = field( + default_factory=lambda: WaypointInferencePipeline + ) + + seed_encoder: Hy15TAEHVEncoderConfig = field(default_factory=Hy15TAEHVEncoderConfig) + """Codec encoder that converts the initial displayed image into history.""" + + +class WaypointInferencePipeline(StreamInferencePipeline): + """Initialize persistent model state from the image that establishes the world.""" + + seed_encoder: Hy15TAEHVEncoder + + def __init__(self, config: WaypointInferencePipelineConfig) -> None: + super().__init__(config) + self.config: WaypointInferencePipelineConfig = config + self.seed_encoder = config.seed_encoder.setup() + + @torch.no_grad() + def initialize_cache( + self, + *, + seed_pixels: Tensor, + transformer_context: dict[str, Any] | None = None, + encoder_context: dict[str, Any] | None = None, + decoder_context: dict[str, Any] | None = None, + ) -> StreamInferencePipelineCache: + """Create a cache whose first historical action is the seed image. + + ``seed_pixels`` is four identical RGB frames in the codec's native + ``[0, 1]`` domain and ``[B, 4, 3, 512, 1024]`` layout. It is committed + as action zero; callers therefore begin generated actions at index one. + """ + if seed_pixels.ndim != 5 or tuple(seed_pixels.shape[1:]) != (4, 3, 512, 1024): + raise ValueError( + "seed_pixels must have [B, 4, 3, 512, 1024] layout, got " + f"{tuple(seed_pixels.shape)}" + ) + if seed_pixels.device != self.device: + raise ValueError( + f"seed_pixels is on {seed_pixels.device}, expected {self.device}" + ) + + batch_size = seed_pixels.shape[0] + transformer_context = dict(transformer_context or {}) + supplied_batch_size = transformer_context.setdefault("batch_size", batch_size) + if supplied_batch_size != batch_size: + raise ValueError( + "transformer_context batch_size must match seed_pixels batch size, got " + f"{supplied_batch_size} and {batch_size}" + ) + cache = super().initialize_cache( + transformer_context=transformer_context, + encoder_context=encoder_context, + decoder_context=decoder_context, + ) + + seed_latent = self.seed_encoder.taehv.encode(seed_pixels) + transformer = self.diffusion_model.transformer + transformer_cache = cast(WaypointTransformerCache, cache.transformer_cache) + transformer_cache.start(0) + transformer_cache.kv_cache.set_frozen(False) + try: + transformer.predict_flow( + seed_latent, + torch.zeros((), device=seed_latent.device, dtype=seed_latent.dtype), + transformer_cache, + WaypointControl(), + ) + finally: + transformer_cache.kv_cache.set_frozen(True) + + if not isinstance(self.decoder, WaypointTAEHVDecoder): + raise TypeError("Waypoint pipeline requires a WaypointTAEHVDecoder") + if cache.decoder_cache is None: + raise RuntimeError("Waypoint pipeline requires a decoder cache") + decoder_input = seed_latent.permute(0, 2, 1, 3, 4).contiguous() + self.decoder( + decoder_input, + autoregressive_index=0, + cache=cache.decoder_cache, + ) + cache.autoregressive_index = 0 + return cache diff --git a/integrations/waypoint/waypoint/runner.py b/integrations/waypoint/waypoint/runner.py new file mode 100644 index 000000000..86d2d9bcb --- /dev/null +++ b/integrations/waypoint/waypoint/runner.py @@ -0,0 +1,277 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Command-line runner for repeated-control Waypoint rollouts.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import cast + +import torch +import torch.nn.functional as F +from loguru import logger +from torch import Tensor + +from flashdreams.core.io.disk import default_flashdreams_cache_dir +from flashdreams.core.io.download import download_to_cache +from flashdreams.infra.config import derive_config +from flashdreams.infra.postprocess import VideoTensorLayout +from flashdreams.infra.runner import Runner, RunnerConfig +from flashdreams.infra.runner_io import runner_artifact_path, write_runner_stats +from flashdreams.runtime.video_output import Mp4VideoOutputTarget +from waypoint.controls import WaypointControl, load_controls_from_file +from waypoint.pipeline import WaypointInferencePipeline + +__all__ = [ + "EXAMPLE_DATA_BASE_URL", + "EXAMPLE_DATA_DIR_LOCAL", + "WaypointRunner", + "WaypointRunnerConfig", + "load_seed_display_frames", + "load_seed_pixels", +] + + +EXAMPLE_DATA_BASE_URL = ( + "https://raw.githubusercontent.com/Overworldai/Biome/14343a6/seeds" +) +"""HTTP directory containing the public example seed image.""" + +EXAMPLE_DATA_DIR_LOCAL = default_flashdreams_cache_dir() / "example_data/waypoint" +"""User-writable cache for the downloaded example seed image.""" + +_EXAMPLE_IMAGE_FILENAME = "crystal_desert_blade.jpg" + +_EXAMPLE_CONTROL_FILE = ( + Path(__file__).parents[3] + / "assets" + / "example_data" + / "waypoint" + / "example_controls.json" +) + + +def load_seed_display_frames(path: Path) -> Tensor: + """Load the four 720p RGB frames used to establish the displayed seed state.""" + import cv2 + import imageio.v3 as iio + + if not path.is_file(): + raise FileNotFoundError(f"seed image does not exist: {path}") + pixels = iio.imread(path) + if pixels.ndim != 3 or pixels.shape[-1] not in (3, 4): + raise ValueError(f"seed image must be RGB or RGBA, got {tuple(pixels.shape)}") + pixels = cv2.resize(pixels[..., :3], (1280, 720), interpolation=cv2.INTER_LINEAR) + return torch.from_numpy(pixels).permute(2, 0, 1).unsqueeze(0).repeat(4, 1, 1, 1) + + +def load_seed_pixels(path: Path, *, device: torch.device, dtype: torch.dtype) -> Tensor: + """Load one image into Waypoint's four-frame native codec input domain.""" + frames = load_seed_display_frames(path) + frames = frames.unsqueeze(0).to(device=device, dtype=dtype).div_(255) + return F.interpolate( + frames[0], size=(512, 1024), mode="bilinear", align_corners=False + ).unsqueeze(0) + + +@dataclass(kw_only=True) +class WaypointRunnerConfig(RunnerConfig): + """User-facing inputs for a repeated-control Waypoint rollout.""" + + _target: type["WaypointRunner"] = field(default_factory=lambda: WaypointRunner) + + seed_image: Path | None = None + """RGB image used to establish the initial world state.""" + + example_data: bool = False + """Download the example seed image and use its bundled controls.""" + + controls_file: Path | None = None + """JSON file containing one keyboard/mouse action per generated step.""" + + actions: int | None = None + """Action count, or a prefix length when :attr:`controls_file` is set.""" + + buttons: tuple[int, ...] = (32,) + """Model button-vocabulary IDs held for every generated action.""" + + mouse_dx: float = 0.10 + """Horizontal mouse displacement applied to every generated action.""" + + mouse_dy: float = 0.0 + """Vertical mouse displacement applied to every generated action.""" + + scroll: int = 0 + """Wheel direction applied to every generated action: ``-1``, ``0``, or ``1``.""" + + fps: int = 60 + """Presentation frame rate for the resulting MP4.""" + + output_height: int = 720 + """Presentation height for the resulting MP4.""" + + output_width: int = 1280 + """Presentation width for the resulting MP4.""" + + seed: int | None = None + """Optional noise-generator seed for deterministic rollout replay.""" + + postprocess_output_layout: VideoTensorLayout = "btchw" + """Fixed decoder output layout consumed by the runner output stream.""" + + +class WaypointRunner(Runner[WaypointRunnerConfig, WaypointInferencePipeline]): + """Generate a repeated-control video from a seed image.""" + + config: WaypointRunnerConfig + + def __init__(self, config: WaypointRunnerConfig) -> None: + if config.seed_image is None and not config.example_data: + raise ValueError("pass --seed-image or --example-data") + if config.seed is None: + seed = torch.seed() + else: + seed = config.seed + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + config = derive_config(config, pipeline={"diffusion_model": {"seed": seed}}) + super().__init__(config) + if config.seed is None: + logger.info(f"[{config.runner_name}] generated seed {seed}") + + def run(self) -> None: + """Prime the image-established state, then write the controlled rollout.""" + self._resolve_example_data() + config = self.config + seed_image = cast(Path, config.seed_image) + + seed_display_frames = load_seed_display_frames(seed_image) + seed_pixels = load_seed_pixels( + seed_image, + device=self.pipeline.device, + dtype=self.pipeline.diffusion_model.dtype, + ) + cache = self.pipeline.initialize_cache(seed_pixels=seed_pixels) + controls = self._resolve_controls() + + output_stream = self.create_video_output_stream(fps=config.fps) + output_target = Mp4VideoOutputTarget( + output_path=runner_artifact_path( + config.output_dir, config.runner_name, "mp4" + ), + fps=config.fps, + output_layout=output_stream.output_layout, + enabled=self.is_rank_zero, + ) + output_target.open() + started_at = time.perf_counter() + seed_video = seed_display_frames.unsqueeze(0).float().div_(127.5).sub_(1.0) + output_target.write( + output_stream.process( + seed_video, + autoregressive_index=0, + ) + ) + for autoregressive_index, control in enumerate(controls, start=1): + video = self.pipeline.generate(autoregressive_index, cache, control) + stats = self.pipeline.finalize(autoregressive_index, cache) + output_target.write( + output_stream.process( + self._resize_for_presentation(video), + autoregressive_index=autoregressive_index, + metrics=stats, + ) + ) + + tail = output_stream.finish() + if tail is not None: + output_target.write(tail) + artifacts = output_target.close() + if not artifacts: + return + artifact = artifacts[0] + output_path = Path(artifact.uri) + logger.info( + f"[{config.runner_name}] wrote {artifact.metadata['shape']} -> " + f"{output_path.resolve()} in {time.perf_counter() - started_at:.2f}s" + ) + stats_history = artifact.metadata["stats_history"] + if stats_history: + stats_path = write_runner_stats( + config.output_dir, config.runner_name, list(stats_history) + ) + logger.info( + f"[{config.runner_name}] wrote per-AR-step stats -> {stats_path.resolve()}" + ) + + def _resolve_example_data(self) -> None: + """Fill omitted inputs from the pinned example-data pair.""" + config = self.config + if not config.example_data: + return + if config.seed_image is None: + config.seed_image = self._fetch_example_image() + if config.controls_file is None: + config.controls_file = _EXAMPLE_CONTROL_FILE + + def _fetch_example_image(self) -> Path: + """Download the pinned example seed image once on rank zero.""" + if self.is_rank_zero: + download_to_cache( + f"{EXAMPLE_DATA_BASE_URL}/{_EXAMPLE_IMAGE_FILENAME}", + cache_dir=EXAMPLE_DATA_DIR_LOCAL, + filename=_EXAMPLE_IMAGE_FILENAME, + validator=load_seed_display_frames, + ) + if torch.distributed.is_initialized(): + torch.distributed.barrier() + return EXAMPLE_DATA_DIR_LOCAL / _EXAMPLE_IMAGE_FILENAME + + def _resolve_controls(self) -> tuple[WaypointControl, ...]: + """Choose a file-driven sequence or repeat one direct control event.""" + config = self.config + if config.controls_file is not None: + controls = load_controls_from_file(config.controls_file) + if config.actions is None: + return controls + if not 1 <= config.actions <= len(controls): + raise ValueError( + f"--actions must be in [1, {len(controls)}] for " + f"{config.controls_file}, got {config.actions}" + ) + return controls[: config.actions] + + actions = 4 if config.actions is None else config.actions + if actions < 1: + raise ValueError(f"--actions must be positive, got {actions}") + if config.scroll not in (-1, 0, 1): + raise ValueError(f"--scroll must be -1, 0, or 1, got {config.scroll}") + control = WaypointControl( + buttons=frozenset(config.buttons), + mouse_dx=config.mouse_dx, + mouse_dy=config.mouse_dy, + scroll_wheel=config.scroll, + ) + return (control,) * actions + + def _resize_for_presentation(self, video: Tensor) -> Tensor: + """Map the codec's internal 2:1 image plane into the displayed 16:9 video.""" + batch_size, frames, channels, height, width = video.shape + if (height, width) == (self.config.output_height, self.config.output_width): + return video + resized = F.interpolate( + video.reshape(batch_size * frames, channels, height, width), + size=(self.config.output_height, self.config.output_width), + mode="bilinear", + align_corners=False, + ) + return resized.reshape( + batch_size, + frames, + channels, + self.config.output_height, + self.config.output_width, + ) diff --git a/integrations/waypoint/waypoint/scheduler.py b/integrations/waypoint/waypoint/scheduler.py new file mode 100644 index 000000000..489a1859e --- /dev/null +++ b/integrations/waypoint/waypoint/scheduler.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Waypoint's checkpoint-specific rectified-flow Euler schedule.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field + +import torch +from torch import Tensor + +from flashdreams.infra.diffusion.scheduler.fm_euler import ( + FlowMatchEulerDiscreteScheduler, + FlowMatchEulerDiscreteSchedulerConfig, +) + + +@dataclass(kw_only=True) +class WaypointEulerSchedulerConfig(FlowMatchEulerDiscreteSchedulerConfig): + """Fixed four-step scheduler whose BF16 arithmetic matches Waypoint.""" + + _target: type["WaypointEulerScheduler"] = field( + default_factory=lambda: WaypointEulerScheduler + ) + + +class WaypointEulerScheduler(FlowMatchEulerDiscreteScheduler): + """Apply Euler steps from a BF16-quantized fixed sigma schedule. + + The checkpoint's inference path stores its schedule in the same BF16 dtype + as its latent. Computing adjacent differences after that quantization is + part of the learned four-step trajectory, not merely a storage choice. + """ + + def sample( + self, + initial_noise: Tensor, + predict_flow: Callable[[Tensor, Tensor], Tensor], + rng: torch.Generator | None = None, + ) -> Tensor: + """Denoise one action with checkpoint-equivalent Euler updates.""" + del rng + schedule = self.sigmas.to( + device=initial_noise.device, dtype=initial_noise.dtype + ) + noisy = initial_noise + for sigma, next_sigma in zip(schedule[:-1], schedule[1:], strict=True): + flow = predict_flow(noisy, sigma) + noisy = (noisy.float() + (next_sigma - sigma).float() * flow.float()).to( + dtype=initial_noise.dtype + ) + return noisy diff --git a/integrations/waypoint/waypoint/spec.py b/integrations/waypoint/waypoint/spec.py new file mode 100644 index 000000000..32eb2f6fd --- /dev/null +++ b/integrations/waypoint/waypoint/spec.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Waypoint 1.5 checkpoint configuration contract.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, kw_only=True) +class WaypointModelSpec: + """Architecture and rollout invariants for a published Waypoint checkpoint.""" + + model_id: str + """Published Hugging Face checkpoint identifier.""" + channels: int + """Number of channels in a single latent frame.""" + latent_height: int + """Pre-patchify latent-frame height.""" + latent_width: int + """Pre-patchify latent-frame width.""" + patch_height: int + """Spatial patch height.""" + patch_width: int + """Spatial patch width.""" + temporal_compression: int + """Presented RGB frames emitted per autoregressive latent frame.""" + inference_fps: int + """Presented RGB frame rate.""" + base_fps: int + """Timestamp base rate expected by the checkpoint.""" + n_layers: int + """Number of transformer blocks.""" + d_model: int + """Transformer channel width.""" + n_heads: int + """Number of query-attention heads.""" + n_kv_heads: int + """Number of key/value attention heads.""" + mlp_ratio: int + """Hidden-width multiplier of each transformer feed-forward network.""" + n_buttons: int + """Size of the multi-hot button-control vocabulary.""" + local_window: int + """Recent latent-frame capacity of local attention layers.""" + global_window: int + """Latent-frame horizon of global attention layers.""" + global_pinned_dilation: int + """Temporal stride of pinned history in global-attention layers.""" + global_attention_period: int + """Stride between global-attention transformer blocks.""" + global_attention_offset: int + """Offset used to select global-attention transformer blocks.""" + controller_conditioning_period: int + """Stride between transformer blocks with controller fusion weights.""" + value_residual: bool + """Whether attention values carry a residual stream across blocks.""" + gated_attention: bool + """Whether attention outputs use an additional learned gate.""" + noise_conditioning: str + """Published noise-conditioning family used by the checkpoint.""" + rope_theta: float + """Base frequency of the geometric temporal rotary spectrum.""" + rope_nyquist_fraction: float + """Fraction of the spatial Nyquist limit used by rotary features.""" + scheduler_sigmas: tuple[float, ...] + """Fixed rectified-flow Euler schedule, including the terminal sigma.""" + text_conditioning: bool + """Whether the checkpoint provides a text-conditioning input.""" + + @property + def head_dim(self) -> int: + """Return the channel dimension of an attention head.""" + assert self.d_model % self.n_heads == 0 + return self.d_model // self.n_heads + + @property + def tokens_per_latent_frame(self) -> int: + """Return the number of spatial tokens generated for one action.""" + assert self.latent_height % self.patch_height == 0 + assert self.latent_width % self.patch_width == 0 + return (self.latent_height // self.patch_height) * ( + self.latent_width // self.patch_width + ) + + @property + def patch_grid_height(self) -> int: + """Return the number of patch tokens along the latent-image height.""" + return self.latent_height // self.patch_height + + @property + def patch_grid_width(self) -> int: + """Return the number of patch tokens along the latent-image width.""" + return self.latent_width // self.patch_width + + @property + def frames_per_action(self) -> int: + """Return the number of presented RGB frames decoded from one latent frame.""" + return self.temporal_compression + + @property + def num_denoising_steps(self) -> int: + """Return the number of Euler velocity evaluations in the fixed schedule.""" + return len(self.scheduler_sigmas) - 1 + + @property + def latent_fps(self) -> int: + """Return the autoregressive latent-frame rate.""" + assert self.inference_fps % self.temporal_compression == 0 + return self.inference_fps // self.temporal_compression + + @property + def frame_timestamp_stride(self) -> int: + """Return the model timestamp increment per latent frame.""" + assert self.base_fps % self.latent_fps == 0 + return self.base_fps // self.latent_fps + + @property + def global_attention_layers(self) -> tuple[int, ...]: + """Return zero-indexed transformer layers that use global cache policy.""" + return tuple( + index + for index in range(self.n_layers) + if (index - self.global_attention_offset) % self.global_attention_period + == 0 + ) + + def latent_shape(self, batch_size: int = 1) -> tuple[int, int, int, int, int]: + """Return the pre-patchify DiT shape for one autoregressive action.""" + if batch_size < 1: + raise ValueError(f"batch_size must be positive, got {batch_size}") + return (batch_size, 1, self.channels, self.latent_height, self.latent_width) + + +WAYPOINT_1_5 = WaypointModelSpec( + model_id="Overworld/Waypoint-1.5-1B", + channels=32, + latent_height=32, + latent_width=64, + patch_height=2, + patch_width=2, + temporal_compression=4, + inference_fps=60, + base_fps=15, + n_layers=24, + d_model=2048, + n_heads=32, + n_kv_heads=16, + mlp_ratio=4, + n_buttons=256, + local_window=16, + global_window=128, + global_pinned_dilation=8, + global_attention_period=4, + global_attention_offset=-1, + controller_conditioning_period=3, + value_residual=True, + gated_attention=False, + noise_conditioning="wan", + rope_theta=10_000.0, + rope_nyquist_fraction=0.8, + scheduler_sigmas=(1.0, 0.9, 0.75, 0.3, 0.0), + text_conditioning=False, +) +"""Static contract for the published ``Overworld/Waypoint-1.5-1B`` checkpoint.""" diff --git a/integrations/waypoint/waypoint/transformer/__init__.py b/integrations/waypoint/waypoint/transformer/__init__.py new file mode 100644 index 000000000..1d6ae7900 --- /dev/null +++ b/integrations/waypoint/waypoint/transformer/__init__.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native Waypoint DiT topology and conditioning primitives.""" + +from waypoint.transformer.cache import ( + WaypointAttentionPolicy, + WaypointKVCache, + WaypointKVView, +) +from waypoint.transformer.impl import ( + WaypointTransformer, + WaypointTransformerCache, + WaypointTransformerConfig, +) +from waypoint.transformer.network import ( + WaypointDiT, + WaypointDiTConfig, + sinusoidal_noise_embedding, +) +from waypoint.transformer.norm import adaptive_gate, adaptive_rms_norm +from waypoint.transformer.rope import WaypointOrthoRoPEAngles, apply_waypoint_ortho_rope + +__all__ = [ + "WaypointDiT", + "WaypointDiTConfig", + "WaypointAttentionPolicy", + "WaypointKVCache", + "WaypointKVView", + "WaypointTransformer", + "WaypointTransformerCache", + "WaypointTransformerConfig", + "WaypointOrthoRoPEAngles", + "adaptive_gate", + "adaptive_rms_norm", + "apply_waypoint_ortho_rope", + "sinusoidal_noise_embedding", +] diff --git a/integrations/waypoint/waypoint/transformer/cache.py b/integrations/waypoint/waypoint/transformer/cache.py new file mode 100644 index 000000000..51eb87d3f --- /dev/null +++ b/integrations/waypoint/waypoint/transformer/cache.py @@ -0,0 +1,360 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Sparse causal KV-history policy for the Waypoint transformer.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import torch +from torch import Tensor +from torch.nn.attention.flex_attention import _DEFAULT_SPARSE_BLOCK_SIZE, BlockMask + +from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec + + +@dataclass(frozen=True, kw_only=True) +class WaypointAttentionPolicy: + """Choose the causal history visible to one Waypoint attention block. + + Most blocks need dense short-term state for motion continuity. Every + fourth block instead sees a stride-sampled long history, which supplies + long-range scene anchors without giving every block a 128-frame attention + cost. The selection is expressed in latent-frame indices so the KV store + and a future fused attention backend share one unambiguous contract. + """ + + spec: WaypointModelSpec = WAYPOINT_1_5 + """Checkpoint architecture whose attention schedule is represented.""" + + def is_global_layer(self, layer_index: int) -> bool: + """Return whether ``layer_index`` uses sparse global history. + + Args: + layer_index: Zero-indexed transformer block. + + Raises: + ValueError: The layer index is outside the checkpoint's block range. + """ + self._validate_layer_index(layer_index) + return ( + layer_index - self.spec.global_attention_offset + ) % self.spec.global_attention_period == 0 + + def visible_frame_indices( + self, *, layer_index: int, frame_index: int + ) -> tuple[int, ...]: + """Return causal latent frames visible while denoising ``frame_index``. + + A local block sees a dense trailing window. A global block sees only + pinned frames aligned to the configured dilation inside its longer + trailing horizon; a non-pinned current frame is added so every action + can attend to itself. + + Args: + layer_index: Zero-indexed transformer block. + frame_index: Zero-indexed latent-frame index of the current action. + + Returns: + Strictly increasing, causal latent-frame indices. + + Raises: + ValueError: ``frame_index`` is negative or the layer index is invalid. + """ + self._validate_layer_index(layer_index) + if frame_index < 0: + raise ValueError(f"frame_index must be non-negative, got {frame_index}") + + if not self.is_global_layer(layer_index): + first = max(0, frame_index - self.spec.local_window + 1) + return tuple(range(first, frame_index + 1)) + + first = max(0, frame_index - self.spec.global_window + 1) + dilation = self.spec.global_pinned_dilation + first_pinned = ((first + dilation - 1) // dilation) * dilation + pinned = tuple(range(first_pinned, frame_index + 1, dilation)) + if pinned and pinned[-1] == frame_index: + return pinned + return (*pinned, frame_index) + + def _validate_layer_index(self, layer_index: int) -> None: + if not 0 <= layer_index < self.spec.n_layers: + raise ValueError( + f"layer_index must be in [0, {self.spec.n_layers}), got {layer_index}" + ) + + +@dataclass(frozen=True, kw_only=True) +class WaypointKVView: + """The selected key/value frames for one causal attention evaluation.""" + + key: Tensor + """Concatenated keys in ``[B, H_kv, selected_frames * S, d_h]`` layout.""" + + value: Tensor + """Concatenated values with the same layout as ``key``.""" + + frame_indices: tuple[int, ...] + """Latent-frame origin of each contiguous ``S``-token segment.""" + + block_mask: BlockMask | None = None + """Optional fixed-cache block mask for checkpoint-equivalent attention.""" + + +@dataclass +class _FixedKVLayer: + """One lazy fixed-capacity K/V store used by the published runtime.""" + + kv: Tensor + written: Tensor + history_tokens: int + pinned_dilation: int + + +@dataclass(kw_only=True) +class WaypointKVCache: + """Keep exactly the model-visible KV history for each Waypoint block. + + A diffusion step may evaluate the same latent frame more than once. An + update for that frame replaces its provisional K/V tensors instead of + extending history; advancing to a later frame evicts entries the sparse + policy can no longer expose. The CUDA path uses a fixed-capacity store and + block mask; CPU uses a compact dictionary representation for the same + selection policy. + """ + + policy: WaypointAttentionPolicy = field(default_factory=WaypointAttentionPolicy) + """Frame-selection policy shared by all transformer blocks.""" + + use_fixed_attention: bool = False + """Use the checkpoint runtime's fixed-capacity masked-attention semantics.""" + + _layers: dict[int, dict[int, tuple[Tensor, Tensor]]] = field(default_factory=dict) + """Stored K/V tensors indexed by block, then latent-frame index.""" + + _latest_frame_indices: dict[int, int] = field(default_factory=dict) + """Latest frame written per block; equal writes replace a diffusion provisional.""" + + _fixed_layers: dict[int, _FixedKVLayer] = field(default_factory=dict) + _fixed_frozen: bool = True + + def update( + self, *, layer_index: int, frame_index: int, key: Tensor, value: Tensor + ) -> WaypointKVView: + """Store K/V for one action and return the model-visible sparse view. + + Args: + layer_index: Zero-indexed transformer block that produced the tensors. + frame_index: Zero-indexed latent action being evaluated. + key: RoPE-applied keys in ``[B, H_kv, S, d_h]`` layout. + value: Values in ``[B, H_kv, S, d_h]`` layout. + + Returns: + Causally selected keys and values concatenated along their token axis. + + Raises: + ValueError: Tensor layouts differ, a frame is negative, or a write + attempts to move a layer's history backwards. + """ + self._validate_kv(key, value) + latest = self._latest_frame_indices.get(layer_index) + if latest is not None and frame_index < latest: + raise ValueError( + f"layer {layer_index} cannot move from frame {latest} back to {frame_index}" + ) + self._latest_frame_indices[layer_index] = frame_index + + if self.use_fixed_attention and key.device.type == "cuda": + return self._fixed_view( + layer_index=layer_index, + frame_index=frame_index, + key=key, + value=value, + ) + + # CPU contract tests use this compact reference representation. The + # CUDA rollout takes the fixed-cache path above and never duplicates + # K/V tensors in a Python dictionary. + visible = self.policy.visible_frame_indices( + layer_index=layer_index, frame_index=frame_index + ) + layer = self._layers.setdefault(layer_index, {}) + layer[frame_index] = (key, value) + + retained = {index: layer[index] for index in visible if index in layer} + if frame_index not in retained: + raise RuntimeError("current frame was not retained by its attention policy") + self._layers[layer_index] = retained + return self._view(layer_index=layer_index, frame_indices=visible) + + def reset(self) -> None: + """Discard all retained K/V tensors while preserving the policy.""" + self._layers.clear() + self._latest_frame_indices.clear() + self._fixed_layers.clear() + self._fixed_frozen = True + + def set_frozen(self, frozen: bool) -> None: + """Choose whether writes update only the provisional current action.""" + self._fixed_frozen = frozen + + def _fixed_view( + self, + *, + layer_index: int, + frame_index: int, + key: Tensor, + value: Tensor, + ) -> WaypointKVView: + """Write one frame into the runtime-shaped fixed cache and mask it.""" + state = self._fixed_layers.get(layer_index) + tokens_per_frame = key.shape[2] + if state is None: + global_layer = self.policy.is_global_layer(layer_index) + frame_capacity = ( + self.policy.spec.global_window + if global_layer + else self.policy.spec.local_window + ) + pinned_dilation = ( + self.policy.spec.global_pinned_dilation if global_layer else 1 + ) + history_tokens = frame_capacity * tokens_per_frame + capacity = history_tokens + tokens_per_frame + state = _FixedKVLayer( + kv=torch.zeros( + 2, + key.shape[0], + key.shape[1], + capacity, + key.shape[-1], + device=key.device, + dtype=key.dtype, + ), + written=torch.cat( + ( + torch.zeros( + history_tokens, device=key.device, dtype=torch.bool + ), + torch.ones( + tokens_per_frame, device=key.device, dtype=torch.bool + ), + ) + ), + history_tokens=history_tokens, + pinned_dilation=pinned_dilation, + ) + self._fixed_layers[layer_index] = state + + bucket_count = state.history_tokens // tokens_per_frame // state.pinned_dilation + bucket = (frame_index + state.pinned_dilation - 1) // state.pinned_dilation + ring_start = (bucket % bucket_count) * tokens_per_frame + ring_slice = slice(ring_start, ring_start + tokens_per_frame) + tail_slice = slice( + state.history_tokens, state.history_tokens + tokens_per_frame + ) + current = torch.stack((key, value)) + state.kv[..., tail_slice, :].copy_(current) + + write_step = frame_index % state.pinned_dilation == 0 + visible = state.written.clone() + visible[ring_slice] &= not write_step + block_mask = _fixed_block_mask(tokens_per_frame, visible) + + if not self._fixed_frozen: + destination = ring_slice if write_step else tail_slice + state.kv[..., destination, :].copy_(current) + state.written[destination] = True + + key_full, value_full = state.kv.unbind(0) + return WaypointKVView( + key=key_full, + value=value_full, + frame_indices=(frame_index,), + block_mask=block_mask, + ) + + def _view( + self, *, layer_index: int, frame_indices: tuple[int, ...] + ) -> WaypointKVView: + layer = self._layers[layer_index] + selected = tuple(index for index in frame_indices if index in layer) + if not selected: + raise RuntimeError(f"layer {layer_index} has no K/V tensors to attend to") + keys, values = zip(*(layer[index] for index in selected), strict=True) + return WaypointKVView( + key=torch.cat(keys, dim=2), + value=torch.cat(values, dim=2), + frame_indices=selected, + ) + + @staticmethod + def _validate_kv(key: Tensor, value: Tensor) -> None: + if key.ndim != 4: + raise ValueError( + f"key must have [B, H_kv, S, d_h] layout, got {tuple(key.shape)}" + ) + if value.shape != key.shape: + raise ValueError( + "value must have the same [B, H_kv, S, d_h] shape as key, got " + f"{tuple(value.shape)} and {tuple(key.shape)}" + ) + + +def _fixed_block_mask(tokens_per_frame: int, written: Tensor) -> BlockMask: + """Create one visibility contract for compiled and eager attention. + + ``full_kv_*`` is the compiled kernel's efficient representation of active + blocks. ``mask_mod`` independently states the same per-token rule for + implementations that materialize dense scores. Keeping both prevents a + fixed-capacity cache from making unwritten zero slots visible outside the + compiled path. + """ + block_size = _DEFAULT_SPARSE_BLOCK_SIZE + if tokens_per_frame % block_size or written.numel() % block_size: + raise ValueError("Waypoint fixed attention requires block-aligned cache sizes") + query_blocks = tokens_per_frame // block_size + key_blocks = written.numel() // block_size + visible = written.view(key_blocks, block_size) + if not torch.equal(visible.any(-1), visible.all(-1)): + raise RuntimeError("Waypoint fixed cache visibility must be block aligned") + active = visible.all(-1).nonzero(as_tuple=False).flatten().to(torch.int32) + # ``BlockMask`` stores a key-block list for *each* query block. Every + # Waypoint query token sees the same cache slots, but omitting the query + # axis makes the mask structurally different and leaves the attention + # backend to interpret a malformed index tensor. + full_indices = torch.zeros( + 1, 1, query_blocks, key_blocks, dtype=torch.int32, device=written.device + ) + full_indices[..., : active.numel()] = active + full_count = torch.full( + (1, 1, query_blocks), + active.numel(), + dtype=torch.int32, + device=written.device, + ) + empty_count = torch.zeros_like(full_count) + empty_indices = torch.zeros( + 1, 1, query_blocks, key_blocks, dtype=torch.int32, device=written.device + ) + + def visible_slot( + batch_index: Tensor, + head_index: Tensor, + query_index: Tensor, + key_index: Tensor, + ) -> Tensor: + del batch_index, head_index, query_index + return written[key_index] + + return BlockMask.from_kv_blocks( + empty_count, + empty_indices, + full_count, + full_indices, + BLOCK_SIZE=block_size, + mask_mod=visible_slot, + seq_lengths=(tokens_per_frame, written.numel()), + compute_q_blocks=False, + ) diff --git a/integrations/waypoint/waypoint/transformer/impl.py b/integrations/waypoint/waypoint/transformer/impl.py new file mode 100644 index 000000000..838c39ac9 --- /dev/null +++ b/integrations/waypoint/waypoint/transformer/impl.py @@ -0,0 +1,255 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FlashDreams transformer adapter for one-action Waypoint denoising.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import torch +from torch import Tensor + +from flashdreams.infra.diffusion.transformer import ( + Transformer, + TransformerAutoregressiveCache, + TransformerConfig, +) +from waypoint.checkpoint import load_waypoint_state_dict +from waypoint.controls import WaypointControl, make_control_context +from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec +from waypoint.transformer.cache import WaypointKVCache +from waypoint.transformer.network import WaypointDiT, WaypointDiTConfig + + +@dataclass(kw_only=True) +class WaypointTransformerCache(TransformerAutoregressiveCache): + """Long-lived sparse history for an autoregressive Waypoint rollout.""" + + kv_cache: WaypointKVCache = field(default_factory=WaypointKVCache) + """Per-block sparse causal K/V history.""" + + batch_size: int + """Batch size fixed when the rollout starts.""" + + autoregressive_index: int = -1 + """Current latent action index; ``-1`` before the first ``start`` call.""" + + def start(self, autoregressive_index: int) -> None: + """Mark the action whose repeated denoise passes may replace K/V state. + + Args: + autoregressive_index: Zero-indexed latent action to generate. + + Raises: + ValueError: The action index is negative or skips rollout history. + """ + if autoregressive_index < 0: + raise ValueError( + f"autoregressive_index must be non-negative, got {autoregressive_index}" + ) + if self.autoregressive_index >= 0 and autoregressive_index not in ( + self.autoregressive_index, + self.autoregressive_index + 1, + ): + raise ValueError( + "Waypoint actions must be generated in order or re-evaluated in place; " + f"got {autoregressive_index} after {self.autoregressive_index}" + ) + self.autoregressive_index = autoregressive_index + self.kv_cache.set_frozen(True) + + +@dataclass(kw_only=True) +class WaypointTransformerConfig(TransformerConfig): + """Construction config for the one-action Waypoint transformer adapter.""" + + _target: type["WaypointTransformer"] = field( + default_factory=lambda: WaypointTransformer + ) + + network: WaypointDiTConfig = field(default_factory=WaypointDiTConfig) + """Native checkpoint-compatible Waypoint DiT.""" + + dtype: torch.dtype = torch.bfloat16 + """Network parameter and activation dtype.""" + + checkpoint_path: str | None = None + """Raw Waypoint safetensors path; ``None`` retains random initialization.""" + + +class WaypointTransformer(Transformer[WaypointTransformerCache]): + """Adapt Waypoint's internal patchifier to FlashDreams flow prediction. + + Waypoint owns its spatial patchifier, so FlashDreams sees a one-frame + latent action rather than pre-patchified tokens. This keeps the external + streaming layout conventional while preserving the checkpoint's learned + convolutional patch embedding exactly. + """ + + config: WaypointTransformerConfig + network: WaypointDiT + + def __init__(self, config: WaypointTransformerConfig) -> None: + super().__init__(config) + self.config = config + self.network = config.network.setup().to(dtype=config.dtype) + self.network.eval() + if config.checkpoint_path is not None: + from flashdreams.core.checkpoint.load import load_checkpoint + + state_dict = load_checkpoint(config.checkpoint_path) + if not isinstance(state_dict, dict): + raise RuntimeError( + "Waypoint checkpoint loader did not return a state dict" + ) + load_waypoint_state_dict(self.network, state_dict, spec=self.spec) + self._batch_size: int | None = None + + @property + def spec(self) -> WaypointModelSpec: + """Return the immutable architecture contract of the owned DiT.""" + return self.network.spec + + @property + def latent_shape(self) -> tuple[int, ...]: + """Return the internal one-action latent layout ``[B, 1, C, H, W]``.""" + if self._batch_size is None: + raise RuntimeError( + "latent_shape requires initialize_autoregressive_cache(batch_size=...)" + ) + return self.spec.latent_shape(self._batch_size) + + def initialize_autoregressive_cache( + self, *, batch_size: int, **context: Any + ) -> WaypointTransformerCache: + """Allocate sparse history for a fixed-batch Waypoint rollout. + + Args: + batch_size: Number of actions generated together. + context: Rejected when non-empty; Waypoint 1.5 has no one-shot context. + + Returns: + Empty K/V history ready for ``cache.start(0)``. + + Raises: + ValueError: The batch size is invalid or one-shot context was supplied. + """ + if batch_size < 1: + raise ValueError(f"batch_size must be positive, got {batch_size}") + if context: + raise ValueError( + "Waypoint 1.5 has no text/image context encoder; unexpected " + f"cache context keys: {sorted(context)}" + ) + self._batch_size = batch_size + return WaypointTransformerCache( + batch_size=batch_size, + kv_cache=WaypointKVCache(use_fixed_attention=True), + ) + + def patchify_and_maybe_split_cp(self, x: Any) -> Any: + """Convert external video latents to Waypoint's internal frame-first layout. + + Args: + x: Video latent in ``[B, C, T, H, W]`` layout or a non-tensor control. + + Returns: + Tensor latents in ``[B, T, C, H, W]`` layout; controls are unchanged. + """ + if not isinstance(x, Tensor): + return x + if x.ndim != 5: + raise ValueError(f"Waypoint latent must have rank 5, got {x.ndim}") + return x.permute(0, 2, 1, 3, 4).contiguous() + + def unpatchify_and_maybe_gather_cp(self, x: Tensor) -> Tensor: + """Convert Waypoint's internal frame-first latent layout to video layout. + + Args: + x: Internal latent in ``[B, T, C, H, W]`` layout. + + Returns: + Video latent in ``[B, C, T, H, W]`` layout. + """ + if x.ndim != 5: + raise ValueError(f"Waypoint latent must have rank 5, got {x.ndim}") + return x.permute(0, 2, 1, 3, 4).contiguous() + + def predict_flow( + self, + noisy_latent: Tensor, + timestep: Tensor, + cache: WaypointTransformerCache, + input: WaypointControl | None = None, + ) -> Tensor: + """Predict one flow field using the current action's control event. + + Args: + noisy_latent: Internal noisy action in ``[B, 1, C, H, W]`` layout. + timestep: Scalar sigma supplied by the fixed Euler scheduler. + cache: Per-rollout K/V history after ``cache.start``. + input: Optional public keyboard/mouse control for this action. + + Returns: + Flow prediction with the same layout as ``noisy_latent``. + + Raises: + RuntimeError: Called before an action has been selected with ``start``. + TypeError: ``input`` is not a :class:`WaypointControl`. + """ + if cache.autoregressive_index < 0: + raise RuntimeError("cache.start(autoregressive_index) must run first") + if input is not None and not isinstance(input, WaypointControl): + raise TypeError( + f"Waypoint input must be WaypointControl or None, got {type(input)}" + ) + sigma = ( + timestep.reshape(1) + .expand(cache.batch_size) + .to(device=noisy_latent.device, dtype=noisy_latent.dtype) + ) + if input is None: + return self.network( + noisy_latent, + sigma=sigma, + frame_index=cache.autoregressive_index, + kv_cache=cache.kv_cache, + ) + control = make_control_context( + input, + frame_index=cache.autoregressive_index, + batch_size=cache.batch_size, + dtype=noisy_latent.dtype, + device=noisy_latent.device, + spec=self.spec, + ) + return self.network( + noisy_latent, + sigma=sigma, + frame_index=cache.autoregressive_index, + kv_cache=cache.kv_cache, + button=control["button"], + mouse=control["mouse"], + scroll=control["scroll"], + ) + + def finalize_kv_cache( + self, + noisy_latent: Tensor, + timestep: Tensor, + cache: WaypointTransformerCache, + input: WaypointControl | None = None, + ) -> None: + """Commit the clean action's K/V entries after its Euler solve. + + Denoising repeatedly replaces a provisional current-frame slot. Only + this final sigma-zero pass may persist that slot into long-term history; + otherwise the next action has no visual world state to condition on. + """ + cache.kv_cache.set_frozen(False) + try: + _ = self.predict_flow(noisy_latent, timestep, cache, input) + finally: + cache.kv_cache.set_frozen(True) diff --git a/integrations/waypoint/waypoint/transformer/network.py b/integrations/waypoint/waypoint/transformer/network.py new file mode 100644 index 000000000..51798ca84 --- /dev/null +++ b/integrations/waypoint/waypoint/transformer/network.py @@ -0,0 +1,719 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checkpoint-compatible Waypoint DiT topology and local tensor operations.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import cast + +import torch +import torch.nn.functional as F +from torch import Tensor, nn +from torch.nn.attention.flex_attention import BlockMask + +from flashdreams.infra.config import InstantiateConfig +from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec +from waypoint.transformer.cache import WaypointKVCache +from waypoint.transformer.norm import adaptive_gate, adaptive_rms_norm +from waypoint.transformer.rope import WaypointOrthoRoPEAngles, apply_waypoint_ortho_rope + + +# Compile the pure fixed-attention operation so FlexAttention receives the +# block-index representation required by the fixed cache. +@torch.compile(dynamic=False) +def _compiled_fixed_attention( + query: Tensor, + key: Tensor, + value: Tensor, + block_mask: BlockMask, +) -> Tensor: + """Run fixed-cache GQA through FlexAttention's compiled kernel path.""" + from torch.nn.attention.flex_attention import flex_attention + + return flex_attention( + query, + key, + value, + block_mask=block_mask, + enable_gqa=True, + ) + + +def sinusoidal_noise_embedding( + dim: int, + sigma: Tensor, + *, + frequencies: Tensor | None = None, +) -> Tensor: + """Embed continuous noise levels with Waypoint's Wan-style Fourier basis. + + Args: + dim: Even output feature width. + sigma: Noise levels with arbitrary leading shape. + frequencies: Optional precomputed positive Fourier frequencies. Supplying + the conditioner's buffer preserves its checkpoint-runtime numeric + behavior after a precision move. + + Returns: + Cosine/sine features with shape ``[*sigma.shape, dim]``. + + Raises: + ValueError: ``dim`` is not even. + """ + if dim % 2: + raise ValueError(f"noise embedding width must be even, got {dim}") + half = dim // 2 + sigma = sigma.to(dtype=torch.float32) + if frequencies is None: + frequencies = torch.logspace( + 0, + -1, + steps=half, + base=10_000.0, + device=sigma.device, + dtype=torch.float32, + ) + elif frequencies.ndim != 1 or frequencies.numel() != half: + raise ValueError( + f"expected {half} one-dimensional Fourier frequencies, got " + f"{tuple(frequencies.shape)}" + ) + frequencies = frequencies.to(device=sigma.device, dtype=torch.float32) + angles = sigma[..., None] * 1_000 * frequencies + return torch.cat((torch.sin(angles), torch.cos(angles)), dim=-1) * (2**0.5) + + +class _TwoLayerMLP(nn.Module): + """Bias-free two-projection MLP with raw checkpoint names.""" + + def __init__( + self, in_features: int, hidden_features: int, out_features: int + ) -> None: + super().__init__() + self.fc1 = nn.Linear(in_features, hidden_features, bias=False) + self.fc2 = nn.Linear(hidden_features, out_features, bias=False) + + def forward(self, input: Tensor) -> Tensor: + """Apply the SiLU MLP used by the public control and noise conditioners.""" + return self.fc2(F.silu(self.fc1(input))) + + +class _NullControl(nn.Module): + """Classifier-free null-control embedding container.""" + + def __init__(self, d_model: int) -> None: + super().__init__() + self.null_emb = nn.Parameter(torch.empty(1, 1, d_model)) + + +class _ConditionHead(nn.Module): + """Three-projection adaptive-conditioning parameter group.""" + + def __init__(self, d_model: int) -> None: + super().__init__() + self.bias_in = nn.Parameter(torch.empty(d_model)) + self.cond_proj = nn.ModuleList( + [nn.Linear(d_model, d_model, bias=False) for _ in range(3)] + ) + + def forward(self, conditioning: Tensor) -> tuple[Tensor, Tensor, Tensor]: + """Project one conditioner into an AdaLN scale, shift, and gate tuple.""" + features = F.silu(conditioning + self.bias_in) + return tuple(projection(features) for projection in self.cond_proj) # type: ignore[return-value] + + +class _WaypointAttention(nn.Module): + """Grouped-query attention projections and value-residual coefficient.""" + + def __init__(self, spec: WaypointModelSpec) -> None: + super().__init__() + kv_dim = spec.n_kv_heads * spec.head_dim + self.n_heads = spec.n_heads + self.n_kv_heads = spec.n_kv_heads + self.head_dim = spec.head_dim + self.k_proj = nn.Linear(spec.d_model, kv_dim, bias=False) + self.out_proj = nn.Linear(spec.d_model, spec.d_model, bias=False) + self.q_proj = nn.Linear(spec.d_model, spec.d_model, bias=False) + self.v_lamb = nn.Parameter(torch.empty(())) + self.v_proj = nn.Linear(spec.d_model, kv_dim, bias=False) + + def project_qkv(self, tokens: Tensor) -> tuple[Tensor, Tensor, Tensor]: + """Project tokens into RMS-normalized Q/K and unnormalized grouped V. + + Args: + tokens: Hidden states shaped ``[batch, tokens, d_model]``. + + Returns: + Query, key, and value tensors shaped ``[batch, tokens, heads, head_dim]``. + + Raises: + ValueError: ``tokens`` does not have Waypoint's model width. + """ + if tokens.ndim != 3 or tokens.shape[-1] != self.q_proj.in_features: + raise ValueError( + "Waypoint attention requires [batch, tokens, d_model] input; " + f"got {tuple(tokens.shape)}" + ) + batch_size, token_count, _ = tokens.shape + query = self.q_proj(tokens).reshape( + batch_size, token_count, self.n_heads, self.head_dim + ) + key = self.k_proj(tokens).reshape( + batch_size, token_count, self.n_kv_heads, self.head_dim + ) + value = self.v_proj(tokens).reshape( + batch_size, token_count, self.n_kv_heads, self.head_dim + ) + return ( + F.rms_norm(query, (self.head_dim,), weight=None, eps=None), + F.rms_norm(key, (self.head_dim,), weight=None, eps=None), + value, + ) + + def blend_value_residual( + self, current_value: Tensor, initial_value: Tensor | None + ) -> tuple[Tensor, Tensor]: + """Blend current values with the first block's value stream. + + Args: + current_value: Values projected by the current transformer block. + initial_value: First-block values; ``None`` on the first block. + + Returns: + Values for attention and the value stream retained for later blocks. + + Raises: + ValueError: A retained value stream does not match the current layout. + """ + if initial_value is None: + return current_value, current_value + if initial_value.shape != current_value.shape: + raise ValueError( + "Waypoint value residual requires matching current and initial " + f"value shapes, got {tuple(current_value.shape)} and " + f"{tuple(initial_value.shape)}" + ) + return torch.lerp(current_value, initial_value, self.v_lamb), initial_value + + def forward( + self, + tokens: Tensor, + *, + cosine: Tensor, + sine: Tensor, + layer_index: int, + frame_index: int, + kv_cache: WaypointKVCache, + initial_value: Tensor | None, + ) -> tuple[Tensor, Tensor]: + """Attend over the sparse causal history of one Waypoint block. + + Args: + tokens: Normalized hidden states shaped ``[B, S, D]``. + cosine: Current-frame packed RoPE cosine factors. + sine: Current-frame packed RoPE sine factors. + layer_index: Zero-indexed transformer block owning the K/V history. + frame_index: Zero-indexed latent action being denoised. + kv_cache: Per-rollout sparse K/V history. + initial_value: Value tensor retained from the first transformer block. + + Returns: + Attention residual in ``[B, S, D]`` layout and the value stream to + retain for subsequent blocks. + """ + query, key, current_value = self.project_qkv(tokens) + query = apply_waypoint_ortho_rope(query, cosine, sine) + key = apply_waypoint_ortho_rope(key, cosine, sine) + value, retained_value = self.blend_value_residual(current_value, initial_value) + + view = kv_cache.update( + layer_index=layer_index, + frame_index=frame_index, + key=key.transpose(1, 2), + value=value.transpose(1, 2), + ) + query = query.transpose(1, 2) + if view.block_mask is None: + attention = F.scaled_dot_product_attention( + query, + view.key, + view.value, + enable_gqa=True, + ) + else: + attention = _compiled_fixed_attention( + query, view.key, view.value, view.block_mask + ) + batch_size, _, token_count, _ = attention.shape + attention = attention.transpose(1, 2).reshape( + batch_size, token_count, self.out_proj.in_features + ) + return self.out_proj(attention), retained_value + + +class _ControlFusion(nn.Module): + """Controller-fusion projections for every third Waypoint block.""" + + def __init__(self, d_model: int) -> None: + super().__init__() + self.fc1_c = nn.Linear(d_model, d_model, bias=False) + self.fc1_x = nn.Linear(d_model, d_model, bias=False) + self.fc2 = nn.Linear(d_model, d_model, bias=False) + + def forward(self, tokens: Tensor, control: Tensor) -> Tensor: + """Fuse controller features through the published residual MLP path.""" + return self.fc2(F.silu(self.fc1_x(tokens) + self.fc1_c(control))) + + +class _WaypointBlock(nn.Module): + """One checkpoint-compatible Waypoint transformer block.""" + + def __init__(self, spec: WaypointModelSpec, *, has_control_fusion: bool) -> None: + super().__init__() + self.attn = _WaypointAttention(spec) + self.attn_cond_head = _ConditionHead(spec.d_model) + if has_control_fusion: + self.ctrl_mlpfusion = _ControlFusion(spec.d_model) + self.dit_mlp = _TwoLayerMLP( + spec.d_model, + spec.d_model * spec.mlp_ratio, + spec.d_model, + ) + self.mlp_cond_head = _ConditionHead(spec.d_model) + + def forward( + self, + tokens: Tensor, + *, + conditioning: Tensor, + control: Tensor | None, + cosine: Tensor, + sine: Tensor, + layer_index: int, + frame_index: int, + kv_cache: WaypointKVCache, + initial_value: Tensor | None, + ) -> tuple[Tensor, Tensor]: + """Apply one conditionally gated Waypoint transformer block. + + Args: + tokens: Hidden states in ``[B, T * S, D]`` layout. + conditioning: Noise/control features in ``[B, T, D]`` layout. + control: Per-token controller features; ``None`` skips periodic fusion. + cosine: Current-frame packed RoPE cosine factors. + sine: Current-frame packed RoPE sine factors. + layer_index: Zero-indexed block index for sparse-history selection. + frame_index: Zero-indexed latent action being denoised. + kv_cache: Per-rollout sparse attention K/V history. + initial_value: First-block value stream; ``None`` for block zero. + + Returns: + Updated hidden states and the first-block value stream. + + Raises: + ValueError: Control tokens do not match the hidden-state layout. + """ + attn_scale, attn_bias, attn_gate = self.attn_cond_head(conditioning) + attn_residual, initial_value = self.attn( + adaptive_rms_norm(tokens, attn_scale, attn_bias), + cosine=cosine, + sine=sine, + layer_index=layer_index, + frame_index=frame_index, + kv_cache=kv_cache, + initial_value=initial_value, + ) + tokens = tokens + adaptive_gate(attn_residual, attn_gate) + + if control is not None and hasattr(self, "ctrl_mlpfusion"): + if control.shape != tokens.shape: + raise ValueError( + "Waypoint control tokens must match hidden states, got " + f"control={tuple(control.shape)}, tokens={tuple(tokens.shape)}" + ) + channels = tokens.shape[-1] + fused_tokens = F.rms_norm(tokens, (channels,), weight=None, eps=None) + fused_control = F.rms_norm(control, (channels,), weight=None, eps=None) + tokens = tokens + self.ctrl_mlpfusion(fused_tokens, fused_control) + + mlp_scale, mlp_bias, mlp_gate = self.mlp_cond_head(conditioning) + mlp_residual = self.dit_mlp(adaptive_rms_norm(tokens, mlp_scale, mlp_bias)) + return tokens + adaptive_gate(mlp_residual, mlp_gate), initial_value + + +class _WaypointBlockStack(nn.Module): + """Ordered Waypoint transformer blocks with periodic control fusion.""" + + def __init__(self, spec: WaypointModelSpec) -> None: + super().__init__() + self.blocks = nn.ModuleList( + [ + _WaypointBlock( + spec, + has_control_fusion=( + layer_index % spec.controller_conditioning_period == 0 + ), + ) + for layer_index in range(spec.n_layers) + ] + ) + + +class _ControlEmbedder(nn.Module): + """Waypoint controller embedding MLP.""" + + def __init__(self, spec: WaypointModelSpec) -> None: + super().__init__() + self.mlp = _TwoLayerMLP( + spec.n_buttons + 3, + spec.d_model * spec.mlp_ratio, + spec.d_model, + ) + + +class _NoiseEmbedder(nn.Module): + """Waypoint diffusion-noise embedding MLP.""" + + freq: Tensor + mlp: _TwoLayerMLP + + def __init__(self, spec: WaypointModelSpec) -> None: + super().__init__() + self.register_buffer( + "freq", + torch.logspace(0, -1, steps=256, base=10_000.0, dtype=torch.float32), + persistent=False, + ) + self.mlp = _TwoLayerMLP(512, spec.d_model * spec.mlp_ratio, spec.d_model) + + def _apply(self, fn): + """Move the conditioner while retaining FP32 Fourier-MLP arithmetic.""" + + def keep_dtype(tensor: Tensor) -> Tensor: + return fn(tensor).to(dtype=tensor.dtype) + + return super()._apply(keep_dtype) + + +class _OutputNorm(nn.Module): + """Checkpoint-compatible final adaptive-normalization projection.""" + + def __init__(self, d_model: int) -> None: + super().__init__() + self.fc = nn.Linear(d_model, 2 * d_model, bias=False) + + +@dataclass(kw_only=True) +class WaypointDiTConfig(InstantiateConfig): + """Static construction config for the published Waypoint 1.5 DiT.""" + + _target: type["WaypointDiT"] = field(default_factory=lambda: WaypointDiT) + + spec: WaypointModelSpec = WAYPOINT_1_5 + """Immutable architecture contract for the target checkpoint.""" + + +class WaypointDiT(nn.Module): + """Denoise one controllable autoregressive Waypoint latent frame. + + Waypoint generates a four-frame video chunk from each 32-channel latent + frame, then feeds that latent history back into the next action. Its control + embedding, grouped-query transformer widths, and fixed checkpoint namespace + are coupled to that rollout contract, so it cannot be represented by a + generic image-to-video DiT without changing the learned function. + """ + + ctrl_cfg: _NullControl + ctrl_emb: _ControlEmbedder + denoise_step_emb: _NoiseEmbedder + out_norm: _OutputNorm + patchify: nn.Conv2d + transformer: _WaypointBlockStack + rope_angles: WaypointOrthoRoPEAngles + unpatchify: nn.ConvTranspose2d + + def __init__(self, config: WaypointDiTConfig) -> None: + super().__init__() + self.config = config + self.spec = config.spec + self.ctrl_cfg = _NullControl(self.spec.d_model) + self.ctrl_emb = _ControlEmbedder(self.spec) + self.denoise_step_emb = _NoiseEmbedder(self.spec) + self.out_norm = _OutputNorm(self.spec.d_model) + self.patchify = nn.Conv2d( + self.spec.channels, + self.spec.d_model, + kernel_size=(self.spec.patch_height, self.spec.patch_width), + stride=(self.spec.patch_height, self.spec.patch_width), + bias=False, + ) + self.transformer = _WaypointBlockStack(self.spec) + self.rope_angles = WaypointOrthoRoPEAngles(self.spec) + self.unpatchify = nn.ConvTranspose2d( + self.spec.d_model, + self.spec.channels, + kernel_size=(self.spec.patch_height, self.spec.patch_width), + stride=(self.spec.patch_height, self.spec.patch_width), + bias=True, + ) + + def patchify_latent(self, latent: Tensor) -> Tensor: + """Patchify one or more Waypoint latent frames into DiT tokens. + + Args: + latent: Raw latent video with shape ``[B, T, C, H, W]``. + + Returns: + Tokens ordered by ``(T, H, W)`` with shape ``[B, T * L, D]``. + + Raises: + ValueError: The latent shape differs from the published contract. + """ + if latent.ndim != 5: + raise ValueError( + "Waypoint latent must have shape [B, T, C, H, W], " + f"got {tuple(latent.shape)}" + ) + batch_size, frames, channels, height, width = latent.shape + expected = (self.spec.channels, self.spec.latent_height, self.spec.latent_width) + if (channels, height, width) != expected: + raise ValueError( + "Waypoint latent C/H/W mismatch: " + f"expected {expected}, got {(channels, height, width)}" + ) + x = self.patchify(latent.reshape(batch_size * frames, channels, height, width)) + patch_height, patch_width = x.shape[-2:] + x = x.reshape(batch_size, frames, self.spec.d_model, patch_height, patch_width) + return x.permute(0, 1, 3, 4, 2).reshape(batch_size, -1, self.spec.d_model) + + def unpatchify_tokens(self, tokens: Tensor, *, frames: int = 1) -> Tensor: + """Unpatchify DiT tokens into raw Waypoint latents. + + Args: + tokens: Tokens with shape ``[B, T * L, D]``. + frames: Latent-frame count ``T`` represented by ``tokens``. + + Returns: + Latent video with shape ``[B, T, C, H, W]``. + + Raises: + ValueError: Token rank, width, count, or frame count is invalid. + """ + if tokens.ndim != 3: + raise ValueError(f"Waypoint tokens must have rank 3, got {tokens.ndim}") + if frames < 1: + raise ValueError(f"frames must be positive, got {frames}") + batch_size, token_count, width = tokens.shape + if width != self.spec.d_model: + raise ValueError( + f"Waypoint token width must be {self.spec.d_model}, got {width}" + ) + tokens_per_frame = self.spec.tokens_per_latent_frame + if token_count != frames * tokens_per_frame: + raise ValueError( + f"Waypoint token count must be frames * {tokens_per_frame}, " + f"got frames={frames}, token_count={token_count}" + ) + patch_height = self.spec.latent_height // self.spec.patch_height + patch_width = self.spec.latent_width // self.spec.patch_width + # The published tensor is laid out as a convolution kernel, but its + # inference operator emits the ``C * patch_h * patch_w`` pixel vector + # independently for every token. Keep the raw kernel layout in the + # module state dict, then expose that learned operator directly here. + output_weight = self.unpatchify.weight.permute(1, 2, 3, 0).reshape( + -1, self.spec.d_model + ) + output_bias = cast(Tensor, self.unpatchify.bias) + output_bias = ( + output_bias[:, None, None] + .expand(-1, self.spec.patch_height, self.spec.patch_width) + .reshape(-1) + ) + x = F.linear(tokens, output_weight, output_bias) + x = x.reshape( + batch_size, + frames, + patch_height, + patch_width, + self.spec.channels, + self.spec.patch_height, + self.spec.patch_width, + ) + return x.permute(0, 1, 4, 2, 5, 3, 6).reshape( + batch_size, + frames, + self.spec.channels, + self.spec.latent_height, + self.spec.latent_width, + ) + + def embed_control(self, *, button: Tensor, mouse: Tensor, scroll: Tensor) -> Tensor: + """Embed one controller state per autoregressive latent frame. + + Args: + button: Multi-hot button tensor with shape ``[B, T, 256]``. + mouse: Pointer deltas with shape ``[B, T, 2]``. + scroll: Wheel direction with shape ``[B, T, 1]``. + + Returns: + Controller embeddings with shape ``[B, T, D]``. + + Raises: + ValueError: Controller tensors do not share the published shapes. + """ + expected_prefix = button.shape[:-1] + if ( + button.ndim != 3 + or button.shape[-1] != self.spec.n_buttons + or mouse.shape != expected_prefix + (2,) + or scroll.shape != expected_prefix + (1,) + ): + raise ValueError( + "Waypoint controls require button=[B, T, 256], mouse=[B, T, 2], " + f"scroll=[B, T, 1]; got button={tuple(button.shape)}, " + f"mouse={tuple(mouse.shape)}, scroll={tuple(scroll.shape)}" + ) + controls = torch.cat((mouse, button, scroll), dim=-1) + return self.ctrl_emb.mlp(controls) + + def embed_noise(self, sigma: Tensor) -> Tensor: + """Embed continuous rectified-flow noise levels. + + Args: + sigma: Scalar or batch-shaped noise level. + + Returns: + Noise embedding with shape ``[*sigma.shape, D]``. + """ + features = sinusoidal_noise_embedding( + 512, + sigma, + frequencies=self.denoise_step_emb.freq, + ) + return self.denoise_step_emb.mlp(features).to(dtype=self.patchify.weight.dtype) + + def forward( + self, + latent: Tensor, + *, + sigma: Tensor, + frame_index: int, + kv_cache: WaypointKVCache, + button: Tensor | None = None, + mouse: Tensor | None = None, + scroll: Tensor | None = None, + ) -> Tensor: + """Predict rectified-flow velocity for one autoregressive latent action. + + Args: + latent: Noisy latent video in ``[B, 1, C, H, W]`` layout. + sigma: One noise level per batch item, shaped ``[B]``. + frame_index: Zero-indexed latent action shared by the batch. + kv_cache: Per-rollout sparse attention K/V history. + button: Optional multi-hot buttons in ``[B, 1, 256]`` layout. + mouse: Optional pointer deltas in ``[B, 1, 2]`` layout. + scroll: Optional wheel directions in ``[B, 1, 1]`` layout. + + Returns: + Rectified-flow velocity with the same shape as ``latent``. + + Raises: + ValueError: The latent, noise, or partial controller state does not + match Waypoint's one-action execution contract. + """ + if latent.ndim != 5 or latent.shape[1] != 1: + raise ValueError( + "WaypointDiT forward expects one latent action in [B, 1, C, H, W] " + f"layout, got {tuple(latent.shape)}" + ) + if sigma.ndim != 1 or sigma.shape[0] != latent.shape[0]: + raise ValueError( + f"sigma must have one value per batch item, got {tuple(sigma.shape)}" + ) + if frame_index < 0: + raise ValueError(f"frame_index must be non-negative, got {frame_index}") + controls = (button, mouse, scroll) + if any(control is not None for control in controls) and any( + control is None for control in controls + ): + raise ValueError("button, mouse, and scroll must be supplied together") + + tokens = self.patchify_latent(latent) + batch_size, token_count, _ = tokens.shape + conditioning = self.embed_noise(sigma.to(device=tokens.device)).to(tokens.dtype) + + if button is None: + control_frame = self.ctrl_cfg.null_emb.to(dtype=tokens.dtype).expand( + batch_size, 1, -1 + ) + else: + mouse = cast(Tensor, mouse) + scroll = cast(Tensor, scroll) + control_frame = self.embed_control( + button=button.to(device=tokens.device, dtype=tokens.dtype), + mouse=mouse.to(device=tokens.device, dtype=tokens.dtype), + scroll=scroll.to(device=tokens.device, dtype=tokens.dtype), + ) + if control_frame.shape[:2] != (batch_size, 1): + raise ValueError( + "Waypoint controller state must describe one frame per batch item, " + f"got {tuple(control_frame.shape)}" + ) + control_tokens = control_frame.expand(-1, token_count, -1) + cosine, sine = self._current_rope_angles( + frame_index=frame_index, device=tokens.device + ) + + initial_value: Tensor | None = None + conditioning = conditioning[:, None] + for layer_index, block in enumerate(self.transformer.blocks): + tokens, initial_value = block( + tokens, + conditioning=conditioning, + control=control_tokens, + cosine=cosine, + sine=sine, + layer_index=layer_index, + frame_index=frame_index, + kv_cache=kv_cache, + initial_value=initial_value, + ) + + scale, bias = self.out_norm.fc(F.silu(conditioning)).chunk(2, dim=-1) + tokens = F.silu(adaptive_rms_norm(tokens, scale, bias)) + return self.unpatchify_tokens(tokens) + + def _current_rope_angles( + self, *, frame_index: int, device: torch.device + ) -> tuple[Tensor, Tensor]: + """Build the fixed spatial and current temporal RoPE factors.""" + rows = torch.arange(self.spec.patch_grid_height, device=device) + columns = torch.arange(self.spec.patch_grid_width, device=device) + row_index = rows.repeat_interleave(self.spec.patch_grid_width) + column_index = columns.repeat(self.spec.patch_grid_height) + # The cache advances once per latent action. Temporal RoPE uses the + # checkpoint's base-rate timestamp, whose stride is part of the + # checkpoint contract (and is one for Waypoint 1.5). + frame_indices = torch.full_like( + row_index, + frame_index * self.spec.frame_timestamp_stride, + ) + return self.rope_angles( + frame_index=frame_indices, + row_index=row_index, + column_index=column_index, + ) diff --git a/integrations/waypoint/waypoint/transformer/norm.py b/integrations/waypoint/waypoint/transformer/norm.py new file mode 100644 index 000000000..8c3ad562c --- /dev/null +++ b/integrations/waypoint/waypoint/transformer/norm.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parameter-free RMS normalization operations used by Waypoint DiT blocks.""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import Tensor + + +def adaptive_rms_norm(tokens: Tensor, scale: Tensor, bias: Tensor) -> Tensor: + """Apply per-latent-frame adaptive RMS normalization. + + Args: + tokens: Token tensor shaped ``[batch, frames * tokens_per_frame, channels]``. + scale: Adaptive RMSNorm scale tensor shaped ``[batch, frames, channels]``. + bias: Adaptive RMSNorm bias tensor shaped ``[batch, frames, channels]``. + + Returns: + Adaptively normalized tokens with the same shape and dtype as ``tokens``. + + Raises: + ValueError: The token and conditioner layouts are incompatible. + """ + if tokens.ndim != 3 or scale.ndim != 3 or bias.ndim != 3: + raise ValueError("tokens, scale, and bias must each have three dimensions") + if scale.shape != bias.shape: + raise ValueError("AdaRMSNorm scale and bias shapes must match") + batch_size, token_count, channels = tokens.shape + if scale.shape[0] != batch_size or scale.shape[2] != channels: + raise ValueError("AdaRMSNorm conditioning must match token batch and channels") + frames = scale.shape[1] + if frames < 1 or token_count % frames: + raise ValueError( + "token count must be divisible by the number of conditioned frames" + ) + + tokens_per_frame = token_count // frames + x = tokens.reshape(batch_size, frames, tokens_per_frame, channels) + output = F.rms_norm(x, (channels,), weight=None, eps=None) + output = output * (1 + scale[:, :, None]) + bias[:, :, None] + return output.reshape_as(tokens) + + +def adaptive_gate(tokens: Tensor, gate: Tensor) -> Tensor: + """Scale each latent frame's residual branch with a conditioner gate. + + Args: + tokens: Residual tensor shaped ``[batch, frames * tokens_per_frame, channels]``. + gate: Per-frame gate tensor shaped ``[batch, frames, channels]``. + + Returns: + Gated residual tensor with the same shape and dtype as ``tokens``. + + Raises: + ValueError: The residual and gate layouts are incompatible. + """ + if tokens.ndim != 3 or gate.ndim != 3: + raise ValueError("tokens and gate must each have three dimensions") + batch_size, token_count, channels = tokens.shape + if gate.shape[0] != batch_size or gate.shape[2] != channels: + raise ValueError("AdaGate conditioning must match residual batch and channels") + frames = gate.shape[1] + if frames < 1 or token_count % frames: + raise ValueError("token count must be divisible by the number of gated frames") + tokens_per_frame = token_count // frames + gated = tokens.reshape(batch_size, frames, tokens_per_frame, channels) + return (gated * gate[:, :, None]).reshape_as(tokens) diff --git a/integrations/waypoint/waypoint/transformer/rope.py b/integrations/waypoint/waypoint/transformer/rope.py new file mode 100644 index 000000000..a6d5c1d64 --- /dev/null +++ b/integrations/waypoint/waypoint/transformer/rope.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Orthogonal three-axis rotary-angle construction for Waypoint attention.""" + +from __future__ import annotations + +import torch +from torch import Tensor, nn + +from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec + + +def apply_waypoint_ortho_rope(tokens: Tensor, cosine: Tensor, sine: Tensor) -> Tensor: + """Apply Waypoint's half-head rotary transform to attention tensors. + + Args: + tokens: Query or key tensor shaped ``[batch, tokens, heads, head_dim]``. + cosine: Packed RoPE cosine factors shaped ``[tokens, 1, head_dim / 2]``. + sine: Packed RoPE sine factors shaped ``[tokens, 1, head_dim / 2]``. + + Returns: + The rotary-transformed tensor with the same shape and dtype as ``tokens``. + + Raises: + ValueError: Tensor or angle shapes cannot represent the same token sequence. + """ + if tokens.ndim != 4 or tokens.shape[-1] % 2: + raise ValueError("tokens must have shape [batch, tokens, heads, even_head_dim]") + expected_angles = (tokens.shape[1], 1, tokens.shape[-1] // 2) + if cosine.shape != expected_angles or sine.shape != expected_angles: + raise ValueError( + "RoPE angles must have shape " + f"{expected_angles}, got cosine={tuple(cosine.shape)}, sine={tuple(sine.shape)}" + ) + # Projection channels arrive as adjacent real/imaginary pairs. The attention + # kernel receives the rotated result in its packed real-half / imaginary-half + # layout, matching Waypoint's grouped-query projections. + first = tokens.float()[..., 0::2] + second = tokens.float()[..., 1::2] + cosine = cosine.unsqueeze(0).float() + sine = sine.unsqueeze(0).float() + rotated = torch.cat( + (first * cosine - second * sine, first * sine + second * cosine), dim=-1 + ) + return rotated.to(dtype=tokens.dtype) + + +class WaypointOrthoRoPEAngles(nn.Module): + """Construct the parameter-free three-axis rotary angles used by Waypoint. + + One quarter of each head's complex dimensions represents horizontal location, + one quarter represents vertical location, and the remaining half represents + autoregressive time. Spatial coordinates are patch centers relative to the + latent-image center. The spatial frequency ceiling preserves circular + frequency under the checkpoint's 16:32 aspect ratio. + """ + + spatial_frequencies: Tensor + temporal_frequencies: Tensor + + def __init__(self, spec: WaypointModelSpec = WAYPOINT_1_5) -> None: + """Initialize an angle generator from a checkpoint architecture contract. + + Args: + spec: Published Waypoint architecture and RoPE constants. + + Raises: + ValueError: The head dimension cannot be evenly partitioned. + """ + super().__init__() + if spec.head_dim % 8: + raise ValueError( + "Waypoint orthogonal RoPE requires a head dimension divisible " + f"by 8, got {spec.head_dim}" + ) + self.spec = spec + spatial_dim = spec.head_dim // 8 + temporal_dim = spec.head_dim // 4 + spatial_frequency_count = (spatial_dim + 1) // 2 + max_frequency = min(spec.patch_grid_height, spec.patch_grid_width) * ( + spec.rope_nyquist_fraction + ) + spatial = ( + torch.linspace( + 1.0, + max_frequency / 2, + spatial_frequency_count, + dtype=torch.float32, + ) + * torch.pi + ).repeat_interleave(2)[:spatial_dim] + temporal = torch.pow( + torch.tensor(spec.rope_theta, dtype=torch.float32), + -torch.arange(0, temporal_dim, 2, dtype=torch.float32) / temporal_dim, + ).repeat_interleave(2) + self.register_buffer("spatial_frequencies", spatial, persistent=False) + self.register_buffer("temporal_frequencies", temporal, persistent=False) + + def _apply(self, fn): + """Retain FP32 angle arithmetic after device precision conversion.""" + + def keep_dtype(tensor: Tensor) -> Tensor: + return fn(tensor).to(dtype=tensor.dtype) + + return super()._apply(keep_dtype) + + def forward( + self, + *, + frame_index: Tensor, + row_index: Tensor, + column_index: Tensor, + ) -> tuple[Tensor, Tensor]: + """Return packed cosine and sine factors for token positions. + + Args: + frame_index: Integer latent-frame positions with shape ``[tokens]``. + row_index: Integer patch-row positions with shape ``[tokens]``. + column_index: Integer patch-column positions with shape ``[tokens]``. + + Returns: + Cosine and sine tensors, each shaped ``[tokens, 1, head_dim / 2]``. + + Raises: + ValueError: Position tensors do not share one one-dimensional shape. + """ + positions = (frame_index, row_index, column_index) + token_count = frame_index.numel() + if any( + position.ndim != 1 or position.numel() != token_count + for position in positions + ): + raise ValueError( + "all RoPE position tensors must have matching [tokens] shape" + ) + if not (frame_index.device == row_index.device == column_index.device): + raise ValueError("all RoPE position tensors must be on one device") + + dtype = torch.float32 + device = frame_index.device + column_position = ( + 2.0 * column_index.to(dtype) + 1.0 + ) / self.spec.patch_grid_width - 1.0 + row_position = ( + 2.0 * row_index.to(dtype) + 1.0 + ) / self.spec.patch_grid_height - 1.0 + temporal_position = frame_index.to(dtype) + angles = torch.cat( + ( + column_position[:, None] * self.spatial_frequencies.to(device=device), + row_position[:, None] * self.spatial_frequencies.to(device=device), + temporal_position[:, None] + * self.temporal_frequencies.to(device=device), + ), + dim=-1, + ) + return torch.cos(angles)[:, None], torch.sin(angles)[:, None] diff --git a/pyproject.toml b/pyproject.toml index 02f62aa87..1a81d657c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ extraPaths = [ "integrations/self_forcing", "integrations/wan21", "integrations/wan22", + "integrations/waypoint", ] # For pyright to pickup the correct python. venvPath = "." @@ -82,6 +83,7 @@ extra-paths = [ "integrations/self_forcing", "integrations/wan21", "integrations/wan22", + "integrations/waypoint", ] [tool.ty.src] diff --git a/uv.lock b/uv.lock index d894ab6a8..d69d18a59 100644 --- a/uv.lock +++ b/uv.lock @@ -32,6 +32,7 @@ members = [ "flashdreams-t2v", "flashdreams-wan21", "flashdreams-wan22", + "flashdreams-waypoint", "ludus-renderer", ] overrides = [ @@ -1483,6 +1484,30 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "flashdreams-waypoint" +version = "0.1.0" +source = { editable = "integrations/waypoint" } +dependencies = [ + { name = "flashdreams" }, + { name = "imageio", extra = ["pyav"] }, + { name = "opencv-python" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "flashdreams", editable = "flashdreams" }, + { name = "imageio", extras = ["pyav"], specifier = ">=2.37" }, + { name = "opencv-python", specifier = ">=4.10" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, +] +provides-extras = ["dev"] + [[package]] name = "flip-evaluator" version = "1.7" @@ -1852,6 +1877,9 @@ ffmpeg = [ { name = "imageio-ffmpeg" }, { name = "psutil" }, ] +pyav = [ + { name = "av" }, +] [[package]] name = "imageio-ffmpeg" @@ -3161,6 +3189,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/bd/a0c8e737b6afda10e42a597787d53d5b66e00268df6f59184701eeae37d9/onnxscript-0.7.1-py3-none-any.whl", hash = "sha256:544763b7fdef49940cdd9412ff5135cbae96d59ac6bc1921457f21280f40f4b7", size = 721970, upload-time = "2026-06-29T23:33:23.298Z" }, ] +[[package]] +name = "opencv-python" +version = "5.0.0.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/4c/a438d23e09ce2033c09f7b784ad2fbdb0adf529e434101ed28f142226f98/opencv_python-5.0.0.93.tar.gz", hash = "sha256:66aac3e5b5faa48d4025816592f3af19e4bfc2c68dec067bae2dbb4ca10aa9e2", size = 81802749, upload-time = "2026-07-02T06:59:53.815Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/75/76f6ade78f6102c61034f828e2a22616708df2c9504bc8d6af9dd8f73dc5/opencv_python-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:198a75138241810206a17c829dbcc40a7cb1841cda538ca86cbbfc6c7d95f898", size = 48322443, upload-time = "2026-07-02T05:50:25.466Z" }, + { url = "https://files.pythonhosted.org/packages/15/8c/bc1bda6aae69a32e9d84fc34153ba104cd25226861eb4aea33b2cea4860d/opencv_python-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:6bbc32f59e1b1a7db7b39c81f63d00625f041d333037fd8702f6da52cc39108b", size = 34782755, upload-time = "2026-07-02T05:51:30.556Z" }, + { url = "https://files.pythonhosted.org/packages/f4/8a/b04776ec45d2dea08a1b176f1829201db3515d4ed16c35f8fcc9fa7beb16/opencv_python-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2b4272e736836f66c2d176e43ab8101f3a00d45654916399f52e150c58981ac", size = 50614064, upload-time = "2026-07-02T06:53:22.604Z" }, + { url = "https://files.pythonhosted.org/packages/95/54/eb47866b94f2b5b42dde17644b78055ef1ee05aae59962c7290e55270803/opencv_python-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f8b6d0a212253dd26ad338c812f1f23ca118fdf05a9c8c6b9444f161aa8c5881", size = 71064711, upload-time = "2026-07-02T06:54:13.148Z" }, + { url = "https://files.pythonhosted.org/packages/93/da/962579f1e703cbf8c5422fd1f576467dcb3b5b0b0b81c1471c979764353a/opencv_python-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:08d5d91d967b58d6db86073b2ad3eaef88ca4ebdfd45c9059bf59f5ded0c7ad2", size = 49798576, upload-time = "2026-07-02T06:54:33.781Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4c/c73f828fdbcd37eaf21d08fa852544a3ca7c2dbb3ea76873d64f2ea413d1/opencv_python-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:c8de2dec111122a02e8beb28e16c31904992dfd6186560b142a92c71403c1039", size = 73783032, upload-time = "2026-07-02T06:55:03.415Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4b/edaf83b996ca5a1a3d8ccad485706b9c6d4742b13b9c4586bf1c1e7d9423/opencv_python-5.0.0.93-cp37-abi3-win32.whl", hash = "sha256:4b4b1a34c79bf8d3738e3cfe9a9e67b51a79663f6b692cbdad8c31f570da4157", size = 35564734, upload-time = "2026-07-02T05:49:57.704Z" }, + { url = "https://files.pythonhosted.org/packages/21/f0/9fa6e85cb10c8eb36a0222d27e50fe381b86ce49a55446bf39f491727564/opencv_python-5.0.0.93-cp37-abi3-win_amd64.whl", hash = "sha256:f90ba04b8f73bc5c3814037699739f0156f597338a98f05956c684e7c3ca10d2", size = 44000345, upload-time = "2026-07-02T05:49:54.971Z" }, +] + [[package]] name = "opencv-python-headless" version = "4.13.0.92"