diff --git a/.agents/skills/ort-genai-config/references/processor-config-fields.md b/.agents/skills/ort-genai-config/references/processor-config-fields.md index b21dfb5ed..fbe14611a 100644 --- a/.agents/skills/ort-genai-config/references/processor-config-fields.md +++ b/.agents/skills/ort-genai-config/references/processor-config-fields.md @@ -29,12 +29,6 @@ used by ort-extensions image processors. For an overview, see the parent "attrs": { "color_space": "RGB" } } }, - { - "operation": { - "name": "convert_to_rgb", - "type": "ConvertRGB" - } - }, { "operation": { "name": "resize", @@ -63,8 +57,7 @@ used by ort-extensions image processors. For an overview, see the parent "type": "Normalize", "attrs": { "mean": [0.48145466, 0.4578275, 0.40821073], - "std": [0.26862954, 0.26130258, 0.27577711], - "qwen2_5_vl": 1 + "std": [0.26862954, 0.26130258, 0.27577711] } } }, @@ -91,12 +84,22 @@ used by ort-extensions image processors. For an overview, see the parent | Type | Purpose | Key attrs | |---|---|---| | `DecodeImage` | Decode from bytes | `color_space` | -| `ConvertRGB` | Ensure RGB | — | +| `ConvertRGB` | Swap BGR to RGB; omit after RGB decode | — | | `Resize` | Smart resize | `width`, `height`, `smart_resize`, `min_pixels`, `max_pixels`, `patch_size`, `merge_size` | | `Rescale` | Scale pixel values | `rescale_factor` | | `Normalize` | Mean/std normalization | `mean`, `std` | | `PatchImage` | Extract patches | `patch_size`, `temporal_patch_size`, `merge_size` | +After `DecodeImage(color_space="RGB")`, use ordinary `Normalize`: both +`ConvertRGB` and the nonzero `qwen2_5_vl`/`qwen3_vl` Normalize flags swap red +and blue. Neither owns patch interleaving; `PatchImage` owns that layout. +Do not add either swap to an already-RGB pipeline. + +Native smart resize and HF can disagree at half-factor boundaries: +the inspected ORT-Extensions implementation uses `std::round`, while HF uses +Python ties-to-even `round`. Grid/token equality must be checked separately +from RGB correctness and successful generation. + --- ## Generating from HuggingFace config @@ -113,7 +116,6 @@ processor_config = { "transforms": [ {"operation": {"name": "decode_image", "type": "DecodeImage", "attrs": {"color_space": "RGB"}}}, - {"operation": {"name": "convert_to_rgb", "type": "ConvertRGB"}}, {"operation": {"name": "resize", "type": "Resize", "attrs": { "width": 540, "height": 360, "smart_resize": 1, @@ -125,7 +127,6 @@ processor_config = { "attrs": {"rescale_factor": ip.rescale_factor}}}, {"operation": {"name": "normalize", "type": "Normalize", "attrs": { "mean": list(ip.image_mean), "std": list(ip.image_std), - "qwen2_5_vl": 1, }}}, {"operation": {"name": "patch_image", "type": "PatchImage", "attrs": { "patch_size": ip.patch_size, @@ -150,7 +151,6 @@ def _write_processor_config(processor, output_dir): "transforms": [ {"operation": {"name": "decode_image", "type": "DecodeImage", "attrs": {"color_space": "RGB"}}}, - {"operation": {"name": "convert_to_rgb", "type": "ConvertRGB"}}, {"operation": {"name": "resize", "type": "Resize", "attrs": { "width": 540, "height": 360, "smart_resize": 1, "min_pixels": ip.size.get("shortest_edge", 3136), @@ -161,7 +161,6 @@ def _write_processor_config(processor, output_dir): "attrs": {"rescale_factor": ip.rescale_factor}}}, {"operation": {"name": "normalize", "type": "Normalize", "attrs": { "mean": list(ip.image_mean), "std": list(ip.image_std), - "qwen2_5_vl": 1, }}}, {"operation": {"name": "patch_image", "type": "PatchImage", "attrs": { "patch_size": ip.patch_size, diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dbf05de3..afb7c3cb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,70 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Qwen3.8-27B standard-only multimodal qualification + +#### Added + +- Corrected-RGB native ORT GenAI CUDA first-200 AI2D execution completes at + 174/200, versus frozen HF 175/200 and raw ORT 174/200 references. Native + smart-resize tie rounding differs from HF for 28 examples; task accuracy + and token agreement do not establish preprocessing or logit parity. +- Offline tiny standard VLM CLI tests cover ordinary and existing + `--runtime ort-genai` packaging, sparse global cache indices, and FP16 + convolution/FP32 recurrent state. A separate opt-in native hybrid probe + checks state-dependent tokens and fresh-generator reset; a metadata-only + qualification example selects CUDA through `og.Config`, without a new + Mobius CLI flag or rebuilding existing ONNX weights. +- Dense Qwen3.5-family composites, including Qwen3.8-27B, support text-only + export through the public Transformers builder. Offline tests cover the + pinned full-size text and complete VLM graphs, plus reduced random-weight + CUDA FP32/FP16 text/image/video/mixed-media prefill/decode parity, not the + official 27B checkpoint weights. + +#### Fixed + +- Native Qwen image processor metadata no longer swaps red/blue after RGB + decode. Ordinary normalization preserves checkpoint mean/std, while + `PatchImage` retains patch packing. Static contracts and an opt-in toy native + before-prefill RGB regression cover the correction; existing metadata must + be regenerated. Real-image resize and numerical parity remain unqualified. +- ORT-GenAI processor generation uses local `--config` HF assets instead of + silently retaining CLIP defaults. Unwrapped `qwen3_5_text` VLM packages + retain Qwen patch preprocessing and vision metadata. +- Explicit `mamba_ssm_dtype` is retained in DeltaNet computation and exported + recurrent-state inputs/outputs. Re-exported FP16/BF16 hybrid models whose + config requests FP32 state now expose FP32 recurrent slots; callers must + allocate state from graph input/output types, not the model weight dtype. + Existing saved models are unchanged. Missing or `auto` policies preserve + the previous model-dtype state behavior. +- Shared Qwen3.5 dense/MoE/VL DeltaNet layers use the HF reference's fixed + `1e-6` Q/K L2 normalization epsilon. Native GenAI and other variants' + official-checkpoint qualification remain separate. +- Shared Qwen3/Qwen3.5 split vision graphs cast processor FLOAT32 pixels to + model dtype before reduced-precision convolution. +- Shared Qwen3/Qwen3.5 split embedding graphs recognize both image and video + placeholders and use feature offsets across the entire batch. Their + existing `image_features` input carries visual rows in flattened + batch/token placeholder order; no new graph input is required. +- Learned offset RMS normalization keeps normalization and `1 + weight` + scaling in FP32 until the final model-dtype cast, avoiding premature + rounding of small trained offsets. + +#### Qualification status + +- Installed GenAI `0.16.0-dev` passed native sparse hybrid state progression + and fresh-generator reset with FP16 convolution/FP32 recurrent state and + non-shared caches. The unchanged real standard graphs then passed bounded + native text/reset and single-/two-image four-token generation. The CUDA + qualification overlay sets matching providers on all three sessions before + the required `og.Config` execution override: explicit CPU auxiliary sessions + did not follow that override and segfaulted during prefill. This is bounded + execution evidence, not numerical parity or full-checkpoint accuracy. +- The complete real-weight FP16 vision/embedding/hybrid-decoder package was + exported, saved and reloaded with standard ONNX operators only. Official + image vision and text prefill comparisons still exceed the unchanged + FP16 tolerance; this is not a full-checkpoint accuracy or numerical-parity pass. + ### GPT-OSS MXFP4 export #### Added diff --git a/docs/cli_reference.md b/docs/cli_reference.md index a774017a6..75bcd5089 100644 --- a/docs/cli_reference.md +++ b/docs/cli_reference.md @@ -91,6 +91,60 @@ mobius build --model meta-llama/Llama-3.2-1B --output output/ --ep webgpu --dtyp mobius build --model meta-llama/Llama-3.2-1B --output output/ --ep onnx-standard ``` +#### Standard Qwen3.8-27B VLM packaging versus runtime selection + +The full VLM supports the ordinary build route and the existing ORT-GenAI +metadata route. Both retain `decoder/model.onnx`, `vision_encoder/model.onnx`, +and `embedding/model.onnx` when built with `--ep onnx-standard`: + +```bash +mobius build --config /absolute/local-qwen-checkpoint --output /absolute/standard-vlm \ + --ep onnx-standard --dtype f16 +mobius build --config /absolute/local-qwen-checkpoint --output /absolute/standard-genai \ + --ep onnx-standard --dtype f16 --runtime ort-genai +``` + +`--ep` controls **graph construction**, not independent runtime selection. +There is no new runtime-EP CLI flag. Prepare CUDA **runtime metadata** for all +three already-standard graphs using the qualification overlay example (or +`write_ort_genai_config(pkg, directory, ep="cuda")`, without saving/rebuilding +graphs). Then select CUDA at execution: + +```python +import onnxruntime_genai as og + +config = og.Config("/absolute/cuda-runtime-overlay") +config.clear_providers() +config.append_provider("CUDAExecutionProvider") +model = og.Model(config) +``` + +In the qualified GenAI `0.16.0-dev` build, explicit CPU vision/embedding session +options do not follow the execution provider override. Mixing those CPU sessions +with CUDA decoder/device buffers segfaulted during native prefill; in-memory +`Config.overlay()` provider patches did not resolve it. Matching per-component +CUDA metadata passed. This is why the example prepares CUDA session metadata +while retaining the exact standard graphs. Do not change build-time `--ep` to +`cuda` to obtain this runtime-only configuration. + +Standard dynamic KV caches require `past_present_share_buffer=false`; CUDA +graph capture remains off. Hybrid cache slots retain global indices and +independent FP16 convolution/FP32 recurrent types. Successful packaging does +not prove a particular native GenAI build can execute those states. + +Qwen processor metadata now preserves RGB from `DecodeImage` through ordinary +`Normalize`; `PatchImage` owns patch packing. Regenerate processor metadata to +remove the former channel-swap flag from existing configurations, without +changing graphs or weights. The residual real-image resize difference is not +qualified as bitwise pixel parity. + +For an existing large artifact, do **not** rebuild or save the weights to add +metadata. Use `examples/qwen38_standard_genai.py` to reconstruct a lazy package +and create a symlink overlay. See +[Qwen3.8 qualification](design/qwen38-27b-standard-onnx.md#native-ort-genai-bounded-qualification) +for the executed bounded evidence and opt-in probe. No CPU/DML, performance, +numerical-parity, video, serving, or graph-capture guarantee is implied. + ### Optimization Rules (`--optimize`) ``` @@ -306,6 +360,22 @@ For a full ORT-GenAI text-only package (with `genai_config.json`), use `auto_export(..., text_only=True)` — see `examples/gemma4_12b_text_ort_genai.py`. +Dense Qwen3.5-family composites, including `Qwen/Qwen3.8-27B`, also support +`--features text-only`. Use `--ep onnx-standard` for standard-operator export. +Omit `--features text-only` to retain the complete vision/embedding/hybrid +decoder package. Both forms retain all hybrid decoder layers. The vision +graph accepts processor FLOAT32 pixels and casts internally to model dtype. +The split embedding graph's `image_features` input carries both image and +video features in flattened batch/token placeholder order; its feature +indices do not restart per batch row. +When the checkpoint declares `mamba_ssm_dtype="float32"`, FP32 recurrent state +is retained with FP16 weights/activations; without an explicit policy, state +remains model dtype. Qwen3.8-27B evidence covers graph construction, reduced +random-weight text/image/video parity, and a saved/reloaded real-weight +standard-only package. Official FP16 numerical qualification is not yet +passing. See +[Qwen3.8-27B qualification](design/qwen38-27b-standard-onnx.md). + ### More Examples ```bash diff --git a/docs/design/qwen38-27b-standard-onnx.md b/docs/design/qwen38-27b-standard-onnx.md new file mode 100644 index 000000000..bdaecd03f --- /dev/null +++ b/docs/design/qwen38-27b-standard-onnx.md @@ -0,0 +1,609 @@ +# Qwen3.8-27B standard-only multimodal export qualification + +## Source and scope + +`Qwen/Qwen3.8-27B` uses the existing `qwen3_5` multimodal architecture and +`qwen3_5_text` decoder, not Flash-Next's `qwen4_exp` architecture. Its text +decoder has 64 layers: 48 Gated DeltaNet layers and 16 full-attention layers, +with a dense MLP in every layer. + +The offline fixture `testdata/configs/qwen3_8-27b.json` preserves the official +config values from revision `1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0`. +The upstream raw `config.json` has SHA256 +`191e0af232104ed8b65258cf3fb2b842e288008baca7633c11b82a1ac7203aab` +and size 4312 bytes. The checked-in fixture is reformatted, so that raw-byte +hash is provenance, not the fixture's byte hash. + +The target is the **complete, unquantized standard ONNX vision-language +package**, not just its text decoder. Text-only export remains an optional +subset: removing vision does not remove any hybrid decoder layers. +The checked-in tests cover full-size graph construction and reduced +random-weight text/image/video parity. They do not download model weights +or establish official-checkpoint accuracy. Separate bounded native ORT GenAI +execution and AI2D first-200 evidence are recorded below; they do not establish +full numerical parity or production readiness. ModelBuilder equivalence +remains deferred, not an acceptance gate for this export. + +## Operators and state contract + +`execution_provider="onnx-standard"` disables custom-op fusion and inlines +the `CausalConvWithState` and `LinearAttention` function bodies. The resulting +graph uses standard convolution, per-head normalization, gate arithmetic, +`Scan` recurrence, standard attention, and dense MLP operations. Tests require +standard domains and schemas recursively, including the Scan bodies, and no +remaining local functions. + +The public Transformers builder supports `text_only=True` for the official +`qwen3_5` composite and its `qwen3_5_text` sibling. The resulting package has +one `model` entry rather than a vision/embedding/decoder pipeline. + +With `text_only=False`, the package contains `vision_encoder`, `embedding`, +and `decoder`. The full decoder still contains all 48 DeltaNet and 16 +full-attention layers; the vision encoder contains all 27 layers. + +### Visual feature contract + +The vision graph accepts processor-native FLOAT32 `pixel_values` and casts +once to model dtype internally. Run image and video packed patches through +the same vision graph, supplying each stream's grid as `image_grid_thw`. + +The embedding graph keeps its existing `input_ids` and `image_features` +input names. Despite the latter's name, it accepts **both image and video +features**. Pack feature rows in the order their placeholder tokens occur +in flattened, row-major `input_ids`, including across batch rows. Do not +concatenate all image rows before all video rows unless the placeholders +actually follow that order. Feature indices do not restart at each batch +row. For text-only input or decode without new media, supply an empty +`[0, hidden_size]` feature tensor of model dtype. + +The same global indices apply to shared Qwen3-VL DeepStack feature maps. +Qwen3.8-27B has no DeepStack maps. A separate embedding-only CPU sentinel +test covers two maps, two-row mixed image/video prompts with reversed order, +and zero-media input; full DeepStack decoder parity remains unqualified. +This fixes shared Qwen3/Qwen3.5 split components; +existing saved graphs must be re-exported to receive the fixes. GenAI's +ability to assemble the packed image/video stream is not established here. + +For each DeltaNet layer: + +- Convolution state is model dtype, `[B, conv_dim, K-1]`. +- Recurrent state is `[B, value_heads, key_dim, value_dim]`. +- The official `mamba_ssm_dtype="float32"` is preserved through config + extraction, task inputs, recurrence function registration, and outputs, + even when weights/activations are FP16. +- Configs without an explicit state-dtype policy (or with `auto`) retain the + prior model-dtype behavior. Invalid state dtypes are rejected. +- Qwen3.5 per-head Q/K normalization uses the reference epsilon `1e-6`, + rather than the generic component's zero-epsilon default. + +The epsilon is fixed by the Qwen3.5 reference architecture, not a configurable +Qwen3.5 export option. The shared decoder change also applies to DeltaNet +layers constructed by Qwen3.5 MoE/VL variants. Their graph/weight regressions +are included, but this qualification does not establish official-checkpoint +parity for those other variants. The existing MTP decoder is full-attention +and does not construct DeltaNet. FP16/BF16/FP64 state policies can be +represented by the config; +only the official FP32 policy is numerically qualified here. + +Learned offset RMS normalization now computes input normalization, the +`1 + weight` offset and scale multiplication in FP32 before casting the +output back to model dtype. The prior implementation rounded the offset +in FP16/BF16. A nonzero learned-weight unit test reproduces that difference; +zero-initialized reference norms do not exercise it. This corrects the +shared component contract but does not resolve full-checkpoint parity alone. + +The HF reference caches K convolution positions; Mobius needs the K-1 +preceding positions. Comparisons explicitly select the final K-1 HF entries. +The recurrent-state comparison uses nonsquare key/value dimensions in the +reduced fixture to detect an accidental transpose. This is a dense +invocation ABI, not the native packed/paged contract in PR #738. + +**Compatibility:** re-exporting a hybrid checkpoint that requests FP32 state +changes its FP16/BF16 recurrent slots to FP32. Existing saved models do not +change. Callers must allocate/rebind cache using the graph's per-slot types, +not assume that every state has model dtype. The raw ORT parity tests exercise +that contract; compatibility of a particular GenAI cache allocator is not +established by the standard export alone. + +## Native ORT-GenAI bounded qualification + +The extension preserves the exact three standard graphs and native Qwen3.5 +multimodal dispatch; it does not substitute a generic decoder, ModelBuilder, +manual Python cache loop, PagedAttention, or fused MoE. The ordinary CLI route +and existing `--runtime ort-genai` route are covered by tiny offline VLM tests +in `integrations/ort_genai/qwen38_standard_test.py`. No new Mobius CLI flag was +added. `--ep onnx-standard` remains the build policy; execution selects CUDA +with `og.Config.clear_providers()`, `append_provider("CUDAExecutionProvider")`, +and `og.Model(config)`. + +Two reproduced metadata gaps were corrected: + +- Local `--config` assets reached tokenizer/generation metadata but not the + vision processor writer. The writer now uses the explicit HF source or, + if absent, `local_config_dir`, retaining checkpoint normalization and + smart-resize bounds. +- The VLM's unwrapped `qwen3_5_text` configuration was missing from the Qwen + vision-family sets, dropping `PatchImage` and Qwen vision metadata. + +The VLM writer already bypasses the text-only `_inspect_decoder_abi`; its +`inputs_embeds` mapping and graph-derived highest global cache slot did not +require an inspector rewrite. Standard hybrid packaging keeps sharing false +and capture off. Generic fast CPU e2e tests remain pinned to GenAI 0.15.2. + +### Native runtime evidence and provider-routing diagnosis (2026-10-08) + +The isolated checkout at `477380e595bc75fcdc6a87939d5c2246741bedc1` imports +Mobius from its own `src`. Installed ORT is 1.30.0 and GenAI reports +`0.16.0-dev`; its build identity has not been established. + +`tests/ort_genai_hybrid_native_test.py` is opt-in through +`MOBIUS_QWEN_NATIVE_PROBE=1`, independent of the generic fast CPU lane. +Its standard-only synthetic three-graph pipeline has eight global slots, +sparse KV at 3 and 7, FP16 convolution state, FP32 recurrent state, and 3D +positions. The final probe uses the real per-slot geometry: hidden width 5120, +KV `[B,4,pastS,256]`, conv `[B,10240,3]`, and recurrent +`[B,48,128,128]`, but only eight slots and a 32-token synthetic vocabulary, +without checkpoint weights. Expected tokens `[0, 4, 8]` depend on *both* state families and +past KV length. Every state output's dtype, shape, and progression is checked; +a fresh generator must reproduce the same sequence. Model construction alone +is not a pass. + +On GPU `GPU-630e9db2-c115-0a2d-789f-4a76ccb92ecc`: + +- The generic native CUDA control passed, generating `[0, 1, 2]`. +- The hybrid native model and bundled multimodal processor were constructed + successfully, and native text inputs were produced. Thus absence of the + Python `onnxruntime_extensions` package is not evidence of missing bundled + processor support. +- The initial mixed-provider probe segfaulted **during + `generator.set_inputs(inputs)` / native prefill**, not generator construction. + The earlier attribution was corrected after explicit stage markers and GDB + backtraces. CPU `ConstantOfShape::Compute`, and then CPU + `GatherCopyData` after replacing the synthetic embedding with Gather, + were called through `MultiModalPipelineState::Run` and + `OgaGenerator_SetInputs`. Recurrent-cache incompatibility was not established. +- With empty auxiliary provider lists, the execution override did not route + vision/embedding to CUDA. `Config.overlay()` patches specifying auxiliary + or all three providers did not resolve the fault in either tested call order. +- Rendering **CUDA runtime session metadata for all three standard graphs** + through the existing writer, keeping capture off, resolved native prefill. + The required `clear_providers()` / `append_provider("CUDAExecutionProvider")` + / `Model(config)` calls remain. This does not select CUDA graph construction + or alter/export any ONNX graph. +- The synthetic native probe then passed all state-dependent token, dtype, + shape, multi-step state progression, and fresh-generator reset assertions: + `[[0,4,8], [0,4,8]]`. The installed dev runtime therefore demonstrated sparse + global indexing, mixed cache types, and non-shared swaps; published source + capability is no longer being used as a substitute for execution evidence. + +Run the passing opt-in probe with one bounded command (fresh absolute basetemp): + +```bash +env PYTHONPATH=/home/titaiwang/workspace/mobius/.worktrees/qwen27b-genai-9a3ab8ed/src \ + TMPDIR=/datadisks/disk5/titaiwang/mobius-qwen38-27b-standard-9a3ab8ed/genai-integration/tmp \ + HF_HOME=/datadisks/disk5/titaiwang/mobius-qwen38-27b-standard-9a3ab8ed/genai-integration/hf \ + HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 \ + CUDA_VISIBLE_DEVICES=GPU-630e9db2-c115-0a2d-789f-4a76ccb92ecc \ + MOBIUS_QWEN_NATIVE_PROBE=1 timeout 120 \ + /home/titaiwang/miniconda3/envs/dev/bin/python -m pytest \ + /home/titaiwang/workspace/mobius/.worktrees/qwen27b-genai-9a3ab8ed/tests/ort_genai_hybrid_native_test.py::test_native_sparse_mixed_state_progression_and_reset \ + -q -s --tb=long \ + -o cache_dir=/datadisks/disk5/titaiwang/mobius-qwen38-27b-standard-9a3ab8ed/genai-integration/native-control-cache \ + --basetemp=/datadisks/disk5/titaiwang/mobius-qwen38-27b-standard-9a3ab8ed/genai-integration/native-probe-next +``` + +Replacing/installing a runtime or rebuilding native C++ requires +separate approval. Do not relax parity tolerances or substitute raw ORT to +declare this gate passed. + +### Metadata-only real-artifact overlay + +`examples/qwen38_standard_genai.py` accepts absolute source, HF-assets, and +fresh output paths. It loads graph metadata lazily with `onnx_ir`, reconstructs +`ArchitectureConfig` from the pinned HF text config and parent, and uses the +repository's config writer. It never calls `ModelPackage.load/save`, loads an +HF model, or serializes/copies external weights. Component directories are +symlinked, preserving their sidecars; small tokenizer/metadata files are copied +before potentially mutable asset patches. Existing destinations are refused. + +CUDA preparation and bounded generation passed for the immutable `onnx-rmsfix` +artifact and produced: + +`/datadisks/disk5/titaiwang/mobius-qwen38-27b-standard-9a3ab8ed/genai-integration/runtime-overlay-cuda-02/overlay-receipt.json` + +and `native-results.json` in that same overlay. + +The audit confirmed zero functions/custom-domain nodes in all three graphs, +global KV indices `3,7,...,63`, FP16 convolution and FP32 recurrent ports, and +unchanged original HF-asset hashes. The decoder, vision, and embedding graph +metadata sizes are respectively 3,421,999, 1,065,742, and 6,663 bytes. No +54GB weight duplication occurred. Transformers emitted documentation errors +for `min_frames`/`max_frames` during processor import, but preparation exited +successfully. These warnings were not generation failures. + +Use `--prepare-only` for metadata preparation. Real generation requires +`--native-proof /absolute/.../native-proof.json` from a successful synthetic +test with the same reported installed runtime version. The example selects +CUDA through `og.Config`, delegates preprocessing/positions/caches/tokens to +GenAI, and bounds generation to 1–16 new tokens. It checks text, a fresh- +generator text reset, and an optional single image. A programmatic +`second_image` argument additionally checks two different images without a new +CLI flag. It releases native objects +after use and has no success-shaped fallback. + +The proof is a **local operator attestation**, not a runtime binary identity +check: development builds can share the same version string. Rerun the native +proof after **any runtime binary change**. Matching installed binary SHA256 +fingerprints for this qualification are stored in +`genai-integration/qualification-receipt.json` on the qualification artifact +disk; the example does not automatically verify them. + +### Bounded real native results + +These results used historical `runtime-overlay-cuda-02` metadata **before the +RGB normalization correction below**. They establish execution only; fresh +metadata and a real-image rerun are required to validate the corrected inputs. + +The 420-second-bounded command completed successfully with four new greedy +tokens per invocation, thinking disabled, and no HF model loaded alongside +GenAI: + +| Invocation | Native-expanded prefill | Tokens | Decoded | +|---|---:|---|---| +| Text | 17 | `[760,6511,314,9338]` | `The capital of France` | +| Fresh-generator text reset | 17 | Same four tokens | Same text | +| Single `pipeline-cat-chonk.jpeg` image | 649 | `[1919,2099,4774,264]` | `This image shows a` | +| Cat + `mage-vl-dog.jpg` images | 2700 | `[760,1118,2099,4774]` | `The first image shows` | + +GenAI owned image preprocessing, 3D positions, all cache slots, and token +generation. This establishes bounded native execution and text reset, not +independent numerical correctness of native preprocessing/positions or +full-checkpoint accuracy. The native model was released on completion. +The immutable source remains 54,779,987,399 bytes; only small metadata/tokenizer +files and symlinks were added. No weights were duplicated or reserialized. + +The first real command covered text/reset/single-image. A second 420-second- +bounded invocation reused the same CUDA metadata overlay and immutable weights +to cover the two-image case as well; it again reproduced the same text/reset +and single-image tokens. It did not create a second model export or load two +native models simultaneously. + +**Bounded phase status: executed.** Synthetic progression/reset and real text/ +single-/two-image few-token generation passed. Full AI2D200 evaluation was not +run in this phase. +The prior FP16 logit-parity failures and unchanged `1e-2` tolerances remain +unresolved. No performance, CPU/DML, video, serving, or capture promise is made. + +### RGB metadata producer correction + +The native `DecodeImage(color_space="RGB")` operation already emits RGB. +In the upstream ORT-Extensions producer contract +(`operators/vision/decode_image.hpp` and `shared/api/image_transforms.hpp`), +nonzero `Normalize.qwen2_5_vl` or `Normalize.qwen3_vl` enables a BGR-to-RGB swap, +not patch interleaving. Injecting that flag after RGB decode therefore reversed +red and blue. The writer now leaves ordinary Normalize mean/std attributes +unchanged; `PatchImage` separately owns temporal/spatial patch layout. +Resize, rescale, all graphs, cache policy, and model tolerances are unchanged. +Regenerate processor metadata; historical overlays are not patched in place. + +The lead's before-prefill diagnostic found matching image/grid/IDs but direct +pixel max-absolute difference 2.0. Red/blue reversal alone, without row reorder, +reduced it to approximately 0.0156862. That residual resize/rounding difference +remains unresolved and is **not** bitwise pixel parity. + +An opt-in toy native regression uses a lossless uniform RGB `[255,128,0]` +image, only three added image-marker tokens, and no checkpoint weights. +It checks the native processor's pixel values against analytical per-channel +normalization **before any generator/prefill call**. No manual flips, row +reorders, graph changes, or tensor substitutions are used. This independently +reproduces the old producer's swapped channels and guards the corrected RGB +contract; it does not qualify real-image resize behavior or model parity. + +## Verification + +### Corrected-RGB native AI2D first-200 evidence + +The corrected metadata-only `runtime-overlay-cuda-03` completed all 200 pinned +AI2D examples through installed ORT GenAI `0.16.0-dev` on CUDA, using native +image processing, positions, caches, and greedy generation (16 new tokens, +thinking disabled). HF processing supplied only the identical chat template +and independent diagnostics, never tensors to the native pipeline. + +| Metric | Corrected native GenAI | Prior HF Torch | Prior raw ORT CUDA | +|---|---:|---:|---:| +| Correct answers | 174/200 | 175/200 | 174/200 | +| Invalid native answers | 3 | — | — | +| Exact native generated tokens | — | 199/200 | 198/200 | + +This is completed execution/task evidence, **not preprocessing or full-logit +parity**. Native IDs, image grids, and pixel shapes matched HF for 172/200 +examples. The other 28 have a diagnosed native smart-resize rounding +difference: ORT-Extensions uses `std::round`, whereas the installed HF +processor uses Python's ties-to-even `round`. Both rounding models predict +their respective observed grids for all 200 examples. For example, the +400-pixel width at zero-based index 130 becomes native 416 versus HF 384 at +factor 32; its native answer is B versus the HF/raw-ORT answer D. This +coincidence does not establish that resizing alone caused the answer change. +Index 124 instead matches HF's C, while prior raw ORT answered A. + +For the 172 shape-comparable examples, maximum pixel absolute difference was +0.0156863928 and minimum correlation was 0.9999995187. Pixel comparisons for +the 28 different-shape examples were not counted as passes. The inspected +upstream Resize attribute contract exposes no rounding-mode override. +Runtime-side alignment remains outside this Mobius change; no tensor flip, +pre-resize workaround, new CLI flag, or ONNX weight rebuild was used. + +The results and rounding diagnosis are retained in +`genai-integration/ai2d-native-rgb-fixed-first200-01/` and +`genai-integration/native-smart-resize-diagnosis.json` on the qualification +artifact disk. Prior Torch/raw-ORT results are frozen references, not fresh +reruns. The existing official FP16 full-logit parity blocker remains open. + +### Input-size and repeated-inference memory findings + +The native AI2D200 runner used one Python process and one Model for the entire +ordered subset, creating and deleting a Generator per question. Its preserved +runner SHA256 is +`e7f0306d524f04d1a527d9470929a447a54edd626e08c8e8afd56012fe0b5541`. +Observed native prompt lengths (including image tokens) have median 314.5 +and maximum 1,705; image patch rows have median 1,064 and maximum 6,580. +Generation was capped at 16 tokens. These are observed subset bounds, not +dataset-wide maxima or a deployment-safe input limit. + +A separate full-resolution large-image diagnostic used 4,875 prompt tokens, +19,200 patch rows, and a 128-token generation cap. With the same Model kept +alive and a fresh Generator per call, the first call completed and exactly +matched the standalone tokens, text, processor tensors, and prompt. The +second call failed at decoder layer 3's `Attention_node_335`, requesting +4,563,000,320 bytes. GPU usage rose from 57,947 MiB after Model loading to +81,133 MiB after the first call, leaving 20 MiB free; deleting the Generator +and collecting garbage did not reduce that usage while the Model remained +alive. Process exit restored 81,150 MiB free and 4 MiB used. + +This diagnostic used an isolated GenAI build at source commit +`265917c09ffe68198a026e086d21e694cc938298` with a one-line last-token logits +allocation correction and package-local `$ORIGIN` RPATH. It retained the +installed ORT library (SHA256 +`5a2e81ad05259bce596e29d6361fe76750b6c2c9569b8cdddf709ab0d944b0ee`), +unchanged ONNX graphs, full-resolution images, and generation policy. +It is not the installed GenAI binary used for AI2D200; that binary's +originating source commit is unknown. The external GenAI correction is +not included in this Mobius PR and did not resolve repeated-inference OOM. + +The exported decoder contains 16 standard Attention nodes with explicit +FP16 masks and dynamic past/present KV ports, and no TensorScatter nodes. +For the failing 4,875-token case, the local ORT unfused workspace formula +matches the requested bytes exactly: +`2 * align256(24 * 4875 * 4875 * sizeof(float)) = 4,563,000,320`. +The inspected CUDA dispatch rejects Flash with an explicit mask and rejects +MEA when a masked KV row stride is not a multiple of four; `4875 % 4 = 3`. +This is strong source-and-size evidence for unfused Attention workspace, +not a captured installed-binary dispatch trace. Exact correspondence of +the inspected ORT source to the installed binary remains unverified. +Other large-image attempts also failed in the vision encoder, so decoder +KV-cache changes alone are not established as a complete remedy. + +Repeated-inference failure is confirmed, but live tensor ownership, arena +retention, and fragmentation have not been distinguished. It is not a proven +KV-cache leak, graph incompatibility, or numerical-parity fix. TensorScatter +static-cache A/B testing has not been performed. Deployment needs validated +post-processor input bounds, generation/concurrency budgets, and repeated +mixed-size request testing; AI2D200 success alone does not qualify those. +No MMMU accuracy result is claimed here. + +Evidence remains under `genai-runtime-fix-01/probes/candidate-case56-repeat01/` +(`receipt.json`, `execution.json`) on the qualification artifact disk, +alongside the immutable AI2D records. No additional benchmark run or graph +change is part of this documentation update. + +### Native numerical diagnostics and reset + +Actual native ORT GenAI was compared with the pinned HF FP16 reference using +native processor snapshots and a native-selected token prefix. HF maintained +its own cache; no HF tensors or states were injected into GenAI. This isolates +model numerics from independent preprocessing and generation, and is not a +replacement for the independently generated AI2D results above. + +The completed cases cover text and AI2D indices 0, 124, and 130, plus repeated +text/image and a fresh-generator text reset. Each completed case compares +prefill and four cached decode forwards: all 248,320 last-position logits +and all 128 convolution, recurrent, and KV outputs. Raw graph logits are +FP16; the GenAI logits API's FP32 promotion was verified separately. + +At unchanged `rtol=atol=1e-2`, all 35 paired forward comparisons failed +full-vocabulary logit tolerance, while all 35 argmax predictions matched. +These include repeated cases, not 35 independent examples. Of 4,480 state +comparisons, 2,907 passed and 1,573 failed. Maximum absolute errors were +0.236328125 for logits and 85.25 for states. There were no shape, dtype, or +nonfinite failures in those numerical comparisons. The first failing cache +layer for text was layer 31; image cases could fail at earlier layers. +Neither this observation nor output-token agreement identifies the root +cause or clears the numerical gate. + +In the repeat run, all 30 directly captured native `position_ids` and +`attention_mask` comparisons exactly matched HF-derived shapes, int64 +dtypes, and values. Recreating the generator after an intervening image +case reproduced all five text logits bitwise, all 640 state hashes, and +the selected tokens. This establishes reset repeatability and input +agreement only for those tested cases. + +The two-image native processor emits two maximum-length patch slots: +1,064 and 864 valid patches become 2,128 stored rows, including 200 zero +padding rows, while the grids describe 1,928 valid patches. Native prefill +executed with finite logits and all expected state schemas, but HF could +not consume that unchanged padded snapshot. Its paired numerical case +remains blocked; this is not proof that the native pipeline is incompatible. +No unpadding, repacking, resizing, or tolerance change was used. + +Receipts, full-vocabulary vectors, input captures, and state hashes are in +`genai-integration/native-numerical-validation-01/attempt-03/`, including +`final-consolidated-receipt.json` and `reset-receipt.json`. The preceding +attempt-02 evidence is preserved unchanged. Official FP16 numerical +qualification remains failed; task-level validation does not waive it. + +### CI disposition for the implementation head + +At implementation head `9e8d6aa88ee16a8db77df1dbe0b1a1e585d99d61`, +CI run `37867805012` did not complete its L4/L5 golden suites. Both +check-run annotations explicitly state that the jobs exceeded the +one-hour execution limit. Cancellation is not a successful numerical +or generation test, and the partial logs also contain failed test markers. + +The exact base is `26a712b210a0a7cd895347f601680e82bbf5e452`. +Its main CI run `37681522335` and scheduled L4/L5 runs `37882646916` and +`37886682810` also ended without completing those suites. All 24 named +L4 failures observed on the PR head have failed markers on that exact +base. For L5, 19 of 20 named head failures have corresponding base +failed markers. This is overlap of observed failures, not proof that +their root causes or complete test outcomes are identical. + +The remaining L5 case is +`test_generation_matches_golden[text-generation/granitemoe-1b]`. +It failed on the PR head, but neither inspected base L5 job reached it +before cancellation. Its baseline disposition is therefore unresolved; +absence from the base failure list is not a base pass. + +The pinned SmolLM FP16 CPU ORT GenAI job was skipped by the PR workflow's +existing configuration: the caller passes `run_real: false`, and the +real lane requires that input or a scheduled/manual event. This is not +evidence that Qwen or the pinned released runtime passed that job. + +The filtered logs, annotations, exact-base comparisons, and unresolved +case coverage are retained in `genai-integration/` as +`ci-disposition-before-mmmu200.json`, +`ci-golden-direct-api-log-proof.json`, +`ci-golden-node-comparison.json`, and +`ci-granitemoe-l5-disposition.json`. This historical disposition must +not be presented as the status of a later documentation/publication head. +Any merge acceptance of incomplete qualification requires an explicit +maintainer decision; this evidence does not declare the PR merge-ready. + +### Reproducing graph checks + +Run from the repository root with Transformers and CUDA ORT available: + +```bash +HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 NVIDIA_TF32_OVERRIDE=0 \ + python -m pytest src/mobius/models/qwen38_standard_test.py -q -rs +``` + +The tests separate these confidence levels: + +1. **Config compatibility:** assert the pinned full text config's layer + schedule, dimensions, interleaved mRoPE sections, and FP32 state policy. +2. **Full-size graph construction:** build the 64-layer decoder and the + complete 64+27-layer VLM in FP32 and FP16, without allocating/loading + checkpoint weights; recursively verify all components' standard-only + nodes and the 48 recurrent/16 KV cache groups. +3. **Reduced random-weight numerical parity:** compare an independently + constructed HF `Qwen3_5ForCausalLM` with the actual exported/saved/reloaded + ONNX package on CUDA FP32 and FP16. Cover single-token and six-token + prefill, batch sizes one and two, then four greedy decode steps. Compare + full logits, exact next-token IDs, and every convolution/recurrent/KV + output, checking shapes and dtypes as well as values. +4. **Reduced VLM numerical parity:** run actual saved/reloaded ONNX vision + outputs through the embedding graph and hybrid decoder. Cover text, + multiple images, video, and mixed image/video prompts in FP32 and FP16, + including two-row media batches with reversed placeholder ordering. + Compare vision features, fused embeddings, full logits, exact greedy + IDs, and all carried states over prefill and four decode steps. + +The focused suite passed 34 tests with no skips in the qualification +environment. The related offline regression selection passed 102 tests +with 16 existing skips. This is a different selection from the later +102-test affected-component run below; their counts are not additive. +The VLM patches are synthetic, nonzero packed +processor-shaped inputs; HF supplies explicit MRoPE positions. This is not +independent qualification of processor preprocessing or host position-ID +generation. + +The reduced model retains one complete 3-DeltaNet/1-attention cycle, the +original full-attention Q/KV head ratio (6:1), the DeltaNet value/key head +ratio (3:1), convolution kernel 4, partial rotary factor 0.25, and interleaved +mRoPE. Widths, vocabulary, context length, and mRoPE sections are reduced; +weights are synthetic. This is L3 evidence, not L4/L5 checkpoint evidence. + +Numerical tolerances are `rtol=atol=1e-3` for FP32 and `1e-2` for FP16. +CUDA profiling must contain actual CUDA matrix execution; ordinary CPU +metadata/control-flow handling remains allowed. Provider registration alone +is not considered proof of GPU computation. CUDA tests explicitly skip when +CUDA ORT/PyTorch is unavailable; such skips are not successful CUDA evidence. + +The test that inserts a custom op inside a Scan body ensures the recursive +standard-only check is exercised, rather than merely checking top-level +nodes. This is a verification assertion, not a claim that main has a generic +fail-closed standard-only production validator. + +## Remaining qualification + +The official 18-shard checkpoint has been downloaded at the pinned revision +(55,562,855,904 tensor bytes). The complete FP16 VLM package, including the +learned-norm correction, was exported and reloaded successfully: +**54,779,987,399 bytes**, with all three components recursively standard-only +and no remaining local functions. CUDA sessions for all three components +loaded successfully. This establishes a real-weight artifact, not numerical +qualification. + +The first export timed out while writing external tensor data. A +production-matrix probe traced slow writes to non-C-contiguous folded +weights and verified byte-identical output after storage conversion. +Operational export preparation copies only strided storage and verifies +every bit, shape and dtype; it does not change graph operators or add a +generic serializer change to Mobius. + +**Official FP16 runtime parity remains failed** at the unchanged +`rtol=atol=1e-2`, using Transformers 5.15.0 and ORT 1.30.0: + +- The image vision comparison exceeded tolerance at 90 of 327,680 values + (0.0275%). The norm correction does not change the vision graph. +- Text embeddings matched exactly. Corrected-source text prefill logits + exceeded tolerance at 79,551 of 5,711,360 values (1.39%), with maximum + absolute error 0.091796875. Validation stopped before greedy-token, + state and reused-cache comparisons. +- Visual-only diagnosis verified all 333 visual tensors and isolated + rounding differences in FP16 projection-plus-bias and rotary arithmetic. + Standard `Gemm` matched the isolated HF QKV projection exactly, unlike + separate `MatMul`/`Add`; FP32 rotary arithmetic also matched locally. + These are candidate remedies, not a completed full-vision fix. + +After the norm correction, the combined affected component and reduced +qualification selection passed 102 tests with no skips. Neither those +passes nor the serialized standard-only audit clear the official failures. +The two additional DeepStack sentinel cases also passed. + +Before declaring the official model production-qualified, compare real-weight +vision/fusion, prefill and multiple reused-state decode steps, full logits +and state, and multi-token generation. Native ORT GenAI driver execution and +AI2D first-200 evaluation have completed, as documented above; this bounded +execution/task evidence does not clear the official FP16 numerical parity +failures. Full-dataset AI2D accuracy and real-processor video/mixed-media +remain unqualified. Full-size runtime memory and throughput require +deployment-scale qualification beyond graph build, reduced-model correctness, +and the recorded bounded runs. Repeated large-image inference still fails +with OOM, as documented below. ModelBuilder equivalence remains deferred. + +In particular, mixed FP16 convolution/FP32 recurrent state must be qualified +against the intended GenAI runtime separately. Quantized text-only component +plans and their module exclusions are outside this unquantized qualification. +BF16, MTP, and performance qualification are also outside the current evidence. + +## Input-size and repeated-inference findings (October 11, 2026) + +The completed native AI2D200 result remains **174/200 (87%)**, compared with frozen HF **175/200 (87.5%)** and raw ORT **174/200 (87%)**. Native/HF tokens matched 199/200 and native/raw tokens 198/200. AI2D used **one process and one Model for all 200 questions**, with a fresh Generator per question; it was not a per-question process-isolation run. + +| Observed native AI2D200 input | Median | Maximum | +|---|---:|---:| +| Prompt tokens, including image tokens | 314.5 | 1,705 | +| Image patch rows | 1,064 | 6,580 | + +Generation was capped at 16 tokens. These are bounds of the tested subset, **not dataset-wide maxima or a validated deployment-safe limit**. + +A separate large-image memory diagnostic used 4,875 prompt tokens, 19,200 patch rows, and a 128-token cap. The first call completed; a second identical call on the same Model with a new Generator failed at decoder layer 3 `Attention_node_335`, requesting **4,563,000,320 bytes**. After the first call, GPU usage was 81,133 MiB with only 20 MiB free; deleting the Generator and GC did not reduce usage while the Model stayed alive. Process exit restored 81,150 MiB free / 4 MiB used. The first call exactly matched the standalone tokens, text, processor tensors, and prompt. + +This diagnostic used an **isolated GenAI build**, source `265917c09ffe68198a026e086d21e694cc938298`, with a one-line last-token logits allocation correction and package-local `$ORIGIN` RPATH; actual loaded installed ORT SHA256 remained `5a2e81ad05259bce596e29d6361fe76750b6c2c9569b8cdddf709ab0d944b0ee`. Graphs, weights, images, and generation policy were unchanged. It is not the installed GenAI binary used for AI2D200; the installed GenAI source origin is unknown. **The external GenAI correction is not included in this Mobius PR and does not resolve repeated-inference OOM.** + +The decoder export has 16 standard Attention nodes, explicit FP16 masks, dynamic KV ports, and no TensorScatter. The inspected ORT unfused workspace formula exactly matches the allocation: `2 * align256(24 * 4875 * 4875 * 4) = 4,563,000,320`. The local dispatch excludes Flash with an explicit mask and excludes MEA for a masked KV stride not divisible by four (`4875 % 4 = 3`). This strongly supports an unfused-workspace attribution, **not a directly captured installed-binary dispatch trace**; the inspected ORT source-to-binary correspondence remains unverified. Other large-image attempts also failed in the vision encoder. + +The repeated-inference memory blocker is reproducible, but live tensor ownership, arena retention, and fragmentation are not yet distinguished. **A KV-cache leak is not proven, and static TensorScatter cache has not been A/B tested.** Deployment requires validated processor-output limits, generation/concurrency budgets, and repeated mixed-size request testing, rather than assuming a successful isolated inference or AI2D subset qualifies serving stability. + +Per the narrowed publication scope, **no MMMU accuracy result is claimed**. Existing official FP16 numerical parity failures, native/HF resize differences, and incomplete historical L4/L5 CI remain open; this update does not declare merge/deployment readiness. Detailed evidence and historical CI disposition are recorded in `docs/design/qwen38-27b-standard-onnx.md`. Qualification-disk receipts include `genai-runtime-fix-01/probes/candidate-case56-repeat01/{receipt,execution}.json` and the unchanged AI2D200 artifacts. diff --git a/examples/qwen38_standard_genai.py b/examples/qwen38_standard_genai.py new file mode 100644 index 000000000..05cf8092a --- /dev/null +++ b/examples/qwen38_standard_genai.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Qualify an existing standard Qwen3.8 VLM with the native GenAI pipeline. + +Never rebuilds/saves ONNX weights. The destination is a fresh metadata overlay: +component directories are symlinks; mutable HF metadata/tokenizers are copies. +Generation is gated on the tiny native hybrid progression/reset receipt. +The proof is a local operator attestation; rerun after any runtime binary change. +CUDA is selected at execution through og.Config, not a Mobius CLI flag. +""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import inspect +import json +import os +import shutil +from pathlib import Path +from typing import Any + +import onnx_ir as ir + +from mobius._configs import ArchitectureConfig +from mobius._model_package import ModelPackage +from mobius.integrations.ort_genai import write_ort_genai_config + +_COMPONENTS = ("decoder", "vision_encoder", "embedding") +_REVISION = "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0" +_CONFIG_SHA256 = "191e0af232104ed8b65258cf3fb2b842e288008baca7633c11b82a1ac7203aab" + + +def _hash(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _disable_core_dumps() -> None: + """Disable POSIX core dumps to avoid huge GPU crash artifacts.""" + if os.name == "posix": + import resource + + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + + +def prepare_overlay(source: Path, assets: Path, output: Path) -> dict[str, Any]: + """Reconstruct a lazy package without a manifest or tensor materialization.""" + from transformers import AutoConfig + + for path in (source, assets, output): + if not path.is_absolute(): + raise ValueError(f"Use absolute paths, got {path}") + if output.exists() or output.is_symlink(): + raise FileExistsError(f"Overlay must be fresh, refusing to overwrite {output}") + source = source.resolve(strict=True) + assets = assets.resolve(strict=True) + output = output.resolve() + if output == source or source in output.parents or assets in output.parents: + raise ValueError("Overlay must not be inside an immutable source directory") + if _hash(assets / "config.json") != _CONFIG_SHA256: + raise ValueError("HF config does not match the pinned Qwen3.8-27B revision") + hf_config = AutoConfig.from_pretrained(str(assets), local_files_only=True) + config = ArchitectureConfig.from_transformers( + hf_config.text_config, parent_config=hf_config + ) + # ir.load keeps external tensors lazy. Do not call const_value.numpy(), + # ModelPackage.load(), save(), or the HF model constructor here. + models = {name: ir.load(source / name / "model.onnx") for name in _COMPONENTS} + for name, model in models.items(): + if model.functions or any( + node.domain not in {"", "ai.onnx"} for node in model.graph.all_nodes() + ): + raise ValueError(f"{name} is not function-free standard ONNX") + decoder = models["decoder"] + inputs = {value.name: value for value in decoder.graph.inputs} + outputs = {value.name: value for value in decoder.graph.outputs} + kv = sorted(int(name.split(".")[1]) for name in inputs if name and name.endswith(".key")) + if kv != list(range(3, 64, 4)): + raise ValueError(f"Unexpected global KV indices: {kv}") + for index in range(64): + roles = ( + (("key", ir.DataType.FLOAT16), ("value", ir.DataType.FLOAT16)) + if index % 4 == 3 + else (("conv_state", ir.DataType.FLOAT16), ("recurrent_state", ir.DataType.FLOAT)) + ) + for role, dtype in roles: + if ( + inputs[f"past_key_values.{index}.{role}"].dtype != dtype + or outputs[f"present.{index}.{role}"].dtype != dtype + ): + raise ValueError(f"Wrong dtype for global slot {index}.{role}") + if inputs["position_ids"].shape[0] != 3: + raise ValueError("Expected native 3D MRoPE positions") + graph_stats = { + name: {"bytes": (source / name / "model.onnx").stat().st_size, "functions": 0} + for name in _COMPONENTS + } + asset_names = ( + "config.json", + "generation_config.json", + "preprocessor_config.json", + "video_preprocessor_config.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt", + "chat_template.jinja", + ) + asset_hashes = {name: _hash(assets / name) for name in asset_names} + output.mkdir(parents=True) + for name in _COMPONENTS: + (output / name).symlink_to(source / name, target_is_directory=True) + # Copy even large tokenizers (small relative to weights): asset patches and + # the writer must never follow a symlink back into the pinned original. + local_assets = output / "hf-assets" + local_assets.mkdir() + for filename in asset_hashes: + shutil.copy2(assets / filename, local_assets / filename) + package = ModelPackage(models, config=config) + result = write_ort_genai_config( + package, + str(output), + hf_model_id=str(local_assets), + # This is runtime metadata, not graph construction: every graph above + # remains standard-only. Explicit auxiliary CPU sessions do not follow + # Config.append_provider in the installed native multimodal pipeline. + ep="cuda", + ) + payload = json.loads((output / "genai_config.json").read_text()) + if payload["model"]["type"] != "qwen3_5": + raise ValueError("Expected native Qwen3.5 multimodal dispatch") + if payload["model"]["decoder"]["num_hidden_layers"] != 64: + raise ValueError("Runtime config compacted sparse global cache slots") + if payload["search"]["past_present_share_buffer"] is not False: + raise ValueError("Standard dynamic KV caches must not share buffers") + for name in ("decoder", "vision", "embedding"): + providers = payload["model"][name]["session_options"]["provider_options"] + if providers[0]["cuda"]["enable_cuda_graph"] != "0": + raise ValueError("CUDA runtime overlay must keep graph capture disabled") + for filename, digest in asset_hashes.items(): + if _hash(assets / filename) != digest: + raise RuntimeError(f"Immutable source asset changed: {filename}") + receipt = { + "revision": _REVISION, + "source": str(source), + "assets": str(assets), + "overlay": str(output), + "global_kv_indices": kv, + "graphs": graph_stats, + "original_asset_hashes": asset_hashes, + "artifacts": result, + } + (output / "overlay-receipt.json").write_text( + json.dumps(receipt, indent=2), encoding="utf-8" + ) + return receipt + + +def _generate_native( + model: Any, + processor: Any, + tokenizer: Any, + prompt: str, + images: Any, + max_new_tokens: int, +) -> list[int]: + import onnxruntime_genai as og + + inputs = processor(prompt, images=images) if images is not None else processor(prompt) + params = og.GeneratorParams(model) + # Count native-expanded image placeholders, not the unexpanded prompt. + prompt_length = int(inputs["input_ids"].as_numpy().size) + params.set_search_options( + max_length=prompt_length + max_new_tokens, + do_sample=False, + past_present_share_buffer=False, + ) + generator = og.Generator(model, params) + print("Native prefill:", prompt_length, "tokens; image:", images is not None, flush=True) + generator.set_inputs(inputs) + print("Native prefill completed", flush=True) + tokens: list[int] = [] + for _ in range(max_new_tokens): + if generator.is_done(): + break + generator.generate_next_token() + tokens.append(int(generator.get_next_tokens()[0])) + if not tokens: + raise RuntimeError("Native generation produced no tokens") + print("Native tokens:", tokens, "decoded:", tokenizer.decode(tokens), flush=True) + del generator, params, inputs + gc.collect() + return tokens + + +def qualify( + output: Path, + proof: Path, + image: Path | None, + max_new_tokens: int, + second_image: Path | None = None, +) -> None: + """Run native text/reset and optional single-/two-image decoding. + + ``second_image`` is a programmatic optional extension, not a new CLI flag. + """ + if not 1 <= max_new_tokens <= 16: + raise ValueError("max_new_tokens must be between 1 and 16") + if second_image is not None and image is None: + raise ValueError("second_image requires the first image") + + import onnxruntime_genai as og + + receipt = json.loads(proof.read_text()) + expected = { + "genai_version": og.__version__, + "provider": "CUDAExecutionProvider", + "sparse_kv_indices": [3, 7], + "conv_dtype": "float16", + "recurrent_dtype": "float32", + "past_present_share_buffer": False, + "progression_and_reset": [[0, 4, 8], [0, 4, 8]], + } + if receipt != expected: + raise ValueError( + "A passing tiny native proof with matching reported runtime version is required" + ) + config = og.Config(str(output)) + config.clear_providers() + config.append_provider("CUDAExecutionProvider") + print("Loading native real VLM:", output, flush=True) + model = og.Model(config) + print("Native real VLM loaded", flush=True) + tokenizer = og.Tokenizer(model) + processor = model.create_multimodal_processor() + + text = ( + "<|im_start|>user\nThe capital of France is<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n" + ) + try: + first = _generate_native(model, processor, tokenizer, text, None, max_new_tokens) + reset = _generate_native(model, processor, tokenizer, text, None, max_new_tokens) + if first != reset: + raise AssertionError(f"Fresh-generator text reset differs: {first} vs {reset}") + results: dict[str, Any] = {"text": first, "reset": reset} + if image is not None: + images = og.Images.open(str(image)) + prompt = ( + "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" + "Describe this image briefly.<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n" + ) + results["image"] = _generate_native( + model, processor, tokenizer, prompt, images, max_new_tokens + ) + del images + if second_image is not None: + images = og.Images.open(str(image), str(second_image)) + prompt = ( + "<|im_start|>user\n" + "<|vision_start|><|image_pad|><|vision_end|>" + "<|vision_start|><|image_pad|><|vision_end|>" + "Describe these two images briefly.<|im_end|>\n" + "<|im_start|>assistant\n\n\n\n\n" + ) + results["multi_image"] = _generate_native( + model, processor, tokenizer, prompt, images, max_new_tokens + ) + del images + (output / "native-results.json").write_text( + json.dumps({"genai_version": og.__version__, **results}, indent=2), + encoding="utf-8", + ) + finally: + del processor, tokenizer, model + gc.collect() + + +def main() -> None: + import mobius + + _disable_core_dumps() + print("Mobius:", inspect.getfile(mobius), flush=True) + if not Path(inspect.getfile(mobius)).is_relative_to( + Path(__file__).resolve().parents[1] / "src" + ): + raise RuntimeError("Mobius import must belong to this example's checkout") + if os.environ.get("HF_HUB_OFFLINE") != "1": + raise RuntimeError("Set HF_HUB_OFFLINE=1; this example never downloads assets") + if not os.environ.get("CUDA_VISIBLE_DEVICES", "").startswith("GPU-"): + raise RuntimeError("Select one explicit GPU UUID through CUDA_VISIBLE_DEVICES") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("assets", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--prepare-only", action="store_true") + parser.add_argument("--native-proof", type=Path) + parser.add_argument("--image", type=Path) + parser.add_argument("--max-new-tokens", type=int, default=4) + args = parser.parse_args() + if not 1 <= args.max_new_tokens <= 16: + parser.error("--max-new-tokens must be between 1 and 16") + if not args.prepare_only and args.native_proof is None: + parser.error("--native-proof is required before loading the real native model") + print(json.dumps(prepare_overlay(args.source, args.assets, args.output), indent=2)) + if not args.prepare_only: + qualify(args.output, args.native_proof, args.image, args.max_new_tokens) + + +if __name__ == "__main__": + main() diff --git a/src/mobius/_configs/_base.py b/src/mobius/_configs/_base.py index 61a7e15ee..622d5eb52 100644 --- a/src/mobius/_configs/_base.py +++ b/src/mobius/_configs/_base.py @@ -569,6 +569,8 @@ class ArchitectureConfig(BaseModelConfig): linear_num_value_heads: int | None = None linear_gate_lower_bound: float | None = None linear_use_full_rank_gate: bool = False + # None preserves model-dtype state for checkpoints without an explicit policy. + mamba_ssm_dtype: ir.DataType | None = None # Double-gated short-convolution config (LFM2-style hybrid layers). short_conv_kernel: int = 3 @@ -892,6 +894,15 @@ def from_transformers( model_type = config.model_type rope_config = _extract_rope_config(config) mrope_fields = _extract_mrope_fields(config) + raw_ssm_dtype = getattr(config, "mamba_ssm_dtype", None) + ssm_dtype = _resolve_dtype_value(raw_ssm_dtype) + if raw_ssm_dtype not in (None, "auto") and ssm_dtype not in { + ir.DataType.FLOAT, + ir.DataType.FLOAT16, + ir.DataType.BFLOAT16, + ir.DataType.DOUBLE, + }: + raise ValueError(f"Unrecognized mamba_ssm_dtype: {raw_ssm_dtype!r}") # Models that use RoPE but don't expose rope_scaling/rope_parameters # in their HF config (e.g. loaded without trust_remote_code, or @@ -1043,6 +1054,7 @@ def _per_layer_value(attribute: str) -> int | None: linear_value_head_dim=(getattr(config, "linear_value_head_dim", None)), linear_num_key_heads=(getattr(config, "linear_num_key_heads", None)), linear_num_value_heads=(getattr(config, "linear_num_value_heads", None)), + mamba_ssm_dtype=ssm_dtype, short_conv_kernel=getattr(config, "conv_L_cache", 3), short_conv_bias=getattr(config, "conv_bias", False), pad_token_id=(getattr(config, "pad_token_id", 0)), diff --git a/src/mobius/_registry.py b/src/mobius/_registry.py index 1dde0d8cb..930991813 100644 --- a/src/mobius/_registry.py +++ b/src/mobius/_registry.py @@ -1193,6 +1193,10 @@ def _create_default_registry() -> ModelRegistry: "gemma4_text": "gemma4_text", "gemma4_unified": "gemma4_unified_text", "gemma4_unified_text": "gemma4_unified_text", + "qwen3_5": "qwen3_5_text", + "qwen3_5_vl": "qwen3_5_text", + "qwen3_5_text": "qwen3_5_text", + "qwen3_5_vl_text": "qwen3_5_text", # Qwen3.5-MoE-VL (Qwen3.6-35B-A3B): export just the hybrid MoE text # backbone as a standalone decoder-only LLM. The builder overrides # ``qwen3_5_moe`` -> ``qwen3_5_moe_vl`` when a ``vision_config`` is present, diff --git a/src/mobius/_registry_test.py b/src/mobius/_registry_test.py index a4aab61a7..d2545e60c 100644 --- a/src/mobius/_registry_test.py +++ b/src/mobius/_registry_test.py @@ -207,6 +207,14 @@ def test_mapping_is_idempotent(self): def test_gemma4_resolves_to_text_backbone(self, model_type): assert _TEXT_ONLY_MODEL_TYPE[model_type] == "gemma4_text" + @pytest.mark.parametrize( + "model_type", ["qwen3_5", "qwen3_5_vl", "qwen3_5_text", "qwen3_5_vl_text"] + ) + def test_qwen35_resolves_to_text_backbone(self, model_type): + target = _TEXT_ONLY_MODEL_TYPE[model_type] + assert target == "qwen3_5_text" + assert _TEXT_ONLY_MODEL_TYPE[target] == target + def test_nested_text_config_type_is_mapped(self): """A VL type and its nested ``text_config`` type must agree. diff --git a/src/mobius/components/_castlike_dtype_test.py b/src/mobius/components/_castlike_dtype_test.py index cc97f3de5..8fbfc6a96 100644 --- a/src/mobius/components/_castlike_dtype_test.py +++ b/src/mobius/components/_castlike_dtype_test.py @@ -36,20 +36,17 @@ class TestPythonLiteralAutocast: """Verify that Python float literals auto-cast to match tensor operand dtypes.""" @pytest.mark.parametrize("dtype", [ir.DataType.FLOAT16, ir.DataType.BFLOAT16]) - def test_offset_rms_norm_constant_autocasts(self, dtype: ir.DataType): - """OffsetRMSNorm: `1.0` literal in op.Add auto-casts to weight dtype. - - After _cast_module_dtype, the weight param is BF16/FP16. The Python - literal 1.0 in op.Add(self.weight, 1.0) must auto-cast to match. - """ + def test_offset_rms_norm_preserves_fp32_scale(self, dtype: ir.DataType): + """Learned offsets use FP32 internally; only the final output is cast.""" norm = OffsetRMSNorm(hidden_size=64, eps=1e-6) _cast_module_dtype(norm, dtype) builder_, op, graph = create_test_builder() x = create_test_input(builder_, "x", [2, 3, 64], dtype) result = norm(op, x) graph.outputs.append(result) - # No CastLike — auto-cast handles it; output stays in the expected dtype - assert count_op_type(graph, "CastLike") == 0 + rms = next(node for node in graph if node.op_type == "RMSNormalization") + assert rms.inputs[0].dtype == ir.DataType.FLOAT + assert rms.inputs[1].dtype == ir.DataType.FLOAT assert _get_output_dtype(graph) == dtype @pytest.mark.parametrize("dtype", [ir.DataType.FLOAT16, ir.DataType.BFLOAT16]) @@ -82,7 +79,6 @@ def test_float32_inputs_produce_float32_output(self): x = create_test_input(builder_, "x", [2, 3, 64], ir.DataType.FLOAT) result = norm(op, x) graph.outputs.append(result) - assert count_op_type(graph, "CastLike") == 0 assert _get_output_dtype(graph) == ir.DataType.FLOAT diff --git a/src/mobius/components/_gated_deltanet.py b/src/mobius/components/_gated_deltanet.py index 44be2185d..55ea823c7 100644 --- a/src/mobius/components/_gated_deltanet.py +++ b/src/mobius/components/_gated_deltanet.py @@ -123,7 +123,7 @@ def __init__(self, config: ArchitectureConfig, linear_class: type | None = None) linear_class = Linear self.hidden_size = config.hidden_size self._dtype = config.dtype - self._stash_type = getattr(config, "mamba_ssm_dtype", config.dtype) + self._stash_type = getattr(config, "mamba_ssm_dtype", None) or config.dtype self._qk_l2norm_eps = getattr(config, "linear_qk_l2norm_eps", 0.0) self.num_v_heads = config.linear_num_value_heads self.num_k_heads = config.linear_num_key_heads @@ -290,6 +290,9 @@ def forward( _domain="com.microsoft", _outputs=2, ) + # The contrib op lacks a standard schema for builder type propagation. + output_3d.type = ir.TensorType(self._stash_type) + new_recurrent_state.type = ir.TensorType(self._stash_type) # output_3d: (B, T, num_v_heads * d_v) — already 3D if self._stash_type != self._dtype: output_3d = op.CastLike(output_3d, hidden_states) diff --git a/src/mobius/components/_gated_deltanet_test.py b/src/mobius/components/_gated_deltanet_test.py index fb72fee63..93039de17 100644 --- a/src/mobius/components/_gated_deltanet_test.py +++ b/src/mobius/components/_gated_deltanet_test.py @@ -5,6 +5,10 @@ from __future__ import annotations +import onnx_ir as ir +import pytest + +from mobius._builder import _cast_module_dtype from mobius._testing import ( count_op_type, create_test_builder, @@ -14,6 +18,33 @@ from mobius.components._gated_deltanet import GatedDeltaNet +@pytest.mark.parametrize( + "dtype", [ir.DataType.FLOAT, ir.DataType.FLOAT16, ir.DataType.BFLOAT16] +) +@pytest.mark.parametrize("state_dtype", [None, ir.DataType.FLOAT]) +def test_gated_deltanet_propagates_activation_and_state_dtypes(dtype, state_dtype): + config = make_config( + dtype=dtype, + mamba_ssm_dtype=state_dtype, + linear_num_value_heads=4, + linear_num_key_heads=2, + linear_key_head_dim=16, + linear_value_head_dim=16, + linear_conv_kernel_dim=4, + ) + module = GatedDeltaNet(config) + _cast_module_dtype(module, dtype) + builder, op, graph = create_test_builder() + hidden = create_test_input(builder, "hidden", [1, 1, 64], dtype) + conv = create_test_input(builder, "conv_state", [1, 128, 3], dtype) + recurrent = create_test_input(builder, "rec_state", [1, 4, 16, 16], state_dtype or dtype) + output, _, new_recurrent = module(op, hidden, conv, recurrent) + attention = next(node for node in graph if node.op_type == "LinearAttention") + assert all(value.dtype == (state_dtype or dtype) for value in attention.outputs) + assert new_recurrent.dtype == (state_dtype or dtype) + assert output.dtype == dtype + + class TestGatedDeltaNet: """Tests for the GatedDeltaNet linear attention module.""" diff --git a/src/mobius/components/_rms_norm.py b/src/mobius/components/_rms_norm.py index 1888880d9..1b70e0ab5 100644 --- a/src/mobius/components/_rms_norm.py +++ b/src/mobius/components/_rms_norm.py @@ -51,13 +51,30 @@ def __init__(self, hidden_size: int, eps: float = 1e-6): self.variance_epsilon = eps def forward(self, op: OpBuilder, hidden_states: ir.Value): - effective_weight = op.Add(self.weight, 1.0) - return op.RMSNormalization( - hidden_states, + # Keep the learned offset and scaling in FP32; rounding 1 + weight + # in model dtype loses small trained offsets before multiplication. + hidden_f32 = ( + hidden_states + if hidden_states.dtype == ir.DataType.FLOAT + else op.Cast(hidden_states, to=ir.DataType.FLOAT) + ) + weight_f32 = ( + self.weight + if self.weight.dtype == ir.DataType.FLOAT + else op.Cast(self.weight, to=ir.DataType.FLOAT) + ) + effective_weight = op.Add(weight_f32, 1.0) + normalized = op.RMSNormalization( + hidden_f32, effective_weight, epsilon=self.variance_epsilon, axis=-1, ) + return ( + normalized + if hidden_states.dtype == ir.DataType.FLOAT + else op.CastLike(normalized, hidden_states) + ) class GatedRMSNorm(nn.Module): diff --git a/src/mobius/components/_rms_norm_test.py b/src/mobius/components/_rms_norm_test.py index 6c1ce9551..e1ab95f42 100644 --- a/src/mobius/components/_rms_norm_test.py +++ b/src/mobius/components/_rms_norm_test.py @@ -5,10 +5,52 @@ from __future__ import annotations +import numpy as np +import onnx_ir as ir +import onnxruntime as ort +import pytest from onnxscript import nn +from mobius._builder import _cast_module_dtype from mobius._testing import count_op_type, create_test_builder, create_test_input -from mobius.components._rms_norm import RMSNorm, apply_rms_norm +from mobius.components._rms_norm import OffsetRMSNorm, RMSNorm, apply_rms_norm + + +@pytest.mark.parametrize("dtype", [ir.DataType.FLOAT, ir.DataType.FLOAT16]) +@pytest.mark.parametrize("unknown_input_type", [False, True]) +def test_offset_rms_norm_learned_scale_precision(tmp_path, dtype, unknown_input_type): + """Keep the learned offset and scale multiplication in FP32 until output cast.""" + values = np.asarray([[1.0, 3.0, 6.0, 10.0]], dtype=dtype.numpy()) + weight = np.asarray([0.0004, -0.0004, 0.0003, -0.0003], dtype=dtype.numpy()) + norm = OffsetRMSNorm(4, eps=1e-6) + _cast_module_dtype(norm, dtype) + norm.weight = nn.Parameter([4], data=ir.Tensor(weight)) + builder, op, graph = create_test_builder() + x = create_test_input(builder, "x", [1, 4], dtype) + hidden = op.Identity(x) if unknown_input_type else x + if unknown_input_type: + hidden.type = None + result = norm(op, hidden) + result.type = ir.TensorType(dtype) + graph.outputs.append(result) + if dtype == ir.DataType.FLOAT and not unknown_input_type: + assert count_op_type(graph, "Cast") == 0 + assert count_op_type(graph, "CastLike") == 0 + else: + assert count_op_type(graph, "Cast") >= 1 + assert count_op_type(graph, "CastLike") == 1 + path = tmp_path / "offset-norm.onnx" + ir.save(ir.Model(graph, ir_version=11), path) + actual = ort.InferenceSession(str(path), providers=["CPUExecutionProvider"]).run( + None, {"x": values} + )[0] + values_f32 = values.astype(np.float32) + expected = ( + values_f32 + / np.sqrt(np.mean(values_f32**2, axis=-1, keepdims=True) + 1e-6) + * (1.0 + weight.astype(np.float32)) + ).astype(dtype.numpy()) + np.testing.assert_allclose(actual, expected, rtol=1e-7, atol=1e-7, strict=True) class TestRMSNorm: diff --git a/src/mobius/integrations/gguf/_builder.py b/src/mobius/integrations/gguf/_builder.py index 834896b7c..272fd17eb 100644 --- a/src/mobius/integrations/gguf/_builder.py +++ b/src/mobius/integrations/gguf/_builder.py @@ -6701,6 +6701,9 @@ def _logical_source_filename(reference: str | Path, resolved_path: str | Path) - def _graph_config_fields_for_fingerprint(config, gguf_arch: str) -> dict[str, object]: """Serialize only fields consumed by an architecture's imported graph.""" fields = asdict(config) + if fields.get("mamba_ssm_dtype") is None: + # Inherited model-dtype state preserves the pre-policy fingerprint. + fields.pop("mamba_ssm_dtype", None) if not fields.get("qmoe_source_paths"): # This field postdates pinned GGUF routes; its empty default must preserve their hashes. fields.pop("qmoe_source_paths", None) diff --git a/src/mobius/integrations/gguf/_builder_core_test.py b/src/mobius/integrations/gguf/_builder_core_test.py index a754e3d7b..dfdd8157c 100644 --- a/src/mobius/integrations/gguf/_builder_core_test.py +++ b/src/mobius/integrations/gguf/_builder_core_test.py @@ -130,6 +130,35 @@ def test_existing_routes_ignore_empty_qmoe_source_paths() -> None: assert changed != legacy +@pytest.mark.parametrize("architecture", ["llama", "qwen2", "lfm2", "qwen35", "qwen35moe"]) +def test_existing_routes_ignore_inherited_recurrent_dtype(architecture: str) -> None: + from mobius._testing import make_config + from mobius.integrations.gguf._builder import _serialize_route_graph_config + + payload = _serialize_route_graph_config(make_config(), architecture) + assert "mamba_ssm_dtype" not in json.loads(payload) + + +@pytest.mark.parametrize("architecture", ["qwen35", "qwen35moe", "qwen3next", "qwen4exp"]) +def test_routes_retain_explicit_recurrent_dtype(architecture: str) -> None: + from mobius._testing import make_config + from mobius.integrations.gguf._builder import _serialize_route_graph_config + + inherited = _serialize_route_graph_config( + make_config(dtype=ir.DataType.FLOAT16), architecture + ) + half_state = _serialize_route_graph_config( + make_config(dtype=ir.DataType.FLOAT16, mamba_ssm_dtype=ir.DataType.FLOAT16), + architecture, + ) + float_state = _serialize_route_graph_config( + make_config(dtype=ir.DataType.FLOAT16, mamba_ssm_dtype=ir.DataType.FLOAT), + architecture, + ) + assert "mamba_ssm_dtype" in json.loads(float_state) + assert len({inherited, half_state, float_state}) == 3 + + class TestReuseGgufWeights: """Tests for mixed GGUF references plus converted ONNX sidecar weights.""" diff --git a/src/mobius/integrations/gguf/_config_mapping.py b/src/mobius/integrations/gguf/_config_mapping.py index beaac253f..2ed26e213 100644 --- a/src/mobius/integrations/gguf/_config_mapping.py +++ b/src/mobius/integrations/gguf/_config_mapping.py @@ -31,6 +31,7 @@ from typing import TYPE_CHECKING, Any import numpy as np +import onnx_ir as ir from mobius._configs import ( ArchitectureConfig, @@ -5308,6 +5309,7 @@ def _qwen4exp_postprocess( mrope_section=[int(value) for value in metadata[f"{prefix}rope.dimension_sections"]], mrope_interleaved=True, norm_topk_prob=True, + mamba_ssm_dtype=ir.DataType.FLOAT, mtp_num_hidden_layers=0, mtp_use_dedicated_embeddings=False, ) diff --git a/src/mobius/integrations/gguf/_moe_cohort_test.py b/src/mobius/integrations/gguf/_moe_cohort_test.py index 45cb02764..b5eb47ec4 100644 --- a/src/mobius/integrations/gguf/_moe_cohort_test.py +++ b/src/mobius/integrations/gguf/_moe_cohort_test.py @@ -81,6 +81,7 @@ def test_new_cohort_fields_preserve_existing_route_fingerprint_bytes( *_SPECIALIZED_ENCODER_FINGERPRINT_FIELDS, "attention_clamp", "component_quantization", + "mamba_ssm_dtype", "moe_layer_frequency", "qmoe_source_paths", "routing_weight_normalization_floor", diff --git a/src/mobius/integrations/gguf/_qwen4_exp_test.py b/src/mobius/integrations/gguf/_qwen4_exp_test.py index 9cc8f161f..a8ef4994c 100644 --- a/src/mobius/integrations/gguf/_qwen4_exp_test.py +++ b/src/mobius/integrations/gguf/_qwen4_exp_test.py @@ -8,6 +8,7 @@ from unittest import mock import numpy as np +import onnx_ir as ir import pytest from mobius._configs import Qwen4ExpConfig @@ -171,6 +172,7 @@ def test_qwen4exp_config_mapping_is_exact(): assert config.linear_num_value_heads == 48 assert config.linear_key_head_dim == 128 assert config.linear_value_head_dim == 128 + assert config.mamba_ssm_dtype == ir.DataType.FLOAT assert config.mrope_section == [11, 11, 10] assert config.mrope_interleaved diff --git a/src/mobius/integrations/ort_genai/auto_export.py b/src/mobius/integrations/ort_genai/auto_export.py index 767936227..fa8b235e6 100644 --- a/src/mobius/integrations/ort_genai/auto_export.py +++ b/src/mobius/integrations/ort_genai/auto_export.py @@ -231,6 +231,7 @@ class _DecoderAbi: "glm_ocr_text", "qwen3_vl_text", "qwen3_5", + "qwen3_5_text", "qwen3_5_vl", "qwen3_5_vl_text", "qwen3_5_moe", @@ -244,6 +245,7 @@ class _DecoderAbi: _QWEN35_VL_MODEL_TYPES = frozenset( { "qwen3_5", + "qwen3_5_text", "qwen3_5_vl", "qwen3_5_vl_text", "qwen3_5_moe", @@ -1290,15 +1292,10 @@ def _write_vision_processor_config( } ) elif model_type in _QWEN_VL_MODEL_TYPES: - # Qwen-VL models need the PatchImage transform to extract - # temporal+spatial patches, and qwen2_5_vl/qwen3_vl flag - # on Normalize for correct interleaving. + # DecodeImage(color_space="RGB") already emits RGB. Normalize's + # qwen2_5_vl/qwen3_vl flags perform BGR->RGB swaps, not interleaving: + # keep ordinary RGB normalization. PatchImage owns patch layout. temporal_patch_size = config.temporal_patch_size - # Add qwen3_vl flag to the Normalize step - for t in transforms: - op = t.get("operation", {}) - if op.get("type") == "Normalize": - op.setdefault("attrs", {})["qwen2_5_vl"] = 1 transforms.append( { "operation": { @@ -1581,25 +1578,12 @@ def _write_genai_config( # node, the model supports shared-buffer mode; otherwise force it off # regardless of the EP capability flag. # - # ``com.microsoft.LinearAttention`` (linear/recurrent-attention layers, - # e.g. Qwen3.5's GatedDeltaNet) is a separate, *mandatory* case: its - # recurrent state requires ``past_present_share_buffer=True`` regardless - # of whether any other layer uses GQA (ORT GenAI raises "RecurrentState - # requires past_present_share_buffer=true" otherwise). - # - # Hybrid models mix LinearAttention layers with full-attention layers, - # which may lower to GQA *or* to the standard (non-GQA) ``Attention`` op - # depending on EP/dtype (e.g. the CPU EP only lowers to GQA for fp32; - # fp16 falls back to standard Attention -- see ``_execution_providers.py`` - # ``gqa_dtypes``). If a hybrid graph has LinearAttention but its - # full-attention layers are still standard (non-GQA) Attention, forcing - # ``past_present_share_buffer=True`` produces an unrunnable config: the - # recurrent state requires it, but standard Attention's dynamic-shape KV - # concat cannot honor a pre-allocated shared buffer, which fails at - # generation time with an ``attn_mask``/``total_sequence_length`` - # mismatch rather than at load time. Rather than silently emit a broken - # config, raise a clear error so the caller picks an EP/dtype combination - # (e.g. fp32 on CPU) that lowers full attention to GQA. + # Preserve the legacy sharing policy for fused LinearAttention graphs. + # This is not a universal recurrent-state requirement: native runtime + # versions differ in their support for non-shared recurrent-state swaps. + # A standard-only hybrid graph has neither custom op and therefore emits + # sharing=False, as its dynamic full-attention KV outputs require. Native + # progression/reset must be qualified separately for the installed build. supports_in_place_kv_cache: bool | None = None if decoder_model is not None: has_gqa = any( @@ -2204,7 +2188,9 @@ def write_ort_genai_config( processor_path = _write_vision_processor_config( config, directory, - hf_model_id=hf_model_id, + # --config uses local HF assets too: otherwise the processor silently + # retains CLIP defaults instead of the checkpoint's normalization/bounds. + hf_model_id=hf_model_id or local_config_dir, trust_remote_code=trust_remote_code, revision=revision, ) diff --git a/src/mobius/integrations/ort_genai/auto_export_test.py b/src/mobius/integrations/ort_genai/auto_export_test.py index be595608a..5f2097142 100644 --- a/src/mobius/integrations/ort_genai/auto_export_test.py +++ b/src/mobius/integrations/ort_genai/auto_export_test.py @@ -347,7 +347,37 @@ def test_qwen25_text_config_writes_packed_patch_processor(self, tmp_path): for transform in transforms if transform["operation"]["type"] == "Normalize" ) - assert normalize["attrs"]["qwen2_5_vl"] == 1 + assert set(normalize["attrs"]) == {"mean", "std"} + + @pytest.mark.parametrize("model_type", ["qwen2_5_vl_text", "qwen3_vl", "qwen3_5_text"]) + def test_qwen_rgb_decode_uses_ordinary_normalize(self, tmp_path, model_type): + """RGB decoding must not be followed by a BGR-to-RGB Normalize swap.""" + config = types.SimpleNamespace( + model_type=model_type, + vision=types.SimpleNamespace(image_size=448, patch_size=16, spatial_merge_size=2), + spatial_merge_size=2, + temporal_patch_size=2, + ) + path = _write_vision_processor_config(config, str(tmp_path)) + assert path is not None + transforms = json.loads(Path(path).read_text())["processor"]["transforms"] + assert [item["operation"]["type"] for item in transforms] == [ + "DecodeImage", + "Resize", + "Rescale", + "Normalize", + "PatchImage", + ] + assert transforms[0]["operation"]["attrs"] == {"color_space": "RGB"} + assert transforms[3]["operation"]["attrs"] == { + "mean": [0.48145466, 0.4578275, 0.40821073], + "std": [0.26862954, 0.26130258, 0.27577711], + } + assert transforms[4]["operation"]["attrs"] == { + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + } def test_mage_vl_writes_packed_patch_processor(self, tmp_path): vision = types.SimpleNamespace( @@ -386,7 +416,7 @@ def test_mage_vl_writes_packed_patch_processor(self, tmp_path): for transform in transforms if transform["operation"]["type"] == "Normalize" ) - assert normalize["attrs"]["qwen2_5_vl"] == 1 + assert set(normalize["attrs"]) == {"mean", "std"} def test_mage_vl_processor_propagates_trust_remote_code(self, tmp_path): vision = types.SimpleNamespace( @@ -490,8 +520,8 @@ def test_muse_glimmer_uses_packed_qwen_image_pipeline(self, tmp_path): """Muse vision consumes flattened patches and image grid dimensions. No ``ConvertRGB``: it unconditionally swaps R and B, so pairing it with - ``DecodeImage(color_space="RGB")`` would hand the encoder BGR. The - ``qwen2_5_vl`` flag therefore lands on ``Normalize`` at index 3. + ``DecodeImage(color_space="RGB")`` would hand the encoder BGR. Ordinary + normalization preserves RGB; PatchImage owns temporal/spatial packing. """ vision = mock.MagicMock() vision.image_size = 448 @@ -520,7 +550,7 @@ def test_muse_glimmer_uses_packed_qwen_image_pipeline(self, tmp_path): "Normalize", "PatchImage", ] - assert transforms[3]["operation"]["attrs"]["qwen2_5_vl"] == 1 + assert set(transforms[3]["operation"]["attrs"]) == {"mean", "std"} assert transforms[4]["operation"]["attrs"] == { "patch_size": 14, "temporal_patch_size": 2, @@ -554,7 +584,7 @@ def test_qwen35_moe_text_uses_packed_qwen_image_pipeline(self, tmp_path): "Normalize", "PatchImage", ] - assert transforms[3]["operation"]["attrs"]["qwen2_5_vl"] == 1 + assert set(transforms[3]["operation"]["attrs"]) == {"mean", "std"} assert transforms[4]["operation"]["attrs"] == { "patch_size": 16, "temporal_patch_size": 2, diff --git a/src/mobius/integrations/ort_genai/qwen38_standard_test.py b/src/mobius/integrations/ort_genai/qwen38_standard_test.py new file mode 100644 index 000000000..471193e32 --- /dev/null +++ b/src/mobius/integrations/ort_genai/qwen38_standard_test.py @@ -0,0 +1,378 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Offline standard Qwen3.8 VLM packaging and local-asset regressions.""" + +from __future__ import annotations + +import importlib.util +import json +import runpy +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import onnx_ir as ir +import pytest +import transformers + +from mobius.__main__ import build_parser, main +from mobius.integrations.ort_genai import write_ort_genai_config +from mobius.integrations.transformers._builder import build_transformers_model + +_FIXTURE = Path(__file__).resolve().parents[4] / "testdata/configs/qwen3_8-27b.json" + + +def _tiny_config(directory: Path) -> None: + """Keep two 3-DeltaNet/1-KV cycles, MRoPE, and the FP32 state policy.""" + config = transformers.Qwen3_5Config.from_dict(json.loads(_FIXTURE.read_text())) + text = config.text_config + text.hidden_size = 64 + text.intermediate_size = 128 + text.num_hidden_layers = 8 + text.layer_types = text.layer_types[:8] + text.num_attention_heads = 6 + text.num_key_value_heads = 1 + text.head_dim = 32 + text.vocab_size = 256 + text.linear_num_key_heads = 2 + text.linear_num_value_heads = 6 + text.linear_key_head_dim = 8 + text.linear_value_head_dim = 8 + text.max_position_embeddings = 128 + text.rope_parameters["mrope_section"] = [2, 1, 1] + text.bos_token_id = 1 + text.eos_token_id = 2 + config.image_token_id = 240 + config.video_token_id = 241 + config.vision_start_token_id = 242 + config.vision_end_token_id = 243 + vision = config.vision_config + vision.depth = 1 + vision.hidden_size = 32 + vision.intermediate_size = 64 + vision.num_heads = 2 + vision.out_hidden_size = 64 + vision.num_position_embeddings = 16 + config.save_pretrained(directory) + + +def _assert_abi(model: ir.Model) -> None: + assert not model.functions + assert all(node.domain in {"", "ai.onnx"} for node in model.graph.all_nodes()) + ports = {value.name: value for value in model.graph.inputs} + outputs = {value.name: value for value in model.graph.outputs} + assert ports["inputs_embeds"].dtype == ir.DataType.FLOAT16 + assert ports["position_ids"].shape[0] == 3 + assert {name for name in ports if name.endswith(".key")} == { + "past_key_values.3.key", + "past_key_values.7.key", + } + for index in range(8): + roles = ( + (("key", ir.DataType.FLOAT16), ("value", ir.DataType.FLOAT16)) + if index % 4 == 3 + else (("conv_state", ir.DataType.FLOAT16), ("recurrent_state", ir.DataType.FLOAT)) + ) + for role, dtype in roles: + assert ports[f"past_key_values.{index}.{role}"].dtype == dtype + assert outputs[f"present.{index}.{role}"].dtype == dtype + + +@pytest.mark.parametrize("runtime", [None, "ort-genai"]) +def test_tiny_standard_vlm_cli_routes(tmp_path: Path, runtime: str | None) -> None: + source = tmp_path / "hf" + _tiny_config(source) + destination = tmp_path / "package" + argv = [ + "build", + "--config", + str(source), + str(destination), + "--ep", + "onnx-standard", + "--dtype", + "f16", + "--no-weights", + ] + if runtime: + argv.extend(["--runtime", runtime]) + main(argv) + assert {path.name for path in destination.iterdir() if path.is_dir()} == { + "decoder", + "embedding", + "vision_encoder", + } + models = { + name: ir.load(destination / name / "model.onnx") + for name in ("decoder", "embedding", "vision_encoder") + } + _assert_abi(models["decoder"]) + for model in models.values(): + assert not model.functions + assert all(node.domain in {"", "ai.onnx"} for node in model.graph.all_nodes()) + assert models["vision_encoder"].graph.inputs[0].dtype == ir.DataType.FLOAT + metadata = destination / "genai_config.json" + assert metadata.exists() == bool(runtime) + if runtime: + payload = json.loads(metadata.read_text()) + assert payload["model"]["type"] == "qwen3_5" + decoder = payload["model"]["decoder"] + assert decoder["num_hidden_layers"] == 8 # Never compact the two KV slots. + assert decoder["inputs"]["inputs_embeds"] == "inputs_embeds" + assert "input_ids" not in decoder["inputs"] + assert decoder["inputs"]["past_key_names"] == "past_key_values.%d.key" + assert decoder["outputs"]["present_key_names"] == "present.%d.key" + assert payload["search"]["past_present_share_buffer"] is False + for section in ("decoder", "vision", "embedding"): + assert payload["model"][section]["session_options"]["provider_options"] == [] + processor = json.loads((destination / "processor_config.json").read_text()) + assert processor["processor"]["name"] == "qwen2_5_image_processor" + assert processor["processor"]["transforms"][-1]["operation"]["type"] == "PatchImage" + + +def test_no_runtime_ep_cli_flag() -> None: + with pytest.raises(SystemExit): + build_parser().parse_args( + ["build", "--model", "Qwen/Qwen3.8-27B", "/unused", "--runtime-ep", "cuda"] + ) + + +def test_local_hf_processor_source_is_used(tmp_path: Path) -> None: + """Local assets must not silently leave CLIP normalization/resize defaults.""" + source = tmp_path / "hf" + _tiny_config(source) + package = build_transformers_model( + str(source), load_weights=False, dtype="f16", execution_provider="onnx-standard" + ) + processor = mock.Mock() + processor.image_processor = mock.Mock( + image_mean=[0.5, 0.5, 0.5], + image_std=[0.5, 0.5, 0.5], + rescale_factor=1 / 255, + resample=3, + size={"shortest_edge": 65536, "longest_edge": 16777216}, + ) + with mock.patch( + "transformers.AutoProcessor.from_pretrained", return_value=processor + ) as load_processor: + result = write_ort_genai_config( + package, str(tmp_path / "overlay"), local_config_dir=str(source) + ) + load_processor.assert_called_once_with(str(source), trust_remote_code=False) + pipeline = json.loads(Path(result["processor_config"]).read_text()) + operations = { + item["operation"]["type"]: item["operation"].get("attrs", {}) + for item in pipeline["processor"]["transforms"] + } + assert operations["Normalize"]["mean"] == [0.5, 0.5, 0.5] + assert operations["Normalize"]["std"] == [0.5, 0.5, 0.5] + assert set(operations["Normalize"]) == {"mean", "std"} + assert operations["DecodeImage"] == {"color_space": "RGB"} + assert operations["Resize"]["min_pixels"] == 65536 + assert operations["Resize"]["max_pixels"] == 16777216 + assert operations["PatchImage"] == { + "patch_size": 16, + "temporal_patch_size": 2, + "merge_size": 2, + } + + +@pytest.mark.parametrize("local_source", [False, True]) +def test_processor_asset_source_precedence(tmp_path: Path, local_source: bool) -> None: + """An explicit HF source still wins over a fallback local config directory.""" + source = tmp_path / "hf" + _tiny_config(source) + package = build_transformers_model( + str(source), load_weights=False, dtype="f16", execution_provider="onnx-standard" + ) + with ( + mock.patch( + "mobius.integrations.ort_genai.auto_export._write_vision_processor_config", + return_value=None, + ) as writer, + mock.patch("mobius.integrations.ort_genai.auto_export._copy_tokenizer_files"), + mock.patch( + "mobius.integrations.ort_genai.auto_export._load_generation_config", + return_value=None, + ), + mock.patch("mobius.integrations.ort_genai.auto_export._fix_chat_template"), + ): + # Both are local to guarantee offline resolution without mocking AutoConfig. + write_ort_genai_config( + package, + str(tmp_path / "overlay"), + hf_model_id=str(source) if local_source else None, + local_config_dir=str(tmp_path / "fallback") if local_source else None, + ) + assert writer.call_args.kwargs["hf_model_id"] == (str(source) if local_source else None) + + +def test_standard_hybrid_cuda_metadata_keeps_capture_and_sharing_off(tmp_path: Path) -> None: + source = tmp_path / "hf" + _tiny_config(source) + package = build_transformers_model( + str(source), load_weights=False, dtype="f16", execution_provider="onnx-standard" + ) + result = write_ort_genai_config(package, str(tmp_path / "overlay"), ep="cuda") + payload = json.loads(Path(result["genai_config"]).read_text()) + assert payload["search"]["past_present_share_buffer"] is False + for section in ("decoder", "vision", "embedding"): + options = payload["model"][section]["session_options"]["provider_options"] + assert options[0]["cuda"]["enable_cuda_graph"] == "0" + _assert_abi(package["decoder"]) + + +def _qualification_example() -> dict: + return runpy.run_path( + str(Path(__file__).resolve().parents[4] / "examples/qwen38_standard_genai.py") + ) + + +@pytest.mark.parametrize( + "relative_path", + ["examples/qwen38_standard_genai.py", "tests/ort_genai_hybrid_native_test.py"], +) +@pytest.mark.parametrize("loader", ["module", "runpy"]) +def test_qualification_import_without_resource( + relative_path: str, loader: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """Simulate missing Windows resource support, not actual Windows execution.""" + path = Path(__file__).resolve().parents[4] / relative_path + monkeypatch.setenv("MOBIUS_QWEN_NATIVE_PROBE", "0") + with mock.patch.dict("sys.modules", {"resource": None}): + if loader == "runpy": + namespace = runpy.run_path(str(path)) + else: + spec = importlib.util.spec_from_file_location("_qwen_import_boundary", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + namespace = vars(module) + if relative_path.startswith("tests/"): + assert namespace["pytestmark"].name == "skipif" + assert namespace["pytestmark"].args == (True,) + assert callable(namespace["test_native_sparse_mixed_state_progression_and_reset"]) + else: + assert callable(namespace["prepare_overlay"]) + + +@pytest.mark.parametrize( + "relative_path", + ["examples/qwen38_standard_genai.py", "tests/ort_genai_hybrid_native_test.py"], +) +@pytest.mark.parametrize("os_name", ["posix", "nt"]) +def test_core_dump_setup_platform_guard(relative_path: str, os_name: str) -> None: + path = Path(__file__).resolve().parents[4] / relative_path + disable = runpy.run_path(str(path))["_disable_core_dumps"] + resource = mock.Mock(RLIMIT_CORE=4) if os_name == "posix" else None + with ( + mock.patch.dict(disable.__globals__, {"os": SimpleNamespace(name=os_name)}), + mock.patch.dict("sys.modules", {"resource": resource}), + ): + disable() + if resource is not None: + resource.setrlimit.assert_called_once_with(4, (0, 0)) + + +@pytest.mark.parametrize( + "relative_path", + ["examples/qwen38_standard_genai.py", "tests/ort_genai_hybrid_native_test.py"], +) +def test_core_dump_setup_does_not_hide_posix_import_failure(relative_path: str) -> None: + path = Path(__file__).resolve().parents[4] / relative_path + disable = runpy.run_path(str(path))["_disable_core_dumps"] + with ( + mock.patch.dict(disable.__globals__, {"os": SimpleNamespace(name="posix")}), + mock.patch.dict("sys.modules", {"resource": None}), + pytest.raises(ModuleNotFoundError, match="resource"), + ): + disable() + + +def test_overlay_refuses_relative_paths_and_existing_destination(tmp_path: Path) -> None: + prepare = _qualification_example()["prepare_overlay"] + with pytest.raises(ValueError, match="absolute paths"): + prepare(Path("relative"), tmp_path, tmp_path) + source = tmp_path / "source" + source.mkdir() + output = tmp_path / "already-exists" + output.mkdir() + with pytest.raises(FileExistsError, match="refusing to overwrite"): + prepare(source, source, output) + assert list(output.iterdir()) == [] + + +def test_overlay_refuses_dangling_destination_symlink(tmp_path: Path) -> None: + from mobius.integrations.gguf._builder_test_utils import _symlink_or_skip + + prepare = _qualification_example()["prepare_overlay"] + source = tmp_path / "source" + source.mkdir() + target = tmp_path / "missing-target" + output = tmp_path / "dangling-overlay" + _symlink_or_skip(output, target) + with pytest.raises(FileExistsError, match="refusing to overwrite"): + prepare(source, source, output) + assert output.is_symlink() + assert not target.exists() + assert list(source.iterdir()) == [] + + +def test_real_native_load_requires_passing_synthetic_proof(tmp_path: Path) -> None: + qualify = _qualification_example()["qualify"] + proof = tmp_path / "failed-proof.json" + proof.write_text(json.dumps({"progression_and_reset": []})) + runtime = mock.Mock(__version__="test-runtime") + with ( + mock.patch.dict("sys.modules", {"onnxruntime_genai": runtime}), + pytest.raises(ValueError, match="passing tiny native proof"), + ): + qualify(tmp_path, proof, None, 4) + runtime.Config.assert_not_called() + runtime.Model.assert_not_called() + + +@pytest.mark.parametrize("early_done", [False, True]) +def test_example_native_generation_contract(early_done: bool) -> None: + """Unit-check orchestration, not a substitute for native qualification.""" + generate = _qualification_example()["_generate_native"] + runtime = mock.Mock() + model = object() + # Native image preprocessing expands placeholders before this length is used. + tensor = mock.Mock() + tensor.as_numpy.return_value = np.zeros((1, 37), dtype=np.int64) + inputs = {"input_ids": tensor} + processor = mock.Mock(return_value=inputs) + tokenizer = mock.Mock() + generator = runtime.Generator.return_value + if early_done: + generator.is_done.return_value = True + else: + generator.is_done.side_effect = [False, False] + generator.get_next_tokens.side_effect = [np.array([4]), np.array([8])] + with mock.patch.dict("sys.modules", {"onnxruntime_genai": runtime}): + if early_done: + with pytest.raises(RuntimeError, match="produced no tokens"): + generate(model, processor, tokenizer, "prompt", None, 2) + else: + assert generate(model, processor, tokenizer, "prompt", None, 2) == [4, 8] + runtime.GeneratorParams.return_value.set_search_options.assert_called_once_with( + max_length=39, do_sample=False, past_present_share_buffer=False + ) + generator.set_inputs.assert_called_once_with(inputs) + assert generator.generate_next_token.call_count == (0 if early_done else 2) + + +@pytest.mark.parametrize("max_new_tokens", [0, 17]) +def test_example_rejects_unbounded_native_generation(max_new_tokens: int) -> None: + qualify = _qualification_example()["qualify"] + with pytest.raises(ValueError, match="between 1 and 16"): + qualify(Path("/unused"), Path("/unused-proof"), None, max_new_tokens) + + +def test_example_second_image_requires_first_image() -> None: + qualify = _qualification_example()["qualify"] + with pytest.raises(ValueError, match="requires the first image"): + qualify(Path("/unused"), Path("/unused-proof"), None, 4, Path("/second")) diff --git a/src/mobius/models/qwen35.py b/src/mobius/models/qwen35.py index 64297e9a4..14c7600bb 100644 --- a/src/mobius/models/qwen35.py +++ b/src/mobius/models/qwen35.py @@ -118,6 +118,14 @@ def _keeps_modules_float(config: ArchitectureConfig) -> bool: ) +class Qwen35GatedDeltaNet(GatedDeltaNet): + """DeltaNet with the Qwen3.5 reference's per-head L2 normalization epsilon.""" + + def __init__(self, config: ArchitectureConfig, linear_class: type | None = None): + super().__init__(config, linear_class=linear_class) + self._qk_l2norm_eps = 1e-6 + + class Qwen35DecoderLayer(nn.Module): """Qwen3.5 decoder layer with hybrid attention. @@ -153,7 +161,7 @@ def __init__(self, config: ArchitectureConfig, layer_idx: int): linear_class = _linear_factory(config) if self.layer_type == "linear_attention": - self.linear_attn = GatedDeltaNet( + self.linear_attn = Qwen35GatedDeltaNet( config, linear_class=self._linear_attn_class(config, linear_class) ) self.linear_attn.component_quantization_excluded_methods = ( diff --git a/src/mobius/models/qwen38_standard_test.py b/src/mobius/models/qwen38_standard_test.py new file mode 100644 index 000000000..8f6926fd6 --- /dev/null +++ b/src/mobius/models/qwen38_standard_test.py @@ -0,0 +1,676 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Offline Qwen3.8-27B standard export and reduced text/multimodal parity. + +The pinned official config exercises the public text-only and full VLM builders. +Numerical tests use random weights and reduced dimensions, not the 27B checkpoint. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import onnx +import onnx_ir as ir +import onnxruntime as ort +import pytest +import torch +import transformers + +from mobius._configs import ArchitectureConfig +from mobius._model_package import ModelPackage +from mobius.integrations._weight_loading import apply_weights +from mobius.integrations.transformers._builder import build_transformers_model + +_CONFIG_PATH = Path(__file__).resolve().parents[3] / "testdata/configs/qwen3_8-27b.json" + + +def _hf_config(*, reduced: bool = False): + config = transformers.Qwen3_5Config.from_dict(json.loads(_CONFIG_PATH.read_text())) + if reduced: + text = config.text_config + text.hidden_size = 64 + text.intermediate_size = 128 + text.num_hidden_layers = 4 + text.layer_types = text.layer_types[:4] + text.num_attention_heads = 6 + text.num_key_value_heads = 1 + text.head_dim = 32 + text.vocab_size = 256 + text.linear_num_key_heads = 2 + text.linear_num_value_heads = 6 + text.linear_key_head_dim = 8 + text.linear_value_head_dim = 12 + text.max_position_embeddings = 128 + text.rope_parameters["mrope_section"] = [2, 1, 1] + text.bos_token_id = 1 + text.eos_token_id = 2 + return config + + +def _assert_standard(model: ir.Model) -> None: + assert not model.functions, "standard-only export must inline local functions" + nodes = list(model.graph.all_nodes()) + for node in nodes: + assert node.domain in {"", "ai.onnx"}, (node.domain, node.op_type) + onnx.defs.get_schema( + node.op_type, model.opset_imports.get(node.domain, model.opset_imports[""]), "" + ) + + +def _build(config, path: Path, dtype: str, *, text_only: bool = True): + config.save_pretrained(path) + package = build_transformers_model( + str(path), + text_only=text_only, + load_weights=False, + dtype=dtype, + execution_provider="onnx-standard", + ) + if text_only: + assert set(package) == {"model"} + assert package.config.model_type == "qwen3_5_text" + decoder = package["model"] + else: + assert set(package) == {"decoder", "vision_encoder", "embedding"} + assert package.config.vision is not None + decoder = package["decoder"] + assert any(node.op_type == "Scan" for node in decoder.graph.all_nodes()) + for model in package.values(): + _assert_standard(model) + return package + + +def _hf_vl_config(): + config = _hf_config(reduced=True) + config.image_token_id = 240 + config.video_token_id = 241 + config.vision_start_token_id = 242 + config.vision_end_token_id = 243 + vision = config.vision_config + vision.depth = 2 + vision.hidden_size = 32 + vision.intermediate_size = 64 + vision.num_heads = 2 + vision.out_hidden_size = config.text_config.hidden_size + vision.num_position_embeddings = 16 + return config + + +def test_qwen38_official_text_config(): + config = ArchitectureConfig.from_transformers(_hf_config().text_config) + assert config.num_hidden_layers == 64 + assert ( + config.layer_types + == ["linear_attention"] * 3 + + ["full_attention"] * 1 + + (["linear_attention"] * 3 + ["full_attention"]) * 15 + ) + assert (config.hidden_size, config.intermediate_size, config.vocab_size) == ( + 5120, + 17408, + 248320, + ) + assert (config.num_attention_heads, config.num_key_value_heads, config.head_dim) == ( + 24, + 4, + 256, + ) + assert config.mrope_interleaved + assert config.mrope_section == [11, 11, 10] + assert getattr(config, "mamba_ssm_dtype", None) == ir.DataType.FLOAT + + +@pytest.mark.parametrize("dtype", ["f32", "f16"]) +@pytest.mark.parametrize("text_config_only", [False, True]) +def test_qwen38_standard_text_only_build(tmp_path: Path, dtype: str, text_config_only: bool): + config = _hf_config(reduced=True) + if text_config_only: + config = config.text_config + package = _build(config, tmp_path / "config", dtype) + graph = package["model"].graph + assert len([value for value in graph.inputs if "past_key_values." in value.name]) == 8 + for value in [*graph.inputs, *graph.outputs]: + if value.name.endswith(".recurrent_state"): + assert value.dtype == ir.DataType.FLOAT + + +@pytest.mark.parametrize("invalid_dtype", ["not-a-dtype", "int64"]) +def test_qwen38_rejects_invalid_state_dtype(invalid_dtype: str): + config = _hf_config(reduced=True).text_config + config.mamba_ssm_dtype = invalid_dtype + with pytest.raises(ValueError, match="mamba_ssm_dtype"): + ArchitectureConfig.from_transformers(config) + + +@pytest.mark.parametrize("state_dtype", [None, "auto"]) +def test_qwen38_unspecified_state_dtype_preserves_existing_behavior( + tmp_path: Path, state_dtype: str | None +): + config = _hf_config(reduced=True) + if state_dtype is None: + del config.text_config.mamba_ssm_dtype + else: + config.text_config.mamba_ssm_dtype = state_dtype + package = _build(config, tmp_path / "config", "f16") + assert package.config.mamba_ssm_dtype is None + states = [ + value + for value in package["model"].graph.inputs + if value.name.endswith(".recurrent_state") + ] + assert states + assert all(value.dtype == ir.DataType.FLOAT16 for value in states) + + +def test_qwen38_standard_check_rejects_nested_custom_op(tmp_path: Path): + model = _build(_hf_config(reduced=True), tmp_path / "config", "f32")["model"] + scan = next(node for node in model.graph if node.op_type == "Scan") + body = scan.attributes["body"].value + next(node for node in body if node.op_type == "MatMul").domain = "com.microsoft" + with pytest.raises(AssertionError, match=r"com\.microsoft"): + _assert_standard(model) + + +@pytest.mark.integration +@pytest.mark.integration_fast +@pytest.mark.parametrize("dtype", ["f32", "f16"]) +def test_qwen38_full_config_standard_build(tmp_path: Path, dtype: str): + """L2 config/graph evidence only: all 64 layers, without weight allocation.""" + package = _build(_hf_config(), tmp_path / "config", dtype) + graph = package["model"].graph + assert len([value for value in graph.inputs if value.name.endswith(".key")]) == 16 + assert len([value for value in graph.inputs if value.name.endswith(".conv_state")]) == 48 + + +@pytest.mark.parametrize("dtype", ["f32", "f16"]) +def test_qwen38_standard_vl_build_roundtrip(tmp_path: Path, dtype: str): + package = _build(_hf_vl_config(), tmp_path / "config", dtype, text_only=False) + assert {value.name for value in package["embedding"].graph.inputs} == { + "input_ids", + "image_features", + } + assert package["vision_encoder"].graph.inputs[0].dtype == ir.DataType.FLOAT + positions = next( + value for value in package["decoder"].graph.inputs if value.name == "position_ids" + ) + assert positions.shape[0] == 3 + for model in package.values(): + for value in model.graph.initializers.values(): + if value.const_value is None: + value.const_value = ir.Tensor( + np.zeros(tuple(value.shape), dtype=value.dtype.numpy()) + ) + package.save(str(tmp_path / "package"), progress_bar=False) + reloaded = ModelPackage.load(str(tmp_path / "package")) + assert set(reloaded) == set(package) + for model in reloaded.values(): + _assert_standard(model) + + +@pytest.mark.parametrize("with_media", [True, False]) +def test_qwen3_split_deepstack_global_media_order(tmp_path: Path, with_media: bool): + """Shared DeepStack maps use the same global mixed-media indices as main features.""" + from mobius.models.qwen_vl import Qwen3VLEmbeddingModel + from mobius.tasks._base import build_embedding_from_features + + hf_config = _hf_vl_config() + config = ArchitectureConfig.from_transformers(hf_config.text_config) + config.image_token_id = hf_config.image_token_id + config.video_token_id = hf_config.video_token_id + config.dtype = ir.DataType.FLOAT + config.deepstack_visual_indexes = [0, 1] + model = build_embedding_from_features( + Qwen3VLEmbeddingModel(config), + config, + feature_name="image_features", + feature_dim=config.hidden_size, + deepstack=True, + ) + _assert_standard(model) + assert { + name for name, value in model.graph.initializers.items() if value.const_value is None + } == {"embed_tokens.weight"} + table = model.graph.initializers["embed_tokens.weight"] + table.const_value = ir.Tensor(np.zeros(tuple(table.shape), dtype=np.float32)) + tokens = np.asarray([[3, 240, 241, 5], [4, 241, 240, 6]], dtype=np.int64) + main = np.repeat(np.arange(1, 5, dtype=np.float32)[:, None], config.hidden_size, axis=1) + packed = np.concatenate((main, main + 100, main + 200), axis=1) + if not with_media: + tokens = np.asarray([[3, 4, 5, 6], [7, 8, 9, 10]], dtype=np.int64) + packed = packed[:0] + path = tmp_path / "deepstack-embedding.onnx" + ir.save(model, path) + session = ort.InferenceSession(str(path), providers=["CPUExecutionProvider"]) + actual = dict( + zip( + (value.name for value in session.get_outputs()), + session.run(None, {"input_ids": tokens, "image_features": packed}), + strict=True, + ) + ) + expected = np.zeros((2, 4, config.hidden_size), dtype=np.float32) + expected_deepstack = np.zeros((2, 4, 2 * config.hidden_size), dtype=np.float32) + if with_media: + expected[:, 1:3] = main.reshape(2, 2, config.hidden_size) + expected_deepstack[:, 1:3] = packed[:, config.hidden_size :].reshape( + 2, 2, 2 * config.hidden_size + ) + np.testing.assert_array_equal(actual["inputs_embeds"], expected) + np.testing.assert_array_equal(actual["per_layer_inputs"], expected_deepstack) + + +@pytest.mark.integration +@pytest.mark.integration_fast +@pytest.mark.parametrize("dtype", ["f32", "f16"]) +def test_qwen38_full_vl_config_standard_build(tmp_path: Path, dtype: str): + """L2: all 64 decoder and 27 vision layers, without allocating weights.""" + package = _build(_hf_config(), tmp_path / "config", dtype, text_only=False) + graph = package["decoder"].graph + assert len([value for value in graph.inputs if value.name.endswith(".key")]) == 16 + assert len([value for value in graph.inputs if value.name.endswith(".conv_state")]) == 48 + assert all( + value.dtype == ir.DataType.FLOAT + for value in graph.inputs + if value.name.endswith(".recurrent_state") + ) + assert package.config.vision.num_hidden_layers == 27 + assert package.config.vision.out_hidden_size == package.config.hidden_size == 5120 + assert {value.name for value in package["embedding"].graph.inputs} == { + "input_ids", + "image_features", + } + + +def _initial_feeds(model: ir.Model, batch: int) -> dict[str, np.ndarray]: + feeds = {} + for value in model.graph.inputs: + if not value.name.startswith("past_key_values."): + continue + shape = [ + dim if isinstance(dim, int) else batch if i == 0 else 0 + for i, dim in enumerate(value.shape) + ] + feeds[value.name] = np.zeros(shape, dtype=value.dtype.numpy()) + return feeds + + +def _compare_states(actual, cache, config, state_feeds, *, tolerance: float, step: int): + for layer_idx, layer_type in enumerate(config.layer_types): + layer = cache.layers[layer_idx] + if layer_type == "linear_attention": + expected_states = { + "conv_state": layer.conv_states[0][ + ..., -(config.linear_conv_kernel_dim - 1) : + ], + "recurrent_state": layer.recurrent_states[0], + } + else: + expected_states = {"key": layer.keys, "value": layer.values} + for role, value in expected_states.items(): + name = f"present.{layer_idx}.{role}" + np.testing.assert_allclose( + actual[name], + value.cpu().numpy(), + rtol=tolerance, + atol=tolerance, + err_msg=f"{name} step {step}", + strict=True, + ) + state_feeds[f"past_key_values.{layer_idx}.{role}"] = actual[name] + + +def _assert_cuda_matrix(profile, role: str): + assert any( + event.get("args", {}).get("provider") == "CUDAExecutionProvider" + and event["args"].get("op_name") in {"MatMul", "FusedMatMul", "Gemm", "FusedGemm"} + for event in profile + ), f"{role} must execute CUDA matrix operations" + + +@pytest.mark.integration +@pytest.mark.integration_fast +@pytest.mark.parametrize("dtype", ["f32", "f16"]) +@pytest.mark.parametrize("batch,prefill", [(1, 1), (2, 6)]) +def test_qwen38_standard_cuda_hf_parity(tmp_path: Path, dtype: str, batch: int, prefill: int): + """Compare full logits and every carried state at prefill and four decode steps.""" + if ( + "CUDAExecutionProvider" not in ort.get_available_providers() + or not torch.cuda.is_available() + ): + pytest.skip("CUDA ORT and PyTorch are required") + from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5ForCausalLM + + torch.manual_seed(38) + config = _hf_config(reduced=True) + config.text_config._attn_implementation = "eager" + torch_dtype = torch.float32 if dtype == "f32" else torch.float16 + reference = ( + Qwen3_5ForCausalLM._from_config(config.text_config, dtype=torch_dtype).eval().cuda() + ) + package = _build(config, tmp_path / "config", dtype) + model = package["model"] + processed = reference.state_dict() + # Exercise the actual standalone model's HF name adapter. + from mobius.models import Qwen35CausalLMModel + + processed = Qwen35CausalLMModel(package.config).preprocess_weights(dict(processed)) + apply_weights(model, processed) + _assert_standard(model) + package.save(str(tmp_path / "package"), progress_bar=False) + reloaded = ModelPackage.load(str(tmp_path / "package")) + model = reloaded["model"] + _assert_standard(model) + options = ort.SessionOptions() + options.intra_op_num_threads = 1 + options.enable_profiling = True + options.profile_file_prefix = str(tmp_path / "cuda-profile") + session = ort.InferenceSession( + str(tmp_path / "package/model.onnx"), + sess_options=options, + providers=[("CUDAExecutionProvider", {"use_tf32": "0"})], + ) + assert session.get_providers()[0] == "CUDAExecutionProvider" + rng = np.random.default_rng(38) + tokens = rng.integers(3, 256, size=(batch, prefill), dtype=np.int64) + state_feeds = _initial_feeds(model, batch) + cache = None + seen = 0 + output_names = [value.name for value in model.graph.outputs] + tolerance = 1e-3 if dtype == "f32" else 1e-2 + for step in range(5): + length = tokens.shape[1] + mask = np.ones((batch, seen + length), dtype=np.int64) + positions = np.broadcast_to(np.arange(seen, seen + length), tokens.shape).copy() + with torch.no_grad(): + expected = reference( + input_ids=torch.from_numpy(tokens).cuda(), + attention_mask=torch.from_numpy(mask).cuda(), + position_ids=torch.from_numpy(positions).cuda(), + past_key_values=cache, + use_cache=True, + ) + cache = expected.past_key_values + actual = dict( + zip( + output_names, + session.run( + None, + { + "input_ids": tokens, + "attention_mask": mask, + "position_ids": positions, + **state_feeds, + }, + ), + strict=True, + ) + ) + target_logits = expected.logits.cpu().numpy() + np.testing.assert_allclose( + actual["logits"], + target_logits, + rtol=tolerance, + atol=tolerance, + err_msg=f"logits step {step}", + strict=True, + ) + expected_tokens = expected.logits[:, -1].argmax(-1).cpu().numpy() + np.testing.assert_array_equal(actual["logits"][:, -1].argmax(-1), expected_tokens) + _compare_states( + actual, + cache, + config.text_config, + state_feeds, + tolerance=tolerance, + step=step, + ) + seen += length + tokens = expected_tokens[:, None] + profile = json.loads(Path(session.end_profiling()).read_text()) + _assert_cuda_matrix(profile, "decoder") + + +def _vl_inputs(config, case: str, batch: int) -> dict[str, np.ndarray]: + rng = np.random.default_rng(38) + rows = [] + image_grids = [] + video_grids = [] + for row in range(batch): + blocks = [] + if case in {"images", "mixed"}: + image_grids.append([1, 4, 6]) + blocks.append([config.image_token_id] * 6) + if case == "images": + image_grids.append([1, 6, 4]) + blocks.append([config.image_token_id] * 6) + if case in {"video", "mixed"}: + video_grids.append([2, 4, 4]) + blocks.extend([[config.video_token_id] * 4 for _ in range(2)]) + if row % 2: + blocks.reverse() + tokens = [3] + for block in blocks: + tokens.extend( + [config.vision_start_token_id, *block, config.vision_end_token_id, 4] + ) + tokens.append(5) + rows.append(tokens) + result = {"input_ids": np.asarray(rows, dtype=np.int64)} + result["attention_mask"] = np.ones_like(result["input_ids"]) + result["mm_token_type_ids"] = np.where( + result["input_ids"] == config.image_token_id, + 1, + np.where(result["input_ids"] == config.video_token_id, 2, 0), + ).astype(np.int32) + vision = config.vision_config + patch_dim = vision.in_channels * vision.temporal_patch_size * vision.patch_size**2 + for grids, grid_name, pixel_name in ( + (image_grids, "image_grid_thw", "pixel_values"), + (video_grids, "video_grid_thw", "pixel_values_videos"), + ): + if grids: + grid = np.asarray(grids, dtype=np.int64) + result[grid_name] = grid + result[pixel_name] = rng.standard_normal( + (int(np.prod(grid, axis=1).sum()), patch_dim) + ).astype(np.float32) + return result + + +def _pack_vl_features(tokens, features, config): + """Pack actual ONNX image/video rows in flattened placeholder-token order.""" + streams = { + config.image_token_id: features["image_features"], + config.video_token_id: features["video_features"], + } + offsets = dict.fromkeys(streams, 0) + packed = [] + for token in tokens.flat: + if token in streams: + packed.append(streams[token][offsets[token]]) + offsets[token] += 1 + assert all(offsets[token] == len(stream) for token, stream in streams.items()) + return np.stack(packed) if packed else features["image_features"] + + +@pytest.mark.integration +@pytest.mark.integration_fast +@pytest.mark.parametrize("dtype", ["f32", "f16"]) +@pytest.mark.parametrize( + "case,batch", + [ + ("text", 1), + ("images", 1), + ("images", 2), + ("video", 1), + ("video", 2), + ("mixed", 1), + ("mixed", 2), + ], +) +def test_qwen38_standard_vl_cuda_hf_parity(tmp_path: Path, dtype: str, case: str, batch: int): + """L3: actual ONNX vision -> fusion -> decoder, then four reused-state steps. + + Nonzero packed patches are synthetic, not official processor outputs. HF + supplies the graph's explicit MRoPE positions; host preprocessing/position + generation and the official checkpoint need separate qualification. + """ + if ( + "CUDAExecutionProvider" not in ort.get_available_providers() + or not torch.cuda.is_available() + ): + pytest.skip("CUDA ORT and PyTorch are required") + from transformers.models.qwen3_5.modeling_qwen3_5 import ( + Qwen3_5ForConditionalGeneration, + ) + + from mobius.models import Qwen35VL3ModelCausalLMModel + + torch.manual_seed(38) + config = _hf_vl_config() + config.text_config._attn_implementation = "eager" + config.vision_config._attn_implementation = "eager" + torch_dtype = torch.float32 if dtype == "f32" else torch.float16 + reference = ( + Qwen3_5ForConditionalGeneration._from_config(config, dtype=torch_dtype).eval().cuda() + ) + package = _build(config, tmp_path / "config", dtype, text_only=False) + weights = Qwen35VL3ModelCausalLMModel(package.config).preprocess_weights( + dict(reference.state_dict()) + ) + for model in package.values(): + apply_weights(model, weights) + _assert_standard(model) + package.save(str(tmp_path / "package"), progress_bar=False) + reloaded = ModelPackage.load(str(tmp_path / "package")) + sessions = {} + for name, model in reloaded.items(): + _assert_standard(model) + options = ort.SessionOptions() + options.intra_op_num_threads = 1 + options.enable_profiling = True + options.profile_file_prefix = str(tmp_path / f"cuda-{name}") + sessions[name] = ort.InferenceSession( + str(tmp_path / f"package/{name}/model.onnx"), + sess_options=options, + providers=[("CUDAExecutionProvider", {"use_tf32": "0"})], + ) + assert sessions[name].get_providers()[0] == "CUDAExecutionProvider" + + def run(name, feeds): + session = sessions[name] + return dict( + zip( + (value.name for value in session.get_outputs()), + session.run(None, feeds), + strict=True, + ) + ) + + inputs = _vl_inputs(config, case, batch) + hf_inputs = {name: torch.from_numpy(value).cuda() for name, value in inputs.items()} + tolerance = 1e-3 if dtype == "f32" else 1e-2 + empty = np.zeros((0, config.text_config.hidden_size), dtype=package.config.dtype.numpy()) + features = {"image_features": empty, "video_features": empty} + with torch.no_grad(): + expected_embeds = reference.get_input_embeddings()(hf_inputs["input_ids"]) + for pixel_name, grid_name, feature_name, token_id in ( + ("pixel_values", "image_grid_thw", "image_features", config.image_token_id), + ("pixel_values_videos", "video_grid_thw", "video_features", config.video_token_id), + ): + if pixel_name not in inputs: + continue + expected_features = reference.model.visual( + hf_inputs[pixel_name].to(torch_dtype), grid_thw=hf_inputs[grid_name] + ).pooler_output + features[feature_name] = run( + "vision_encoder", + { + "pixel_values": inputs[pixel_name], + "image_grid_thw": inputs[grid_name], + }, + )["image_features"] + np.testing.assert_allclose( + features[feature_name], + expected_features.cpu().numpy(), + rtol=tolerance, + atol=tolerance, + strict=True, + ) + mask = (hf_inputs["input_ids"] == token_id).unsqueeze(-1) + expected_embeds = expected_embeds.masked_scatter(mask, expected_features) + positions, deltas = reference.model.get_rope_index( + hf_inputs["input_ids"], + hf_inputs["mm_token_type_ids"], + image_grid_thw=hf_inputs.get("image_grid_thw"), + video_grid_thw=hf_inputs.get("video_grid_thw"), + attention_mask=hf_inputs["attention_mask"], + ) + tokens = inputs["input_ids"] + positions = positions.cpu().numpy() + deltas = deltas.cpu().numpy() + state_feeds = _initial_feeds(reloaded["decoder"], batch) + cache = None + seen = 0 + for step in range(5): + mask = np.ones((batch, seen + tokens.shape[1]), dtype=np.int64) + embeds = run( + "embedding", + { + "input_ids": tokens, + "image_features": _pack_vl_features(tokens, features, config), + }, + )["inputs_embeds"] + with torch.no_grad(): + if step == 0: + np.testing.assert_allclose( + embeds, + expected_embeds.cpu().numpy(), + rtol=tolerance, + atol=tolerance, + strict=True, + ) + expected = reference( + **(hf_inputs if step == 0 else {"input_ids": torch.from_numpy(tokens).cuda()}), + position_ids=torch.from_numpy(positions).cuda(), + past_key_values=cache, + use_cache=True, + **({"attention_mask": torch.from_numpy(mask).cuda()} if step else {}), + ) + cache = expected.past_key_values + actual = run( + "decoder", + { + "inputs_embeds": embeds, + "attention_mask": mask, + "position_ids": positions, + **state_feeds, + }, + ) + np.testing.assert_allclose( + actual["logits"], + expected.logits.cpu().numpy(), + rtol=tolerance, + atol=tolerance, + err_msg=f"logits step {step}", + strict=True, + ) + expected_tokens = expected.logits[:, -1].argmax(-1).cpu().numpy() + np.testing.assert_array_equal(actual["logits"][:, -1].argmax(-1), expected_tokens) + _compare_states( + actual, cache, config.text_config, state_feeds, tolerance=tolerance, step=step + ) + seen += tokens.shape[1] + tokens = expected_tokens[:, None] + positions = np.broadcast_to((seen + deltas)[None], (3, batch, 1)).copy() + features = {"image_features": empty, "video_features": empty} + profiles = { + name: json.loads(Path(session.end_profiling()).read_text()) + for name, session in sessions.items() + } + for role in ["decoder", *([] if case == "text" else ["vision_encoder"])]: + _assert_cuda_matrix(profiles[role], role) diff --git a/src/mobius/models/qwen_vl.py b/src/mobius/models/qwen_vl.py index a17d9b161..91344abf5 100644 --- a/src/mobius/models/qwen_vl.py +++ b/src/mobius/models/qwen_vl.py @@ -1006,6 +1006,7 @@ def __init__(self, config: ArchitectureConfig): assert vc.num_hidden_layers is not None assert vc.num_attention_heads is not None self.num_deepstack = len(config.deepstack_visual_indexes or []) + self._dtype = config.dtype self.visual = Qwen3VLVisionModel( depth=vc.num_hidden_layers, hidden_size=vc.hidden_size, @@ -1026,6 +1027,8 @@ def forward( pixel_values: ir.Value, image_grid_thw: ir.Value, ): + # Processors emit float32 packed pixels even for reduced-precision models. + pixel_values = op.Cast(pixel_values, to=self._dtype) outputs = self.visual( op, hidden_states=pixel_values, @@ -1071,19 +1074,19 @@ def preprocess_weights( class Qwen3VLEmbeddingModel(Qwen25VLEmbeddingModel): """Qwen3-VL embedding model for the 3-model split. - Scatters merged image features at image-token positions (like - Qwen2.5-VL) and, when the vision encoder produces DeepStack features, + Scatters merged image/video features at their placeholder-token positions + and, when the vision encoder produces DeepStack features, also scatters each intermediate DeepStack map into a full-length tensor - (zero at non-image positions). The maps are flattened into the generic + (zero at non-visual positions). The maps are flattened into the generic rank-3 ``per_layer_inputs`` output consumed by ORT GenAI. Inputs: - input_ids: (batch, seq_len) INT64 - - image_features: (num_image_tokens, (D + 1) * hidden_size) FLOAT - when DeepStack is active + - image_features: merged visual features in flattened batch/token order, + (num_visual_tokens, (D + 1) * hidden_size) when DeepStack is active Outputs: - - inputs_embeds: (batch, seq_len, hidden_size) FLOAT - - per_layer_inputs: (batch, seq_len, D * hidden_size) FLOAT + - inputs_embeds: (batch, seq_len, hidden_size), model dtype + - per_layer_inputs: (batch, seq_len, D * hidden_size), model dtype (only when DeepStack is active) """ @@ -1096,17 +1099,20 @@ def forward( ): text_embeds = self.embed_tokens(op, input_ids) - # Image-token positions and their running index into the packed - # feature tensors (shared by the main image scatter and every - # DeepStack scatter). - image_mask = op.Equal(input_ids, op.Constant(value_int=self.image_token_id)) - image_mask_3d = op.Unsqueeze(image_mask, [-1]) - mask_int = op.Cast(image_mask, to=7) # INT64 - cumsum = op.CumSum(mask_int, op.Constant(value_int=1)) + # Packed features follow image/video placeholders across the whole batch, + # matching HF's flattened masked_scatter rather than restarting per row. + visual_mask = op.Or( + op.Equal(input_ids, op.Constant(value_int=self.image_token_id)), + op.Equal(input_ids, op.Constant(value_int=self.video_token_id)), + ) + visual_mask_3d = op.Unsqueeze(visual_mask, [-1]) + mask_int = op.Reshape(op.Cast(visual_mask, to=7), [-1]) # (B*S,) INT64 + cumsum = op.CumSum(mask_int, op.Constant(value_int=0)) indices = op.Clip( op.Sub(cumsum, op.Constant(value_int=1)), op.Constant(value_int=0), ) + indices = op.Reshape(indices, op.Shape(input_ids)) def _scatter(features: ir.Value, fallback: ir.Value) -> ir.Value: # Pad with one zero row so Gather stays in-bounds for text-only @@ -1121,7 +1127,7 @@ def _scatter(features: ir.Value, fallback: ir.Value) -> ir.Value: ) padded = op.Concat(features, pad_row, axis=0) gathered = op.Gather(padded, indices, axis=0) - return op.Where(image_mask_3d, gathered, fallback) + return op.Where(visual_mask_3d, gathered, fallback) inputs_embeds = _scatter(image_features, text_embeds) diff --git a/src/mobius/tasks/_cache_utils.py b/src/mobius/tasks/_cache_utils.py index da4162c4c..900dce837 100644 --- a/src/mobius/tasks/_cache_utils.py +++ b/src/mobius/tasks/_cache_utils.py @@ -325,7 +325,7 @@ def _make_hybrid_cache_inputs( ) rec_state = builder.input( f"{prefix}.{i}.recurrent_state", - dtype=dtype, + dtype=getattr(config, "mamba_ssm_dtype", None) or dtype, shape=[batch, dims.num_v_heads, dims.head_k_dim, dims.head_v_dim], ) pairs.append((conv_state, rec_state)) @@ -480,7 +480,7 @@ def _register_linear_attention_functions( kv_num_heads=dims.num_v_heads, update_rule="gated_delta", scale=1.0 / (dims.head_k_dim**0.5), - stash_type=getattr(config, "mamba_ssm_dtype", config.dtype), + stash_type=getattr(config, "mamba_ssm_dtype", None) or config.dtype, ) model.functions[conv_func.identifier()] = conv_func model.functions[attn_func.identifier()] = attn_func diff --git a/src/mobius/tasks/_causal_lm.py b/src/mobius/tasks/_causal_lm.py index 68f0d0836..ece0f73cf 100644 --- a/src/mobius/tasks/_causal_lm.py +++ b/src/mobius/tasks/_causal_lm.py @@ -342,8 +342,8 @@ class HybridCausalLMTask(ModelTask): - past_key_values.{i}.recurrent_state: [batch, num_v_heads, k_dim, v_dim] Outputs: - - logits: FLOAT - - present.{i}.{key|value|conv_state|recurrent_state}: FLOAT + - logits and present.{i}.{key|value|conv_state}: model dtype + - present.{i}.recurrent_state: configured mamba_ssm_dtype, or model dtype Args: prune_prefill_prefix: If ``True``, insert ``Gather(axis=1, index=-1)`` diff --git a/testdata/configs/qwen3_8-27b.json b/testdata/configs/qwen3_8-27b.json new file mode 100644 index 000000000..b9bca61ad --- /dev/null +++ b/testdata/configs/qwen3_8-27b.json @@ -0,0 +1,86 @@ +{ + "architectures": ["Qwen3_5ForConditionalGeneration"], + "image_token_id": 248056, + "language_model_only": false, + "model_type": "qwen3_5", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": 248044, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "layer_types": [ + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention", + "linear_attention", "linear_attention", "linear_attention", "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 48, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "model_type": "qwen3_5_text", + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 24, + "num_hidden_layers": 64, + "num_key_value_heads": 4, + "output_gate_type": "swish", + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-6, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [11, 11, 10], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.8.0.dev0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 5120, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/tests/ort_genai_hybrid_native_test.py b/tests/ort_genai_hybrid_native_test.py new file mode 100644 index 000000000..6af870a5c --- /dev/null +++ b/tests/ort_genai_hybrid_native_test.py @@ -0,0 +1,482 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Opt-in native Qwen hybrid state proof, separate from the pinned fast CPU lane. + +Run with MOBIUS_QWEN_NATIVE_PROBE=1. No downloads, real weights, or manual ORT +cache loop: GenAI owns preprocessing, positions, caches, and token generation. +""" + +from __future__ import annotations + +import gc +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import onnx_ir as ir +import pytest +from onnxscript import GraphBuilder + +from mobius._model_package import ModelPackage +from mobius.integrations.ort_genai import write_ort_genai_config + +pytestmark = pytest.mark.skipif( + os.environ.get("MOBIUS_QWEN_NATIVE_PROBE") != "1", + reason="opt-in native hybrid qualification, not the generic 0.15.2 CPU lane", +) + +_HIDDEN_SIZE = 5120 +_KV_HEADS = 4 +_HEAD_DIM = 256 +_CONV_CHANNELS = 10240 +_STATE_HEADS = 48 +_STATE_DIM = 128 + + +def _disable_core_dumps() -> None: + """Disable POSIX core dumps to avoid huge GPU crash artifacts.""" + if os.name == "posix": + import resource + + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + + +def _output( + builder: GraphBuilder, + value: ir.Value, + name: str, + dtype: ir.DataType, + shape: list[int | str], +) -> None: + value.type = ir.TensorType(dtype) + value.shape = ir.Shape(shape) + builder.add_output(value, name) + + +def _hybrid_package() -> ModelPackage: + graph = ir.Graph([], [], nodes=[], name="decoder", opset_imports={"": 24}) + b = GraphBuilder(graph) + embeds = b.input("inputs_embeds", ir.DataType.FLOAT16, ["batch", "sequence", _HIDDEN_SIZE]) + b.input("attention_mask", ir.DataType.INT64, ["batch", "total_sequence"]) + b.input("position_ids", ir.DataType.INT64, [3, "batch", "sequence"]) + score = b.op.Constant(value_float=0.0) + for index in range(8): + if index % 4 == 3: + for role in ("key", "value"): + past = b.input( + f"past_key_values.{index}.{role}", + ir.DataType.FLOAT16, + ["batch", _KV_HEADS, "past_sequence", _HEAD_DIM], + ) + if index == 3 and role == "key": + length = b.op.Shape(past, start=2, end=3) + score = b.op.Add(score, b.op.Cast(length, to=ir.DataType.FLOAT)) + # Form new zero KV rows from zero token embeddings with ordinary + # CUDA-capable tensor ops, not a CPU ConstantOfShape output. + update = b.op.Slice( + embeds, + b.op.Constant(value_ints=[0]), + b.op.Constant(value_ints=[_KV_HEADS * _HEAD_DIM]), + b.op.Constant(value_ints=[2]), + ) + update = b.op.Reshape( + update, + b.op.Concat( + b.op.Shape(embeds, start=0, end=2), + b.op.Constant(value_ints=[_KV_HEADS, _HEAD_DIM]), + axis=0, + ), + ) + update = b.op.Transpose(update, perm=[0, 2, 1, 3]) + _output( + b, + b.op.Concat(past, update, axis=2), + f"present.{index}.{role}", + ir.DataType.FLOAT16, + ["batch", _KV_HEADS, "present_sequence", _HEAD_DIM], + ) + else: + for role, dtype, shape, increment in ( + ("conv_state", ir.DataType.FLOAT16, ["batch", _CONV_CHANNELS, 3], 1), + ( + "recurrent_state", + ir.DataType.FLOAT, + ["batch", _STATE_HEADS, _STATE_DIM, _STATE_DIM], + 2, + ), + ): + past = b.input(f"past_key_values.{index}.{role}", dtype, shape) + # Both independently typed state families determine the token. + # The other slots are verified via native outputs below. + if index == 0: + score = b.op.Add( + score, + b.op.ReduceMean(b.op.Cast(past, to=ir.DataType.FLOAT), keepdims=0), + ) + constant = b.op.Constant( + value=ir.tensor(np.array(increment, dtype=dtype.numpy())) + ) + _output(b, b.op.Add(past, constant), f"present.{index}.{role}", dtype, shape) + token = b.op.Cast(score, to=ir.DataType.INT64) + token = b.op.Mod(token, b.op.Constant(value_int=32)) + logits = b.op.OneHot( + token, b.op.Constant(value_int=32), b.op.Constant(value_floats=[0.0, 10.0]) + ) + logits = b.op.Expand( + logits, + b.op.Concat( + b.op.Shape(embeds, start=0, end=2), b.op.Constant(value_ints=[32]), axis=0 + ), + ) + _output(b, logits, "logits", ir.DataType.FLOAT, ["batch", "sequence", 32]) + + embedding = ir.Graph([], [], nodes=[], name="embedding", opset_imports={"": 24}) + e = GraphBuilder(embedding) + ids = e.input("input_ids", ir.DataType.INT64, ["batch", "sequence"]) + e.input("image_features", ir.DataType.FLOAT16, ["features", _HIDDEN_SIZE]) + # Use the real embedding's standard Gather contract. GDB located the prior + # zero-fill probe's native prefill crash in CPU ConstantOfShape::Compute. + table = e.op.Constant(value=ir.tensor(np.zeros((32, _HIDDEN_SIZE), np.float16))) + _output( + e, + e.op.Gather(table, ids, axis=0), + "inputs_embeds", + ir.DataType.FLOAT16, + ["batch", "sequence", _HIDDEN_SIZE], + ) + + vision = ir.Graph([], [], nodes=[], name="vision", opset_imports={"": 24}) + v = GraphBuilder(vision) + pixels = v.input("pixel_values", ir.DataType.FLOAT, ["patches", 1536]) + v.input("image_grid_thw", ir.DataType.INT64, ["images", 3]) + features = v.op.ReduceMean(pixels, v.op.Constant(value_ints=[1]), keepdims=1) + features = v.op.Slice( + features, + v.op.Constant(value_ints=[0]), + v.op.Constant(value_ints=[np.iinfo(np.int64).max]), + v.op.Constant(value_ints=[0]), + v.op.Constant(value_ints=[4]), + ) + features = v.op.Tile( + v.op.Cast(features, to=ir.DataType.FLOAT16), + v.op.Constant(value_ints=[1, _HIDDEN_SIZE]), + ) + _output( + v, + features, + "image_features", + ir.DataType.FLOAT16, + ["features", _HIDDEN_SIZE], + ) + config = SimpleNamespace( + model_type="qwen3_5", + vocab_size=32, + hidden_size=_HIDDEN_SIZE, + num_hidden_layers=8, + num_attention_heads=24, + num_key_value_heads=_KV_HEADS, + head_dim=_HEAD_DIM, + max_position_embeddings=64, + bos_token_id=1, + eos_token_id=31, + pad_token_id=30, + image_token_id=24, + video_token_id=25, + vision_start_token_id=26, + vision_end_token_id=27, + spatial_merge_size=2, + temporal_patch_size=2, + vision=SimpleNamespace(patch_size=16, spatial_merge_size=2), + ) + return ModelPackage( + { + "decoder": ir.Model(graph, ir_version=10), + "embedding": ir.Model(embedding, ir_version=10), + "vision_encoder": ir.Model(vision, ir_version=10), + }, + config=config, + ) + + +def _write_tokenizer(directory: Path) -> None: + vocab = {f"token{i}": i for i in range(32)} + vocab["hello"] = vocab.pop("token2") + # Only the image markers needed by the native image prompt; IDs match the + # toy config without changing its vocabulary size or text token. + image_markers = {"<|image_pad|>": 24, "<|vision_start|>": 26, "<|vision_end|>": 27} + for content, token_id in image_markers.items(): + vocab.pop(f"token{token_id}") + vocab[content] = token_id + tokenizer = { + "version": "1.0", + "truncation": None, + "padding": None, + "added_tokens": [ + { + "id": token_id, + "content": content, + "single_word": False, + "lstrip": False, + "rstrip": False, + "normalized": False, + "special": True, + } + for content, token_id in image_markers.items() + ], + "normalizer": None, + "pre_tokenizer": {"type": "Whitespace"}, + "post_processor": None, + "decoder": None, + "model": { + "type": "BPE", + "dropout": None, + "unk_token": "token0", + "continuing_subword_prefix": "", + "end_of_word_suffix": "", + "fuse_unk": False, + "byte_fallback": False, + "ignore_merges": False, + "vocab": vocab, + "merges": [], + }, + } + (directory / "tokenizer.json").write_text(json.dumps(tokenizer), encoding="utf-8") + (directory / "tokenizer_config.json").write_text( + json.dumps( + { + "tokenizer_class": "Qwen2Tokenizer", + "additional_special_tokens": list(image_markers), + } + ), + encoding="utf-8", + ) + + +def test_native_sparse_mixed_state_progression_and_reset(tmp_path: Path) -> None: + import onnxruntime_genai as og + + _disable_core_dumps() + package = _hybrid_package() + for graph in package.values(): + assert not graph.functions + assert all(node.domain in {"", "ai.onnx"} for node in graph.graph.all_nodes()) + package.save(tmp_path, progress_bar=False) + _write_tokenizer(tmp_path) + write_ort_genai_config( + package, + str(tmp_path), + ep="cuda", + context_length=64, + ) + payload = json.loads((tmp_path / "genai_config.json").read_text()) + assert payload["model"]["type"] == "qwen3_5" + assert payload["model"]["decoder"]["num_hidden_layers"] == 8 + assert payload["search"]["past_present_share_buffer"] is False + print("GenAI version:", og.__version__, flush=True) + print("Synthetic config:", json.dumps(payload, indent=2), flush=True) + config = og.Config(str(tmp_path)) + # The installed Config override does not route explicit CPU auxiliary + # sessions to CUDA. All three session options must match before prefill. + for name in ("decoder", "vision", "embedding"): + providers = payload["model"][name]["session_options"]["provider_options"] + assert providers[0]["cuda"]["enable_cuda_graph"] == "0" + config.clear_providers() + config.append_provider("CUDAExecutionProvider") + model = og.Model(config) + print("Native model constructed", flush=True) + processor = model.create_multimodal_processor() + inputs = processor("hello") + print("Native processor text inputs created", flush=True) + receipts = [] + for run in range(2): + params = og.GeneratorParams(model) + params.set_search_options(max_length=4, do_sample=False) + print("Constructing native generator", run, flush=True) + generator = og.Generator(model, params) + print("Native generator constructed; starting set_inputs/prefill", run, flush=True) + generator.set_inputs(inputs) + print("Native set_inputs/prefill completed", run, flush=True) + tokens = [] + for step in range(3): + generator.generate_next_token() + tokens.append(int(generator.get_next_tokens()[0])) + for index in range(8): + if index % 4 == 3: + for role in ("key", "value"): + state = generator.get_output(f"present.{index}.{role}") + assert state.dtype == np.float16 + assert state.shape == (1, _KV_HEADS, step + 1, _HEAD_DIM) + else: + for role, dtype, increment in ( + ("conv_state", np.float16, 1), + ("recurrent_state", np.float32, 2), + ): + state = generator.get_output(f"present.{index}.{role}") + assert state.dtype == dtype + assert state.shape == ( + (1, _CONV_CHANNELS, 3) + if role == "conv_state" + else (1, _STATE_HEADS, _STATE_DIM, _STATE_DIM) + ) + np.testing.assert_array_equal(state, (step + 1) * increment) + print("Native run/step/token:", run, step, tokens[-1], flush=True) + assert tokens == [0, 4, 8] + receipts.append(tokens) + del generator, params + gc.collect() + print("Native progression/reset receipts:", receipts, flush=True) + (tmp_path / "native-proof.json").write_text( + json.dumps( + { + "genai_version": og.__version__, + "provider": "CUDAExecutionProvider", + "sparse_kv_indices": [3, 7], + "conv_dtype": "float16", + "recurrent_dtype": "float32", + "past_present_share_buffer": False, + "progression_and_reset": receipts, + }, + indent=2, + ), + encoding="utf-8", + ) + del inputs, processor, model + gc.collect() + + +def test_generic_native_control(tmp_path: Path) -> None: + """Distinguish a hybrid-runtime blocker from a broken native installation.""" + import onnxruntime_genai as og + from ort_genai_e2e_test import _synthetic_decoder_package, _write_tokenizer + + _disable_core_dumps() + package = _synthetic_decoder_package() + package.save(tmp_path, progress_bar=False) + _write_tokenizer(tmp_path) + write_ort_genai_config(package, str(tmp_path), context_length=32) + config = og.Config(str(tmp_path)) + config.clear_providers() + config.append_provider("CUDAExecutionProvider") + model = og.Model(config) + params = og.GeneratorParams(model) + params.set_search_options(max_length=4, do_sample=False) + generator = og.Generator(model, params) + generator.append_tokens([2]) + tokens = [] + for _ in range(3): + generator.generate_next_token() + tokens.append(int(generator.get_next_tokens()[0])) + assert tokens == [0, 1, 2] + print("Generic native control receipts:", tokens, flush=True) + del generator, params, model + gc.collect() + + +def test_native_processor_text_contract(tmp_path: Path) -> None: + """Validate the native expanded-input API used by the real runner.""" + import onnxruntime_genai as og + + _disable_core_dumps() + package = _hybrid_package() + package.save(tmp_path, progress_bar=False) + _write_tokenizer(tmp_path) + write_ort_genai_config(package, str(tmp_path), ep="onnx-standard", context_length=64) + config = og.Config(str(tmp_path)) + config.clear_providers() + config.append_provider("CUDAExecutionProvider") + model = og.Model(config) + processor = model.create_multimodal_processor() + inputs = processor("hello") + ids = inputs["input_ids"].as_numpy() + print("Native expanded input_ids:", ids, "shape:", ids.shape, flush=True) + assert ids.dtype == np.int32 or ids.dtype == np.int64 + assert ids.size == 1 + assert int(ids.reshape(-1)[0]) == 2 + del inputs, processor, model + gc.collect() + + +def test_native_processor_preserves_uniform_rgb_channels(tmp_path: Path) -> None: + """Check true native pixels before prefill, without checkpoint weights.""" + import onnxruntime_genai as og + from PIL import Image + + _disable_core_dumps() + package = _hybrid_package() + package.save(tmp_path, progress_bar=False) + _write_tokenizer(tmp_path) + write_ort_genai_config(package, str(tmp_path), ep="cuda", context_length=64) + payload = json.loads((tmp_path / "processor_config.json").read_text()) + operations = { + item["operation"]["type"]: item["operation"].get("attrs", {}) + for item in payload["processor"]["transforms"] + } + rgb = [255, 128, 0] + image_path = tmp_path / "distinct-rgb.png" + Image.new("RGB", (64, 64), tuple(rgb)).save(image_path) + config = og.Config(str(tmp_path)) + config.clear_providers() + config.append_provider("CUDAExecutionProvider") + model = og.Model(config) + processor = model.create_multimodal_processor() + images = og.Images.open(str(image_path)) + inputs = processor("<|vision_start|><|image_pad|><|vision_end|>hello", images=images) + try: + pixels = inputs["pixel_values"].as_numpy() + grid = inputs["image_grid_thw"].as_numpy() + ids = inputs["input_ids"].as_numpy() + assert pixels.dtype == np.float32 + np.testing.assert_array_equal(grid, [[1, 4, 4]]) + assert pixels.shape == (16, 1536) + assert np.count_nonzero(ids == 24) == 4 + norm = operations["Normalize"] + expected = ( + np.asarray(rgb, dtype=np.float32) * operations["Rescale"]["rescale_factor"] + - np.asarray(norm["mean"], dtype=np.float32) + ) / np.asarray(norm["std"], dtype=np.float32) + # Inspect the documented C,T,H,W patch layout only. No flips/reorders, + # no tensor feeds, and no Generator/set_inputs/prefill are used here. + patches = pixels.reshape(16, 3, 2, 16, 16) + observed = patches.mean(axis=(0, 2, 3, 4)) + print( + "Native before-prefill RGB:", + rgb, + "expected normalized channels:", + expected.tolist(), + "observed channels:", + observed.tolist(), + "grid:", + grid.tolist(), + flush=True, + ) + np.testing.assert_allclose( + patches, + np.broadcast_to(expected[None, :, None, None, None], patches.shape), + rtol=0, + atol=1e-6, + ) + (tmp_path / "native-rgb-proof.json").write_text( + json.dumps( + { + "genai_version": og.__version__, + "stage": "native processor before prefill", + "rgb": rgb, + "expected_normalized_channels": expected.tolist(), + "observed_normalized_channels": observed.tolist(), + "max_abs_error": float( + np.max(np.abs(patches - expected[None, :, None, None, None])) + ), + "pixel_shape": list(pixels.shape), + "grid": grid.tolist(), + "manual_channel_or_row_reorder": False, + }, + indent=2, + ), + encoding="utf-8", + ) + finally: + del inputs, images, processor, model + gc.collect()