From 50b489b9c5dac343bbe0eeebd41671d7feed197d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 8 Sep 2026 03:17:04 -0700 Subject: [PATCH] the Qwen family on Vulkan, one model at a time: every Qwen text model on the box (27 GGUF files, 0.5B to 48 GB) measured against llama.cpp on the same card and the three silent declines fixed - the qwen2 q/k/v projection bias folds into the rope stage on the resident driver and rides the per-op attention chain's hooks, a hybrid's KV mirror binding cap sums per-layer rows instead of reading one byte past 4 GiB, and a BPE vocab without add_bos_token takes upstream's per-pre default; the resident plan's auto arm sizes against the room the OS reports on the adapter (a hand-bound vk_os_video_memory query, no VRAM pin needed) and device allocations carry no memory-priority tag (the 27B UD-IQ4_XS with 1.7 GB demoted decodes 4.2 -> 23.5 t/s); the grid GEMVs stage their codebooks from a device buffer (iq2s 84 -> 388 GB/s) and the codebook scale row is two words; the SPIR-V emitter lets a kernel call a [spirv_decode] method directly - a struct parameter spills to a Function local, a plane-element argument passes its index and the callee chains through the plane - with four fail-closed fixtures and a golden; the kq prefill tile gains a KHR cooperative-matrix arm for all thirteen formats (a device with KHR coopmat and no NV_coopmat2 prefilled K-quants at a third of llama.cpp's rate: 4B Q4_K_M 1564 -> 2979 t/s, the tile's two uint staging arrays its whole workgroup footprint after the 8 KB edge-store slab cost it one workgroup per SM); the twin kernels collapse onto templates and shared bases with a census of what stays apart; the serving census sweeps the format vehicles under every coopmat mode, so every format's sdot4, cm2 and KHR tile families dispatch under a census model; the deltanet conv kernel stages only the q/k region it norms (the whole-row slab overflowed at the 27B's 10240 conv width) and the add-rms row stash stays within a 32 KiB workgroup budget at 6144 floats; the ast-fuzz suite spawns the binary running it and the module-cache file's child-spawning cells skip on an AOT-consuming host; the harvest round lands every fact as an architecture section its code cites and every rule in the checklist it binds, the twenty-one audits' violations fixed in code or in the checklists themselves; the board (plans/qwen_arc_board.md) and ledger item 42 carry the numbers and where the KHR gap still sits Co-Authored-By: Claude Fable 5.1 --- modules/REVIEW_SHADER_EMITTERS.md | 53 +- modules/dasLLAMA/ARCHITECTURE.md | 10 +- modules/dasLLAMA/ARCHITECTURE_ENGINE.md | 16 +- modules/dasLLAMA/ARCHITECTURE_GPU.md | 16 +- .../dasLLAMA/ARCHITECTURE_GPU_QUANT_PLANES.md | 5 +- modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md | 175 +- .../ARCHITECTURE_GPU_VULKAN_DECODE.md | 36 +- .../dasLLAMA/ARCHITECTURE_GPU_VULKAN_GEMM.md | 211 ++ .../ARCHITECTURE_GPU_VULKAN_RESIDENCY.md | 77 +- modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md | 23 +- modules/dasLLAMA/ENVIRONMENT.md | 6 +- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 33 +- modules/dasLLAMA/README.md | 3 +- modules/dasLLAMA/REVIEW.das | 4 +- modules/dasLLAMA/REVIEW.md | 29 +- modules/dasLLAMA/REVIEW_GPU.md | 55 +- modules/dasLLAMA/REVIEW_GPU_RACE.md | 71 +- modules/dasLLAMA/REVIEW_GPU_VULKAN.md | 52 +- modules/dasLLAMA/REVIEW_IMAGE.md | 20 +- modules/dasLLAMA/REVIEW_MEASUREMENT.md | 51 +- modules/dasLLAMA/REVIEW_PLACEMENT.md | 25 +- modules/dasLLAMA/REVIEW_UPSTREAM.md | 35 +- modules/dasLLAMA/benchmarks/REVIEW.md | 33 +- modules/dasLLAMA/dasllama/dasllama_blocks.das | 43 +- modules/dasLLAMA/dasllama/dasllama_bpe.das | 15 +- modules/dasLLAMA/dasllama/dasllama_common.das | 8 + modules/dasLLAMA/dasllama/dasllama_env.das | 8 +- .../dasllama/dasllama_gemm_schema.das | 22 +- modules/dasLLAMA/dasllama/dasllama_gguf.das | 58 + .../dasllama/dasllama_gpu_resident.das | 141 +- .../dasLLAMA/dasllama/dasllama_gpu_tier.das | 87 +- modules/dasLLAMA/dasllama/dasllama_image.das | 14 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 41 +- modules/dasLLAMA/dasllama/dasllama_load.das | 166 +- .../dasllama/dasllama_math_vulkan.das | 3 + .../dasllama/dasllama_vulkan_classes.das | 2052 ++++++++--------- .../dasllama/dasllama_vulkan_common.das | 141 +- .../dasllama/dasllama_vulkan_decode.das | 142 +- .../dasllama/dasllama_vulkan_prefill.das | 195 +- .../dasllama/dasllama_vulkan_seams.das | 6 +- modules/dasLLAMA/followup_general.md | 11 + modules/dasLLAMA/followup_metal.md | 38 + modules/dasLLAMA/followup_vulkan.md | 108 +- modules/dasLLAMA/harness/REVIEW.md | 6 + .../dasLLAMA/harness/_vk_probe_fixture.das | 22 + modules/dasLLAMA/harness/vk_gemm_probe.das | 1231 +++++++++- modules/dasLLAMA/harness/vk_gemv_probe.das | 136 ++ modules/dasLLAMA/performance/REVIEW.md | 34 +- modules/dasLLAMA/performance/model_specs.das | 4 +- modules/dasLLAMA/tests/CLAUDE.md | 46 +- modules/dasLLAMA/tests/REVIEW.md | 30 +- modules/dasLLAMA/tests/REVIEW_KERNEL_CELLS.md | 16 +- modules/dasLLAMA/tests/_model_tier.das | 4 + modules/dasLLAMA/tests/_vkd_oracles.das | 32 +- modules/dasLLAMA/tests/_vkd_toy.das | 30 + modules/dasLLAMA/tests/run.das | 3 +- modules/dasLLAMA/tests/test_gguf_quant.das | 31 + .../tests/test_gpu_resident_hybrid.das | 38 +- .../tests/test_gpu_resident_qwen2.das | 128 + .../tests/test_gpu_serving_declines.das | 73 +- modules/dasLLAMA/tests/test_gpu_tier.das | 3 +- .../dasLLAMA/tests/test_kernel_coverage.das | 34 +- modules/dasLLAMA/tests/test_kquant.das | 114 + modules/dasLLAMA/tests/test_tokenizer.das | 20 + .../dasLLAMA/tests/test_vulkan_dec_tail.das | 54 +- .../dasLLAMA/tests/test_vulkan_kernels.das | 1020 +++++--- modules/dasLLAMA/tests/test_vulkan_tier.das | 47 +- modules/dasLLVM/REVIEW.md | 4 + modules/dasMetal/ARCHITECTURE.md | 9 +- modules/dasSpirv/ARCHITECTURE.md | 47 +- modules/dasSpirv/REVIEW.md | 7 + modules/dasSpirv/spirv/spirv_emit.das | 160 +- modules/dasUnitTest/unit_test.das.inc | 64 +- modules/dasVulkan/ARCHITECTURE.md | 26 +- modules/dasVulkan/REVIEW.md | 29 +- modules/dasVulkan/daslib/vulkan_boost.das | 29 + modules/dasVulkan/src/dasVULKAN.main.cpp | 143 ++ modules/dasVulkan/tests/REVIEW.md | 4 + .../integration/test_os_video_memory.das | 33 + modules/dasVulkan/utils/vulkan2rst.das | 1 + plans/REVIEW.md | 7 + plans/kernel_twins_census.md | 135 ++ plans/qwen_arc_board.md | 202 ++ plans/shader_emitter_followups.md | 19 + skills/internal/make_pr.md | 7 +- .../module_cache/test_default_cache_path.das | 26 +- tests/msl/REVIEW.md | 3 +- tests/spirv/REVIEW.md | 8 +- .../_fail_closed/_fc_decode_helper_copy.das | 35 + .../spirv/_fail_closed/_fc_decode_index64.das | 30 + tests/spirv/_fail_closed/_fc_decode_mixed.das | 32 + .../_fail_closed/_fc_decode_twoplanes.das | 32 + tests/spirv/_gen_golden.das | 5 +- tests/spirv/_golden/spdirect.txt | 131 ++ tests/spirv/_golden/spelem.txt | 125 + tests/spirv/_golden/spelemi.txt | 127 + tests/spirv/_spirv_common.das | 91 +- tests/spirv/test_census.das | 3 + tests/spirv/test_fail_closed.das | 5 + tests/spirv/test_golden.das | 3 + tests/spirv/test_struct_param.das | 113 + utils/dasllama-server/REVIEW.md | 4 + utils/internal/ast-fuzz/test_ast_fuzz.das | 8 +- 103 files changed, 6971 insertions(+), 2230 deletions(-) create mode 100644 modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_GEMM.md create mode 100644 modules/dasLLAMA/harness/_vk_probe_fixture.das create mode 100644 modules/dasLLAMA/harness/vk_gemv_probe.das create mode 100644 modules/dasLLAMA/tests/test_gpu_resident_qwen2.das create mode 100644 modules/dasVulkan/tests/integration/test_os_video_memory.das create mode 100644 plans/REVIEW.md create mode 100644 plans/kernel_twins_census.md create mode 100644 plans/qwen_arc_board.md create mode 100644 tests/spirv/_fail_closed/_fc_decode_helper_copy.das create mode 100644 tests/spirv/_fail_closed/_fc_decode_index64.das create mode 100644 tests/spirv/_fail_closed/_fc_decode_mixed.das create mode 100644 tests/spirv/_fail_closed/_fc_decode_twoplanes.das create mode 100644 tests/spirv/_golden/spdirect.txt create mode 100644 tests/spirv/_golden/spelem.txt create mode 100644 tests/spirv/_golden/spelemi.txt create mode 100644 tests/spirv/test_struct_param.das diff --git a/modules/REVIEW_SHADER_EMITTERS.md b/modules/REVIEW_SHADER_EMITTERS.md index 86493bb6f5..318b754272 100644 --- a/modules/REVIEW_SHADER_EMITTERS.md +++ b/modules/REVIEW_SHADER_EMITTERS.md @@ -1,16 +1,18 @@ -# Shared emitter rules - dasSpirv and dasMetal +# Shader Emitters Code Review Checklist -**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist. A change -under either module, or to any kernel body or fixture either emitter compiles, applies this -list with that folder's own.** Architecture docs: `dasMetal/ARCHITECTURE.md` and -`dasSpirv/ARCHITECTURE.md`. +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture +docs: `dasMetal/ARCHITECTURE.md`, `dasSpirv/ARCHITECTURE.md`. + +**Routed here by another checklist: a diff under either emitter module, or to any kernel body +or fixture either emitter compiles, applies this list together with its own folder's.** **Never put anything that cannot run on the CPU into a kernel body or into a function a kernel calls - keep both in ordinary das.** The CPU run of the same body is what the tests compare against. -**A diff that adds or changes an emitter builtin ships a CPU body that returns what the emitted -form returns, argument for argument.** +**A diff that adds or changes an emitter builtin - a declaration in +`daslib/shader_lingua_franca.das` or an emitter's builtin table - ships a CPU body that returns +what the emitted form returns, argument for argument.** **Never let a construct the emitter cannot lower produce a kernel or a crash - the emitter reports a compile error that names the construct.** @@ -19,17 +21,18 @@ reports a compile error that names the construct.** constant.** A shape constant is any value that fixes the kernel's tiling: a tile row count, a tile column count, a cooperating-simdgroup count, a staged chunk depth. -**Never pass a matmul reduction width known only at run time into a kernel that accumulates -through the emitter's matrix or tensor ops any way but through the emitter's runtime-extent -descriptor - `dynamic_extent` on Metal, and on SPIR-V a `tensorLayout2D` or `tensorLayout2DPad` -whose dimension `tensorLayoutSetDimension` sets.** The reduction width is the K dimension - the -length of the loop the kernel accumulates over; it does not fix tiling, so it is not a shape -constant. A hand-written reduction loop takes K as a plain uniform. +**Never pass a matmul reduction width known only at run time into a kernel that loads its +operands with the emitter's tensor-load ops (`coopmatLoadTensor*`, `coopmatLoadTensorDecode`) +any way but through the emitter's runtime-extent descriptor - `dynamic_extent` on Metal, and on +SPIR-V a `tensorLayout2D` or `tensorLayout2DPad` whose dimension `tensorLayoutSetDimension` +sets.** The reduction width is the K dimension - the length of the loop the kernel accumulates +over; it does not fix tiling, so it is not a shape constant. A loop that stages its own tiles +and loads fragments with `coopmatLoad` takes K as a plain uniform. -**A diff that makes a kernel need a shape value known only at run time ships a specialization -path, or records in the emitter's architecture doc - `dasMetal/ARCHITECTURE.md` for a Metal -kernel, `dasSpirv/ARCHITECTURE.md` for a SPIR-V kernel - that the kernel cannot have one.** A -specialization path is one compiled variant per constant shape. +**A diff that makes a kernel need a shape constant known only at run time ships a +specialization path, or records in the emitter's architecture doc - `dasMetal/ARCHITECTURE.md` +for a Metal kernel, `dasSpirv/ARCHITECTURE.md` for a SPIR-V kernel - that the kernel cannot +have one.** A specialization path is one compiled variant per constant shape. **Never check a claim about emitted shape against the das source - check it in the emitted words or text.** Emitted shape is the structure of the emitted kernel - its signature, its @@ -37,8 +40,10 @@ parameter attributes, its statement forms - and its stamped shape values (tile, threadgroup sizes). **A diff that adds a kernel-model capability to one emitter adds it to the other, or records -the asymmetry in the shared ledger (`dasMetal/ARCHITECTURE.md`).** A kernel-model -capability is anything that changes how a kernel is written or how its body is lowered. +the asymmetry in the shared ledger (`dasMetal/ARCHITECTURE.md`).** A kernel-model capability +is anything a kernel author can write differently because of it - an annotation, a call form, +a declaration the emitter now accepts or refuses; a change to lowering alone is the folder's +own checklist's. **A diff that puts a `daslib/shader_lingua_franca` declaration into a kernel body or fixture an emitter compiles, where that emitter does not handle it, ships, in the same change, either @@ -46,8 +51,8 @@ that emitter's lowering of the declaration or a test showing the emitter rejects declaration by name.** A declaration in that module is available to both emitters. **A `?:`, `&&`, or `||` in a `[spirv_kernel]` or `[compute_shader]` body, or in any `def` that -body calls, never indexes a global - a module global, a `@workgroup` array, or a -`self.` resource - in an operand the condition can skip; read the value into a local -above the operator instead, and when the condition was the only thing keeping the index in -range, clamp the index.** The emitter lowers such an operand as a branch -(`dasSpirv/ARCHITECTURE.md`, "Operand laziness follows the language"). +body calls, whose skippable operand subscripts a global-rooted array - a module global, a +`@workgroup` array, or a `self.` resource - is written as the language reads it: never +rewritten around the emitter with an if/else, a clamp or an unconditional read.** The emitter +lowers such an operand as a branch (`dasSpirv/ARCHITECTURE.md`, "Operand laziness follows the +language"); a branchless form where it matters waits for `select` (`plans/shader_emitter_followups.md`). diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index e4e6bb4112..c2325cb8f2 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -48,10 +48,12 @@ re-transcoding `$LCPP/src/unicode-data.cpp`). - `ARCHITECTURE_GPU_PREFILL.md` - sec.2.2c-2.2i, 2.2u-2.2v, 2.2aa: the Metal prefill driver's GEMM form ladder, dev-W knee map, attention slab, MoE bucket rail, chunked submission, the f16 twin dual-store, the last-layer FFN tail, and the dense-KQ tensor mul_mm scaffold. -- `ARCHITECTURE_GPU_VULKAN.md` - sec.2.2j-2.2m, 2.2p-2.2q: the Vulkan resident driver's kernels - and encode chains - the prefill window chain, the cm2 decode lanes and tile pick, the - class-pipeline build seat, the Q8 requant byte store, and the MoE expert chain on the cm2 - tiles. +- `ARCHITECTURE_GPU_VULKAN.md` - sec.2.2j, 2.2p, 2.2ab: the Vulkan resident driver's prefill + chain and byte stores - the prefill window chain, the Q8 requant byte store, and the decode + GEMV family's grid codebook buffer. +- `ARCHITECTURE_GPU_VULKAN_GEMM.md` - sec.2.2k-2.2m, 2.2q: the cooperative-matrix tiles the + Vulkan tier's GEMMs run on - the cm2 decode lanes, the tile pick and the coopmat mode ladder, + the class-pipeline build seat, and the MoE expert chain on those tiles. - `ARCHITECTURE_GPU_VULKAN_RESIDENCY.md` - sec.2.2n-2.2o: what a model has to fit on the card before the driver runs - the residency plan, and the GPU-slot marks swap that lets one slot serve many models. diff --git a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md index da79194ccf..b818ca9eec 100644 --- a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md +++ b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md @@ -138,7 +138,12 @@ stay the reviewer's. A mis-numbered arm dispatches, reads the wrong buffer, and share no state; a third merge algorithm gets a fourth file. Two sanctioned family-name tests live here rather than in `dasllama_pretok`: the `pre`-name selector inside `bpe_encode`, and the gemma-4 newline-run split in `bpe_encode_spm_space`; `load_bpe_tokenizer_gguf`'s per-family - metadata defaults are the third and last. + metadata defaults are the third and last. A GGUF without `tokenizer.ggml.add_bos_token` takes + upstream's per-pre default (`bpe_pre_adds_bos`): the pres on its list - the llama3 family, + tekken, chameleon, a few smaller ones - prepend BOS, every other pre - qwen2, qwen35, gpt-2, + gpt-4o among them - prepends none. The + default matters: a Qwen hybrid fed a leading `<|endoftext|>` degenerates for the whole + generation, and the unsloth Qwen3.8 conversions omit the key. - **`dasllama_pretok.das`** - the pre-tokenizer: one hand-compiled split function per family (llama3/qwen2/qwen35, gpt-2, gpt-4o, tekken), selected by the BPE `pre` name. Regex-port growth lands here, never in the merge engine - the two change for different reasons (new model family @@ -152,7 +157,14 @@ stay the reviewer's. A mis-numbered arm dispatches, reads the wrong buffer, and detection, the eager and streamed conversion ladders, and the load entry points. It owns nothing the forward path touches at run time - a loaded `Model` is the whole handoff - and it requires `dasllama_common` back for `Model`/`Session`, so the transformer umbrella requires it `public` and -breaks the cycle. That re-export is what keeps every consumer on the facade. +breaks the cycle. That re-export is what keeps every consumer on the facade. Every big-weight +conversion is accounted as it runs (`conv_account`, on both the eager and the streamed ladder), +and a load that converts ends with the report: the file's bytes, then each (disk type -> +conversion kind) as bytes on disk against bytes served with the delta, biggest overhead first, +and the count of tags the row-length rule demoted to the q8 rail (each demotion logged by +name as it is decided). The delta column is where a served form costs more than the file - a +decoded scale row, an f32 copy of a quantized token table - and it is read as a number, not +inferred from image sizes. - **`dasllama_image.das`** - the prepared-model `.dlim` rail, and it is ONE rail (`ARCHITECTURE_IMAGE.md` sec.2.1). Nothing diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU.md b/modules/dasLLAMA/ARCHITECTURE_GPU.md index 709b85d50e..714b147b38 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU.md @@ -9,8 +9,8 @@ that a question answered for one backend has an obvious address in the other. Th | role | holds | must not hold | |---|---|---| -| the kernel home
`dasllama_metal_kernels`, `dasllama_vulkan_classes` | kernel source, the derived-access/PSO census | device state, engine types | -| `dasllama__common`
`dasllama_metal_common`, `dasllama_vulkan_common` | device state, buffer/command plumbing, hazard + capture rail, profiler, shared quant-decode helpers (`iq4_lut`) | driver policy | +| the kernel home
`dasllama_metal_kernels`, `dasllama_vulkan_classes` | kernel source, the kernel-side quant-decode helpers and codebook tables, the derived-access/PSO census; on Vulkan the one device buffer kernel data fills (`kq_grid_dev`, the grid codebooks) | device state other than `kq_grid_dev`, engine types | +| `dasllama__common`
`dasllama_metal_common`, `dasllama_vulkan_common` | device state, buffer/command plumbing, hazard + capture rail, profiler, host-side quant-decode helpers (Metal's `iq4_lut`) | driver policy | | `dasllama__decode`
`dasllama_metal_decode`, `dasllama_vulkan_decode` | the resident token-step driver + decode-time arms | kernel bodies | | `dasllama__prefill`
`dasllama_metal_prefill`, `dasllama_vulkan_prefill` | the batched prefill driver + batch arms | kernel bodies | | `dasllama__shapes`
`dasllama_metal_shapes` | PORTABLE servability gates - no GPU C++ require, so any box can bake | device calls | @@ -88,7 +88,10 @@ that a question answered for one backend has an obvious address in the other. Th `set_moe_gpu_attn_dec_hooks`, the decode FFN tail `set_moe_gpu_ffn_tail_hooks`, the deltanet decode step's state seams `set_moe_gpu_dn_state_hooks` (flush, invalidate, release), the whole-token span `set_moe_gpu_span_dec_hook` - the span rides common's decode override - registry as `vulkan_moe_span`, selected by the MoE placement and declining per token). The + registry as `vulkan_moe_span`, selected by the MoE placement and declining per token - the + resident driver's q/k/v projection-bias seat `install_moe_gpu_resident_bias`, the OS + video-memory seat `install_moe_gpu_os_memory` the residency plan sizes against, and the + weight-bytes seat `install_rdec_note_weight_bytes` the decode warm-up guard reads). The installs are one-way: a test that arms the tier installs the seats and never restores them, because no uninstall exists and none is needed - a seat serves whatever model loads next; Metal deliberately does not, because UMA makes residency moot there and Metal @@ -223,7 +226,6 @@ its PORT/PORTH/PORTC arms and `benchmarks/attn/lcpp_flash_dk72.metal` are the or external reference the shipped `MetalTowerFlash` was decided against, and the DIAL arm races the production dialect against them - the bisect seat when the flash regresses. - - **Per-simdgroup register-fragment matmul2d (16x32x16, device -> `vec` fragments):** 1.7-2.0x slower for weight GEMMs, vectorized loads and deep n-blocking included. The fragment architecture pays for attention's streaming operand reuse (one resident Q against @@ -258,8 +260,10 @@ consecutive staging runs, relaxed_precision always - are `REVIEW_GPU.md` rules a `modules/dasMetal/REVIEW.das` descriptor gate; this section keeps only the refuted shapes and why they lose. -Sections 2.2j-2.2m and 2.2p-2.2q, the Vulkan resident driver, are `ARCHITECTURE_GPU_VULKAN.md`; -its 2.2n-2.2o - the residency plan and the marks swap - are `ARCHITECTURE_GPU_VULKAN_RESIDENCY.md`. +Sections 2.2j, 2.2p and 2.2ab, the Vulkan resident driver's prefill chain and byte stores, are +`ARCHITECTURE_GPU_VULKAN.md`; its 2.2k-2.2m and 2.2q - the cooperative-matrix GEMM tiles - are +`ARCHITECTURE_GPU_VULKAN_GEMM.md`; its 2.2n-2.2o - the residency plan and the marks swap - are +`ARCHITECTURE_GPU_VULKAN_RESIDENCY.md`. ### 2.2w The tower attention routes {#tower-attn-routes} diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_QUANT_PLANES.md b/modules/dasLLAMA/ARCHITECTURE_GPU_QUANT_PLANES.md index 26ce9b59cb..fcf277649e 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_QUANT_PLANES.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_QUANT_PLANES.md @@ -19,7 +19,10 @@ a region's bind offset must be a multiple of 512 elements - the `(off/256)*2` d- 4-byte aligned only then - which is what `metal_blob_off_ok` and `moe_site_ok` check. iq4nl is the exception: it reuses q40's 16-byte plane of eight f16 d per superblock, binds once, and ignores `doff`. The Vulkan tier does not use this form - it binds the decoded 20-byte row as five -uints per superblock. +uints per superblock, except for iq4xs, iq3s, iq3xxs and iq2xxs, whose device row is two words +(`kq_dev_ssb`): f16 d with the format's native high bits above it, then eight 4-bit sub-scale +fields - iq4xs's `scales_h` and `scales_l` verbatim, the others' `s` nibbles - which the +gather packs from the plane's decoded bytes and `iq4xs_sc` / `grid_sc` decode in the kernels. ### 2.2z The iquant GEMV grid read and its f4-slab twin {#metal-iquant-gemv-grid} diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md index 4bcdef7c97..cc4ebe2dc3 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md @@ -1,11 +1,13 @@ # dasLLAMA Architecture - the Vulkan resident driver Companion to `ARCHITECTURE_GPU.md`; section numbers are `ARCHITECTURE.md`'s. This document -carries sections 2.2j-2.2m and 2.2p-2.2q, the kernels and encode chains of the Vulkan resident -driver: the prefill window chain, how a cm2 tile decodes its quant bytes and how one is picked, -the class-pipeline build seat both shader instruments hang on, the Q8 requant byte store, and -the MoE expert chain on the cm2 tiles. What a model has to fit on the card before any of this -runs - the residency plan, and the marks swap that lets one GPU slot serve many models - is +carries sections 2.2j, 2.2p and 2.2ab, the prefill chain and the byte stores of the Vulkan +resident driver: the prefill window chain, the Q8 requant byte store, and the decode GEMV +family's grid codebook buffer. The cooperative-matrix tiles the chain's GEMMs run on - the cm2 +decode spelling, the tile pick and the coopmat mode ladder, the class-pipeline build seat, and +the MoE expert chain on those tiles - are `ARCHITECTURE_GPU_VULKAN_GEMM.md`'s sections 2.2k-2.2m +and 2.2q. What a model has to fit on the card before any of this runs - the residency plan, and +the marks swap that lets one GPU slot serve many models - is `ARCHITECTURE_GPU_VULKAN_RESIDENCY.md`'s sections 2.2n-2.2o. The decode-era mechanisms of the per-op tier are `ARCHITECTURE_GPU_VULKAN_DECODE.md`'s sections 2.2r-2.2v. The GPU backend role table these sections build on stays in `ARCHITECTURE_GPU.md` sec.1.5. @@ -27,8 +29,8 @@ gate, up and down GEMMs, the activation and the residual step of the last layer starting 32 rows below the window's end (`fill_arena_batch_sched`'s `row0`, `ActArgs.elem0`, `ArArgs.row0`). Thirty-two, not one, because the s tile's fast path loads a whole 32-row column unclamped and the resident prefill's activation planes (`pf_xf`, `pf_hf`) carry no read -slack past the window - unlike the MoE chain's gathered image and hidden plane, which sec.2.2l -sizes with 32 rows of slack past their last region. Rows below the +slack past the window - unlike the MoE chain's gathered image and hidden plane, which +`_GEMM.md` sec.2.2l sizes with 32 rows of slack past their last region. Rows below the slice keep stale gate, up, hidden and residual values that nothing reads. The sliced GEMMs do not split k: the split-k reduce sums partial planes from row 0, so a region starting below the window's end would reduce the wrong rows. The slice takes the f16-fed cm2 route only @@ -106,123 +108,6 @@ writes the `wo` feed plane directly, so the per-layer attn-to-f16 convert never f32 instance serves the quant route. The two device converts agree bit for bit; the CPU's `float16()` rounds ties differently, so the twin's gate compares device against device. -### 2.2k The cm2 decode callbacks read their quant bytes as 16-bit lanes {#cm2-decode-16bit-lanes} - -A cm2 tile's decode callback runs inside the driver's block load, and the vendor driver's shader -compiler pattern-matches only one spelling into that path: a 16-bit load (`int16[N]` block -members) followed by `unpack8(w)[i & 1u]` - a byte2 lane select - with sub-fields pulled out by -shift and mask. A 32-bit word with a variable shift runs slower; an `unpack8` of a 32-bit word -indexed by a runtime value (a byte4 dynamic select) drops the whole kernel off the block-load -path, to about a third of the rate. Every cm2 decode - q8 and the thirteen kq superblock formats - -is spelled the 16-bit way, which is why the block structs are `int16` arrays over the same -bytes. The IQ4_XS codebook is the one runtime-indexed read a decode makes: it is staged into a -16-entry `@workgroup` f16 table ahead of the tile loop (the reference exe's shared-memory table-staging form), -never selected out of a register vector per element. - -Every kq format's four-wide twin is hand-laid (`decode_v4`, the template's `DECV4` axis): it -keeps the same spelling and shares what four consecutive elements share. A K-quant twin reads -its four quant bytes as two 16-bit lanes and extracts the sub-block's scale pair once; a grid -format's twin looks its grid word up once and takes the four bytes and the four sign bits from -it. The synthesized twin (`DECVEC`, the axis a new format starts on) repeats the whole scalar -body four times - the lane selects, the scale-plane words, the grid lookup and the sign parity -- and on the grid formats it lost to the scalar callback for exactly that reason. The twin -computes each element in the scalar's operation order, so the tile's CPU oracle holds under -either callback; which callback a box runs is the `device ready` line's `four-wide decode`. - -### 2.2l The cm2 tile pick and the coopmat default ladder {#cm2-tile-pick-and-default} - -**The l/m tile pick is a wave-efficiency comparison.** For a GEMM of width `d` over `cnt` rows -the l tile (256-row columns) and the m tile (128-row columns) each need some number of -workgroups; each grid runs in whole waves over the device's SM count, and the pick compares -occupied slots over allocated slots, cross-multiplied. The m tile wins only on a strict win; a -tie goes to l, whose bigger tile carries twice the arithmetic intensity. Three rules sit ahead -of the comparison: a region of 64 rows or fewer takes the s tile (32-row columns - the MoE -expert-bucket shape, where a 512-token window routes ~32 rows to each of 128 experts and an m -column would pad three quarters of every tile and take the edge path on all of them), a window -of 128 rows or fewer takes m (the l column would run half empty), and a device that reports no -SM count takes l and never splits k. The pick is PURE in `(d, cnt, sm_count)`, so the class the -pipeline binds and the tile rule the meta fill writes can never disagree; `cnt` is the AVERAGE -rows per active region of the dispatch, so one tile serves every region of a MoE schedule. The -narrow-n end below s is GEMV's. The s tile's fast path loads a partial 32-row column UNCLAMPED -and clamps only the store, so every f16 plane the chain feeds it - the gathered activation -image and the hidden plane - is sized with 32 rows of slack past its last region -(`ffn_cm2_chunk_rows`). - -**The split-k pick counts the dispatch group, not the GEMM.** Long K (2048 and up) on a grid -that would fill under half the SMs splits the reduction across f32 partial planes that -`SplitKReduce` sums (three chunks up to two thirds full, at most eight, each chunk 256-aligned -and a split that would strand an empty tail shed). The grid it measures is the role's own -workgroups PLUS those of the chain neighbours it runs beside - q with k and v, gate with up -(`cm2_tiles`, the same pick each neighbour's own dispatch makes) - because the hazard-mask rail -lets independent roles co-run, while every split role serializes through the one scratch plane -(`VHZ_SK`) its neighbours would also claim. Counted alone, a 512-wide k or v projection over a -512-row window fills 16 of 36 SMs and splits in two; counted beside q it runs whole, and k and v -fill the device together. Split-k is left to the lone role - wo, down, a small model's -classifier - whose grid nothing else pads. - -**The f16 feed admits q8 and every kq superblock format** (`kq_sb`) - the set the cm2 decode -callbacks cover (sec.2.2k) - and each (format, tile) pair has ONE stamped class. The -prefill driver reaches them through one dispatcher per stage (`cm2_cls_ensure`, `cm2_cls_set`, -`cm2_cls_enc`), all three keyed on the same `(fmt, ml)` pair, so the pipeline a role ensures, -the set it binds and the kernel it encodes can never be three different classes. The three are -not private: the per-format arm of `harness/vk_gemm_probe.das` drives the same ladders, so a -probe row times the class the driver would serve rather than a copy of it. The decode -GEMV keeps its quant chains: the feed format pick is decoupled from the weight format. - -**The served GEMM mode resolves once, at init, through one ladder.** cm2 where the device has -NV_cooperative_matrix2, else mm where it has KHR_cooperative_matrix, else sdot4; -`DASLLAMA_COOPMAT` overrides the ladder by name, and a cm2 request or force on a device without -the extension lands on mm. The same resolver stamps the mode into the `.dlim` flavor -configuration, so the recorded mode and the running mode cannot drift. The four-wide decode -callback is NOT in that configuration: a cm2 tile names both callbacks -(`coopmatLoadTensorDecode`'s tenth argument: the format's own `decode_v4` where the template's -`DECV4` axis is on - every kq superblock format today, sec.2.2k - else the `DECVEC` axis, -which synthesizes the twin from the scalar body and is where a new format starts, its -`cm2:` probe row deciding whether a hand-laid twin is owed), the device created with -`DASLLAMA_VK_DECVEC` and the extension decides which -one the driver runs, and neither choice shapes an image byte, so the bake identity ignores it -(the configuration's own rule: a serve-only knob is never a field). `decvec_on` is the run's arm, -announced on the `device ready` line. - -**The tile's fast path is what makes the loads unclamped.** It runs when the weight tile is -whole (`m0 + 128 <= d`), the token column is whole or stamped s, and K is a whole number of BK -steps; the layouts are then created clamp-Undefined and the B and output strides are masked to -a multiple of 8 f16 (`stride &= ~7`). The mask is an identity on today's shapes - `n` and `d` -are 32-multiples - and it exists to make the alignment PROVABLE to the driver's address -analysis, which is what keeps the loads on the wide path. The s column gates only the weight -tile: its partial token column loads unclamped and its store clamps. Everything else takes the -edge path with clamped layouts. - -**The no-split arm keeps literal loop bounds and a literal store base.** Where `ksplit` is zero -the k loop runs the literal `0 .. n` with the store at the row base rather than the general -`k0`/`k1`/`ybase` form, although those values are exactly `0`, `n` and `0` on that path: the -general spelling cost 27% of prefill throughput (`benchmarks/lcpp_bench.das` pp512, 5060 Ti). -The split arm keeps the general form. - -### 2.2m Class-pipeline creation is the Vulkan tier's one shader A/B seat {#vk-class-pipeline-build} - -`vkd_class_pipe` is the single place a class kernel's SPIR-V becomes a pipeline, so both shader -instruments hang there and nothing else has to know about them. The four-wide decode fallback -hangs there too: when the device was created without `VK_NV_cooperative_matrix_decode_vector` -(`decvec_on` false) the served words go through `strip_decode_vector` (the capability, the -extension and every load's `DecodeVectorFunc` operand removed, the scalar callback left to -serve), after the override and before the shader module, so a dumped or overridden blob is -always the emitted, unstripped one. The seat is also the in-process A/B: `vkd_pipes_rebuild` -marks every class slot stale, so the next ensure rebuilds it under whatever `decvec_on` says, -which is how the `cm2:` probe runs both arms interleaved in one process. - -**The dump runs before the override.** `DASLLAMA_VK_SPV_DUMP=` writes the EMITTED words as -`/.spv`; `DASLLAMA_VK_SPV_OVERRIDE=` then replaces them with that directory's -file. The order is what makes the pair a round trip: dump a kernel, edit or spirv-opt the file, -serve it back. A dump taken after the override would capture the served words, not the emitted -ones. - -**Full subgroups are a whole-run arm, never a per-pipeline one.** `DASLLAMA_VK_FULLSG` plus a -device that reports the feature sets `g_gpu.full_sg_on` once at device init, and every class -pipeline is then built with `REQUIRE_FULL_SUBGROUPS`. A run never mixes pinned and plain -pipelines, so an A/B compares two whole runs. Plain is the default: pinned measured slower on -the mm_a gate shape. - ### 2.2p The Q8 requant writers store one quant per byte {#q8-requant-byte-store} Every requant writer on the class rail - the prefill and decode-tail kernels that write Q8_0 or @@ -235,29 +120,19 @@ the same under either form: the amax fold, the scale and the rounding decide the three sit above the store. The path needs the device's 8/16-bit storage feature set, which the family's device creator enables. -### 2.2q The MoE expert batch arm rides the cm2 tiles through a device-side f16 gather {#cm2-expert-chain} - -The per-op tier's expert FFN batch arm has two forms over the same region schedule. The quant -form takes the CPU's gathered activation image (the engine requantizes the normed rows, then -copies each bucket row's quants into expert order) and encodes the kq batch tiles. The f16 form -takes the window's f32 activation rows themselves - one per position - plus the combine's slot -map, and does the gather on the device: one workgroup per (position, slot) grid entry scatters -its position's row as f16 into the entry's bucket row, the inverse walk of the combine over the -same map. Gate and up then run the cm2 decode-in-load tiles over that f16 image, the act writes -the hidden plane as f16, and down runs the cm2 tiles again - the resident dense chain's -`pf_gemm_enc` feed, with the region records the quant form already fills. The engine asks the -tier per layer (`moe_gpu_ffn_xf_ok`): the answer is yes only in mode 4 on a coopmat2 device, for -a gate/up/down triple whose every format the f16 feed admits (sec.2.2l), with the window inside -the x plane's cap - and on yes it skips its own requant and gather, so the CPU cost of the -layer's FFN is the routing alone. The f16 form is the combined (`npos > 0`) form only: the -combine is what makes the device-side gather pay, since neither the gathered image nor the -bucket rows ever cross PCIe. Streamed groups take the same arm after the slot bind. - -**The per-op attention chain runs the same cm2 flash-attention tile the resident chain runs** -(`fa_cm2_h64` / `h128`, sec.2.2j) when the device carries the coopmat2-fa trio, the fa knob is -on, the head size is 64 or 128, and the model's attention is not gated - this chain wires neither -the h256 stamps nor their gated epilogue, so gated models keep the flash-style `at_attn` pass. The tile reads -f16 K/V: the chain keeps its f32 roped-k / raw-v planes at absolute positions for the host -readback the CPU cache store consumes, and fills f16 shadows of them with the base-less -`f16cvt` over the whole attended prefix each window; the fa output lands in the same out plane -`at_attn` writes, so the requant and `wo` stages never learn which pass ran. +### 2.2ab The decode GEMV family stages its grid codebooks from one device buffer {#kq-gemv-grid-buffer} + +The grid formats' codebooks (iq2s, iq2xs, iq2xxs, iq3s, iq3xxs) live in one 17 KB model-owned +device buffer, bound at binding 6 of the kq GEMV family and built on the family's first set +(`kq_grid_dev`); each format's table sits at a fixed word offset (`KQ_GRID_`), and a grid +format's GEMV stages its table into workgroup memory from that buffer - a coalesced 256 B read +per 64 lanes. The tables also exist as per-index accessors over a `fixed_array` local +(`iq2s_grid_word` and kin), which the batch and cm2 tiles stage from, because a tile amortizes +one stage over 128 rows x 64 columns. A per-row kernel cannot: the emitter lowers such an +accessor to a constant composite stored into a Function variable, the driver serves the +per-lane indexed read of it serially, and the two-row GEMV workgroup paid that serial read on +every 2 x 5120 weights it walked - iq2s streamed at 84 GB/s where k4 streams at 410. The +buffer form puts every grid GEMV in the k-format band (iq2s 388, iq2xs 407, iq2xxs 400, +iq3s 415 GB/s on the reference card, `harness/vk_gemv_probe.das`). The buffer dies in the +model-drop sweep with every other device buffer, and its handle zeroes there, so the next +model's first kq set rebuilds it. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_DECODE.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_DECODE.md index abe5af17af..0fd18caa30 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_DECODE.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_DECODE.md @@ -4,8 +4,9 @@ Companion to `ARCHITECTURE_GPU_VULKAN.md`; section numbers are `ARCHITECTURE.md` document carries sections 2.2r-2.2v: the decode attention block over per-layer K/V mirrors, the streamed expert layer's GPU/CPU split, the whole-token decode span, the deltanet decode step's per-session resident state, and the whole-model driver's hybrid token command. The -prefill window chain, the cm2 tiles and the MoE expert chain these build on are -`ARCHITECTURE_GPU_VULKAN.md` sections 2.2j-2.2m and 2.2p-2.2q; the residency plan and the marks +prefill window chain and byte stores these build on are `ARCHITECTURE_GPU_VULKAN.md` sections +2.2j, 2.2p and 2.2ab; the cm2 tiles and the MoE expert chain on them are +`ARCHITECTURE_GPU_VULKAN_GEMM.md` sections 2.2k-2.2m and 2.2q; the residency plan and the marks swap under them are `ARCHITECTURE_GPU_VULKAN_RESIDENCY.md` sections 2.2n-2.2o. ### 2.2r The per-op tier's decode attention block {#decode-attention-block} @@ -25,9 +26,12 @@ The PREFILL chain's readback is the other way round; its k rows come home roped. **The block serves one shape, and `attn_dec_shape_ok` is the whole gate.** The layer's q, k, v and o planes are resident, the k source is the layer itself (no shared KV), the session's cache is flat (not paged) and neither codec is tq4, the rope covers the whole head, and the model -carries no qkv or output bias, no v-norm, no attention sinks, no logit softcap, no q gate and no +carries no output bias, no v-norm, no attention sinks, no logit softcap, no q gate and no sliding window - the chain's kernels implement none of them. q, k and v also share one quant -class, because k and v read the q stack's activation image. +class, because k and v read the q stack's activation image. A q/k/v projection bias (qwen2moe) +is served: the layer's `[q | k | v]` row uploads once when its sets are made and binds where the +rope kernels expect it, so the mirror rows carry the bias; the raw rows that come home are the +GEMV output before it, and the host store adds the bias before its own norm and rope. **The mirror is per layer, keyed by the q plane offset, and capped** (`DAT_MIRROR_ROWS` rows; VRAM is layers x rows x kvd x 4 bytes). The loader reports that need @@ -35,7 +39,7 @@ VRAM is layers x rows x kvd x 4 bytes). The loader reports that need way the stream slots are carved: the mirrors are allocated after placement, and un-carved they came out of the desktop reserve and paged the resident expert stacks (the FFN chain's submit went from 147 us to 741 us). The prefill chain fills it from its own f16 K/V -shadows at the end of each layer's window (`ARCHITECTURE_GPU_VULKAN.md` sec.2.2q's fa arm converts the attended prefix; +shadows at the end of each layer's window (`ARCHITECTURE_GPU_VULKAN_GEMM.md` sec.2.2q's fa arm converts the attended prefix; the fill is a device copy of it), and the block appends one row per served token. **Ownership is a generation plus a per-layer count.** Every prefill claims ONCE, at the end of @@ -186,9 +190,10 @@ natively where this driver will be attempted or no GPU rail wants them (Metal of grouped, since a grouped file has no exact transcode, and either the whole-model driver armed on an expert-free model or a tier whose per-op dense and deltanet rails are unwanted; every other case transcodes them to q8, because Metal and the per-op rails bind deltanet q8 only), each GEMV -dispatches per format and a superblock plane takes the Q8_K x feed - while the out plane is q8, -since the step's o row feeds it as Q8_0 -(a K-quant out plane declines the layer); the beta and alpha rows are q8 arena planes when +dispatches per format and a superblock plane takes the Q8_K x feed. The out plane rides its +file format too: for a q8 plane the step writes the o row as Q8_0 blocks, for a superblock plane +it writes the f32 row and the Q8_K requant - the attention head's wo twin - makes the out GEMV's +feed, billed to the out role so the stamp count stands; the beta and alpha rows are q8 arena planes when the file carries them quantized, or - the F32-on-disk case - one f16 device copy of every recurrent layer's `[beta ; alpha]` rows that the router-form GEMV's f16 twin reads with an output base into the smalls. A hybrid takes the split activation rail (no fused add+rms+requant): the f32 @@ -211,6 +216,21 @@ mirror is sized `n_attn x seq_cap x kv_dim` and each attention layer carries its once on a recurrent layer. The attention geometry (head size, q and kv widths) is the first attention layer's - on qwen35 layer 0 is recurrent. +**A q/k/v projection bias (qwen2) folds into the rope stage.** The biased models' bias rows +upload once as one row per layer in the projection buffer's own `[q | k | v]` layout +(`vk_rdec_upload_bias`, a recurrent layer's row zero), bound at the last binding of the three +rope kernels - the decode rope+store, the prefill's batched twin and the fused qk-norm+rope - +which add the bias to each element as they read it, before the rotation (or the norm) and +before the v copy, so no dispatch is added: the CPU chain adds the bias between the projection +and the norm, and so does this. A model without a bias binds the norms buffer in that slot as +a placeholder the kernel never reads (`hasb` 0). The seat installs separately +(`install_moe_gpu_resident_bias`), so a tier without it names the bias in its decline instead +of serving the model unbiased. The per-op tier carries the same rows through its hooks: the +decode block binds the layer's row to the same rope kernels (sec.2.2r), and the prefill chain's +`AtPrep` stage adds the q and k rows before its norm and rope and runs a third pass over the raw +v window - `AtPrep` with no rope and no norm is a copy plus bias, in place - so the attention and +the v rows that come home both carry it (`ARCHITECTURE_GPU_VULKAN_GEMM.md`, the per-op chain). + **Gated attention and partial rotary ride the fused qk-norm+rope kernel and the decode attention kernel, not a detour.** On a gated model the q GEMV writes `2 x qd` rows in the loader's per-head `[q | gate]` layout; the fused kernel reads and writes q head-strided diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_GEMM.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_GEMM.md new file mode 100644 index 0000000000..2a0282626d --- /dev/null +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_GEMM.md @@ -0,0 +1,211 @@ +# dasLLAMA Architecture - the Vulkan tier's GEMM tile family + +Companion to `ARCHITECTURE_GPU_VULKAN.md`; section numbers are `ARCHITECTURE.md`'s. This +document carries sections 2.2k-2.2m and 2.2q, the cooperative-matrix tiles the Vulkan tier's +GEMMs run on: how a cm2 tile decodes its quant bytes, how a tile and the served GEMM mode are +picked, the class-pipeline build seat both shader instruments hang on, and the MoE expert chain +on those tiles. The prefill window chain that dispatches them, the Q8 requant byte store and the +decode GEMV family's grid codebook buffer are `ARCHITECTURE_GPU_VULKAN.md`'s sections 2.2j, 2.2p +and 2.2ab. What a model has to fit on the card before any of this runs is +`ARCHITECTURE_GPU_VULKAN_RESIDENCY.md`'s sections 2.2n-2.2o. The decode-era mechanisms of the +per-op tier are `ARCHITECTURE_GPU_VULKAN_DECODE.md`'s sections 2.2r-2.2v. The GPU backend role +table these sections build on stays in `ARCHITECTURE_GPU.md` sec.1.5. + +### 2.2k The cm2 decode callbacks read their quant bytes as 16-bit lanes {#cm2-decode-16bit-lanes} + +A cm2 tile's decode callback runs inside the driver's block load, and the vendor driver's shader +compiler pattern-matches only one spelling into that path: a 16-bit load (`int16[N]` block +members) followed by `unpack8(w)[i & 1u]` - a byte2 lane select - with sub-fields pulled out by +shift and mask. A 32-bit word with a variable shift runs slower; an `unpack8` of a 32-bit word +indexed by a runtime value (a byte4 dynamic select) drops the whole kernel off the block-load +path, to about a third of the rate. Every cm2 decode - q8 and the thirteen kq superblock formats - +is spelled the 16-bit way, which is why the block structs are `int16` arrays over the same +bytes. The IQ4_XS codebook is the one runtime-indexed read a decode makes: it is staged into a +16-entry `@workgroup` f16 table ahead of the tile loop (the reference exe's shared-memory table-staging form), +never selected out of a register vector per element. + +Every kq format's four-wide twin is hand-laid (`decode_v4`, the template's `DECV4` axis): it +keeps the same spelling and shares what four consecutive elements share. A K-quant twin reads +its four quant bytes as two 16-bit lanes and extracts the sub-block's scale pair once; a grid +format's twin looks its grid word up once and takes the four bytes and the four sign bits from +it. The synthesized twin (`DECVEC`, the axis a new format starts on) repeats the whole scalar +body four times - the lane selects, the scale-plane words, the grid lookup and the sign parity +- and on the grid formats it lost to the scalar callback for exactly that reason. The twin +computes each element in the scalar's operation order, so the tile's CPU oracle holds under +either callback; which callback a box runs is the `device ready` line's `four-wide decode`. + +### 2.2l The cm2 tile pick and the coopmat default ladder {#cm2-tile-pick-and-default} + +**The l/m tile pick is a wave-efficiency comparison.** For a GEMM of width `d` over `cnt` rows +the l tile (256-row columns) and the m tile (128-row columns) each need some number of +workgroups; each grid runs in whole waves over the device's SM count, and the pick compares +occupied slots over allocated slots, cross-multiplied. The m tile wins only on a strict win; a +tie goes to l, whose bigger tile carries twice the arithmetic intensity. Three rules sit ahead +of the comparison: a region of 64 rows or fewer takes the s tile (32-row columns - the MoE +expert-bucket shape, where a 512-token window routes ~32 rows to each of 128 experts and an m +column would pad three quarters of every tile and take the edge path on all of them), a window +of 128 rows or fewer takes m (the l column would run half empty), and a device that reports no +SM count takes l and never splits k. The pick is PURE in `(d, cnt, sm_count)`, so the class the +pipeline binds and the tile rule the meta fill writes can never disagree; `cnt` is the AVERAGE +rows per active region of the dispatch, so one tile serves every region of a MoE schedule. The +narrow-n end below s is GEMV's. The s tile's fast path loads a partial 32-row column UNCLAMPED +and clamps only the store, so every f16 plane the chain feeds it - the gathered activation +image and the hidden plane - is sized with 32 rows of slack past its last region +(`ffn_cm2_chunk_rows`). + +**The split-k pick counts the dispatch group, not the GEMM.** Long K (2048 and up) on a grid +that would fill under half the SMs splits the reduction across f32 partial planes that +`SplitKReduce` sums (three chunks up to two thirds full, at most eight, each chunk 256-aligned +and a split that would strand an empty tail shed). The grid it measures is the role's own +workgroups PLUS those of the chain neighbours it runs beside - q with k and v, gate with up +(`cm2_tiles`, the same pick each neighbour's own dispatch makes) - because the hazard-mask rail +lets independent roles co-run, while every split role serializes through the one scratch plane +(`VHZ_SK`) its neighbours would also claim. Counted alone, a 512-wide k or v projection over a +512-row window fills 16 of 36 SMs and splits in two; counted beside q it runs whole, and k and v +fill the device together. Split-k is left to the lone role - wo, down, a small model's +classifier - whose grid nothing else pads. + +**The f16 feed admits q8 and every kq superblock format** (`kq_sb`) - the set the cm2 decode +callbacks cover (sec.2.2k) - and each (format, tile) pair has ONE stamped class. The +prefill driver reaches them through one dispatcher per stage (`cm2_cls_ensure`, `cm2_cls_set`, +`cm2_cls_enc`), all three keyed on the same `(fmt, ml)` pair, so the pipeline a role ensures, +the set it binds and the kernel it encodes can never be three different classes. The three are +not private: the per-format arm of `harness/vk_gemm_probe.das` drives the same ladders, so a +probe row times the class the driver would serve rather than a copy of it. The decode +GEMV keeps its quant chains: the feed format pick is decoupled from the weight format. + +**The served GEMM mode resolves once, at init, through one ladder.** cm2 where the device has +NV_cooperative_matrix2, else mm where it has KHR_cooperative_matrix, else sdot4; +`DASLLAMA_COOPMAT` overrides the ladder by name, and a cm2 request or force on a device without +the extension lands on mm. The same resolver stamps the mode into the `.dlim` flavor +configuration, so the recorded mode and the running mode cannot drift. The four-wide decode +callback is NOT in that configuration: a cm2 tile names both callbacks +(`coopmatLoadTensorDecode`'s tenth argument: the format's own `decode_v4` where the template's +`DECV4` axis is on - every kq superblock format today, sec.2.2k - else the `DECVEC` axis, +which synthesizes the twin from the scalar body and is where a new format starts, its +`cm2:` probe row deciding whether a hand-laid twin is owed), the device created with +`DASLLAMA_VK_DECVEC` and the extension decides which +one the driver runs, and neither choice shapes an image byte, so the bake identity ignores it +(the configuration's own rule: a serve-only knob is never a field). `decvec_on` is the run's arm, +announced on the `device ready` line. + +**The mm mode serves the kq formats through the same template's KHR arm.** A device with +KHR_cooperative_matrix and no NV_coopmat2 (every AMD and Intel part, the GTX and Turing lines) +has no decode-in-load tensor API, so the cm2 template carries a second body under its `KHR` +axis: the f16 weight tile is staged by hand - each of the 256 threads copies one weight row's +16-wide half of the 32-wide k step through the format's OWN decode (`decode_v4` where the format +has one, else `decode`, the same methods the tensor load names as callbacks, here called +directly), the activation half from the f16 plane as two 16-byte words (the plane aliased as +`uint4` on the same binding; sixteen scalar f16 loads ran the tile at two thirds of the rate - +the staging's global loads, not the multiply-adds, set the pace), both into `@workgroup` `uint` +arrays of f16 pairs at a stride of 20 words (16 plus 4 pad, so the fragment loads spread across +banks; the activation words land as they are, the decoded weights packed two halves to a word) - +and eight subgroups each own a 16-row weight strip across the 128 tokens as eight 16x16 f32 +accumulators, two 16-deep multiply-adds per staged step. A whole tile stores its fragments +straight into y column-major at stride `d` (y is token-major, so a fragment's (weight, token) +is `y[token * d + weight]`); an edge tile bounces each fragment through the weight staging +array - free once the k loop ends, and its 2560 words hold the eight subgroups' 256-word +fragments - and writes under the row and column bounds through a bit cast. The two staging +arrays are the whole footprint by design: a third array for the bounce cost the tile a +seventh of its rate (the probe's `slab` arm, 28.5 against 33.0 TFLOP/s on the 4B gate shape), +because 8 KB more shared memory per workgroup is one workgroup fewer per SM. The k step is 32 +(the reference exe's BK): a 64-deep step doubles the staging tiles, halves the workgroups an SM +holds, and with the 8 KB iq2s grid beside them reaches the 49152 B of workgroup memory the tier +requires of a device - its floor; a 32 KiB part is not a target, and no kernel here is sized +for one. The arm exists at ONE geometry - 128 weights +by 128 tokens, k step 32 - so in mode 3 the tile pick answers 128 and split-k never engages, and +`cm2_cls_ensure/set/enc` route to the `khr_cls_*` ladders, the same `(fmt)` key on both. The +f16 feed admits a kq format in mode 3 only on a 32-lane subgroup (`khr_kq_tile_on`): the body +indexes eight subgroups over the 128 weight rows, so a wave64 device (four subgroups per +256-thread workgroup) keeps its kq planes on the sdot4 batch tile until the wave64 twin lands. q8 never arrives here +- its mm-mode GEMM is the q8-fed mul_mm L-tile. The direct call passes the plane element +itself (`decode_v4(wq[i], ...)`): the emitter hands the callee the element's index and the +body chains through the plane, so no block is ever copied. The copy form - a `let` of the +element passed by value, which the emitter spills to a Function-storage local - ran the same +tile at well under half the rate (8.0 against 19.3 TFLOP/s on the 4B gate shape with scalar +activation loads, 29.6 with the 16-byte ones), the whole 128 to 176 B block loaded from the +plane and stored to the local on every call, four calls per thread per k step; the +plane-element form measures within noise of hand-inlined decode arithmetic. The measured +alternatives that lost: B fragments loaded straight from the plane with no staging (17.1), a +64-deep k step (15.7 - its 40 KB footprint halves the workgroups an SM holds), and the separate +bounce slab (28.7). The rows are `harness/vk_gemm_probe.das -- khrx`, best of four interleaved +rounds on the RTX 5060 Ti: the shipped class reads 32.1 on the 4B gate shape, the probe's copy +of its body 33.2, and the weight stage replaced by a constant fill 41.2 - the ceiling the +staging's global loads leave. +Either way the decode body is authored once and serves the tensor load, the KHR staging and +the CPU oracle alike. + +**The tile's fast path is what makes the loads unclamped.** It runs when the weight tile is +whole (`m0 + 128 <= d`), the token column is whole or stamped s, and K is a whole number of BK +steps; the layouts are then created clamp-Undefined and the B and output strides are masked to +a multiple of 8 f16 (`stride &= ~7`). The mask is an identity on today's shapes - `n` and `d` +are 32-multiples - and it exists to make the alignment PROVABLE to the driver's address +analysis, which is what keeps the loads on the wide path. The s column gates only the weight +tile: its partial token column loads unclamped and its store clamps. Everything else takes the +edge path with clamped layouts. + +**The no-split arm keeps literal loop bounds and a literal store base.** Where `ksplit` is zero +the k loop runs the literal `0 .. n` with the store at the row base rather than the general +`k0`/`k1`/`ybase` form, although those values are exactly `0`, `n` and `0` on that path: the +general spelling cost 27% of prefill throughput (`benchmarks/lcpp_bench.das` pp512, 5060 Ti). +The split arm keeps the general form. + +### 2.2m Class-pipeline creation is the Vulkan tier's one shader A/B seat {#vk-class-pipeline-build} + +`vkd_class_pipe` is the single place a class kernel's SPIR-V becomes a pipeline, so both shader +instruments hang there and nothing else has to know about them. The four-wide decode fallback +hangs there too: when the device was created without `VK_NV_cooperative_matrix_decode_vector` +(`decvec_on` false) the served words go through `strip_decode_vector` (the capability, the +extension and every load's `DecodeVectorFunc` operand removed, the scalar callback left to +serve), after the override and before the shader module, so a dumped or overridden blob is +always the emitted, unstripped one. The seat is also the in-process A/B: `vkd_pipes_rebuild` +marks every class slot stale, so the next ensure rebuilds it under whatever `decvec_on` says, +which is how the `cm2:` probe runs both arms interleaved in one process. + +**The dump runs before the override.** `DASLLAMA_VK_SPV_DUMP=` writes the EMITTED words as +`/.spv`; `DASLLAMA_VK_SPV_OVERRIDE=` then replaces them with that directory's +file. The order is what makes the pair a round trip: dump a kernel, edit or spirv-opt the file, +serve it back. A dump taken after the override would capture the served words, not the emitted +ones. + +**Full subgroups are a whole-run arm, never a per-pipeline one.** `DASLLAMA_VK_FULLSG` plus a +device that reports the feature sets `g_gpu.full_sg_on` once at device init, and every class +pipeline is then built with `REQUIRE_FULL_SUBGROUPS`. A run never mixes pinned and plain +pipelines, so an A/B compares two whole runs. Plain is the default: pinned measured slower on +the mm_a gate shape. + +### 2.2q The MoE expert batch arm rides the cm2 tiles through a device-side f16 gather {#cm2-expert-chain} + +The per-op tier's expert FFN batch arm has two forms over the same region schedule. The quant +form takes the CPU's gathered activation image (the engine requantizes the normed rows, then +copies each bucket row's quants into expert order) and encodes the kq batch tiles. The f16 form +takes the window's f32 activation rows themselves - one per position - plus the combine's slot +map, and does the gather on the device: one workgroup per (position, slot) grid entry scatters +its position's row as f16 into the entry's bucket row, the inverse walk of the combine over the +same map. Gate and up then run the cm2 decode-in-load tiles over that f16 image, the act writes +the hidden plane as f16, and down runs the cm2 tiles again - the resident dense chain's +`pf_gemm_enc` feed, with the region records the quant form already fills. The engine asks the +tier per layer (`moe_gpu_ffn_xf_ok`): the answer is yes only in mode 4 on a coopmat2 device, for +a gate/up/down triple whose every format the f16 feed admits (sec.2.2l), with the window inside +the x plane's cap - and on yes it skips its own requant and gather, so the CPU cost of the +layer's FFN is the routing alone. The f16 form is the combined (`npos > 0`) form only: the +combine is what makes the device-side gather pay, since neither the gathered image nor the +bucket rows ever cross PCIe. Streamed groups take the same arm after the slot bind. + +**The per-op attention chain runs the same cm2 flash-attention tile the resident chain runs** +(`fa_cm2_h64` / `h128`, `ARCHITECTURE_GPU_VULKAN.md` sec.2.2j) when the device carries the coopmat2-fa trio, the fa knob is +on, the head size is 64 or 128, and the model's attention is not gated - this chain wires neither +the h256 stamps nor their gated epilogue, so gated models keep the flash-style `at_attn` pass. The tile reads +f16 K/V: the chain keeps its f32 roped-k / raw-v planes at absolute positions for the host +readback the CPU cache store consumes, and fills f16 shadows of them with the base-less +`f16cvt` over the whole attended prefix each window; the fa output lands in the same out plane +`at_attn` writes, so the requant and `wo` stages never learn which pass ran. + +**The per-op attention chain adds a q/k/v projection bias (qwen2moe) in its prep stage.** The +layer's `[q | k | v]` row uploads to one device buffer per call and binds to `AtPrep`, whose q and +k passes add their slice (`boff` 0 and `qd`) to each projection element before the norm and the +rope - where the CPU chain adds it. v has no prep pass of its own, so a biased layer runs a third +`AtPrep` over the raw v window with the rope half 0 and the norm off, which makes the kernel a +copy plus bias in place (`boff` `qd + kv_dim`); the copy into the absolute-position v plane and +the host readback then both carry the bias. A model without the bias runs the two passes with +`hasb` 0 and never reads the binding, and the rail's `arch_ok` no longer names the bias. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_RESIDENCY.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_RESIDENCY.md index f3837f9881..96e5f934b1 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_RESIDENCY.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN_RESIDENCY.md @@ -2,11 +2,12 @@ Companion to `ARCHITECTURE_GPU_VULKAN.md`; section numbers are `ARCHITECTURE.md`'s. This document carries sections 2.2n-2.2o: the residency plan that sizes a whole model before a byte -uploads, and the marks swap that lets one GPU slot serve many models. The kernels and encode -chains that run once a model is resident are `ARCHITECTURE_GPU_VULKAN.md` sections 2.2j-2.2m -and 2.2p-2.2q; the per-op tier's decode era is `ARCHITECTURE_GPU_VULKAN_DECODE.md` sections -2.2r-2.2v. The GPU backend role table these sections build on stays in `ARCHITECTURE_GPU.md` -sec.1.5. +uploads, and the marks swap that lets one GPU slot serve many models. The prefill chain and byte +stores that run once a model is resident are `ARCHITECTURE_GPU_VULKAN.md` sections 2.2j, 2.2p +and 2.2ab, and the cooperative-matrix GEMM tiles under them are +`ARCHITECTURE_GPU_VULKAN_GEMM.md` sections 2.2k-2.2m and 2.2q; the per-op tier's decode era is +`ARCHITECTURE_GPU_VULKAN_DECODE.md` sections 2.2r-2.2v. The GPU backend role table these +sections build on stays in `ARCHITECTURE_GPU.md` sec.1.5. ### 2.2n The residency plan sizes a whole model before a byte uploads {#resident-plan} @@ -18,13 +19,65 @@ grows: on a discrete card the two compete directly, and evicting weights to grow re-uploading gigabytes. A decline carries a reason, and where the numbers allow one it carries the remedy that works - a shorter context, because the weights are fixed and the KV is not. -**The auto arm's headroom is the larger of 2 GiB and 27% of the tier's cap.** WDDM demotes a -process's buffers to system memory by how full the card is, not by a fixed leave-behind, and a -demoted plane reads at PCIe speed with no error: on the 16 GB reference card the 9B hybrid at a -12.5 GB plan (the fixed 2 GiB headroom under the 14.7 GB cap) decoded at 6.8 tok/s, at 12 GB -pinned 3.4, at 11 GB pinned 49.8. The share keeps a 16 GB card's plan near 10.7 GB and leaves -an 8 GB card's plan where the fixed term already put it. A pinned `VRAM_MB` takes no headroom: -the number is the user's contract, and the plan fills it. +**The mirror's context is capped by the device's single-binding range before any byte is +counted.** Each K/V side binds as one SSBO range, so `seq_cap` is at most `maxStorageBufferRange` +over the side's bytes per position summed across the layers (`resident_binding_ctx`). The sum is +what makes a hybrid right: a recurrent layer's KV width is zero, so the stride counts the +attention layers alone; read from layer 0 instead - recurrent on Qwen3.5 - the stride is zero, +the cap is skipped, the plan sizes a mirror the device prepare refuses, and the whole driver +declines to the per-op rails with the dense FFN on the CPU. The prepare keeps its own guard for +direct callers. + +**The auto arm sizes against the room the OS reports on the adapter, where the OS reports +it.** Vulkan cannot see the desktop: `VK_EXT_memory_budget` reads a flat 16024 MB on the 16 GB +reference card with 3 GiB of another process's memory resident, and so does the process's own +WDDM budget (`D3DKMTQueryVideoMemoryInfo`) - on the NVIDIA driver that number is the card minus +the OS reserve, whatever anyone else holds. What does carry the desktop is the adapter's +system-wide dedicated usage, the "GPU Adapter Memory / Dedicated Usage" performance counter +summed over the adapter's segments. The boost helper `os_video_memory` returns all three - +the ceiling, this process's usage, the adapter's usage - for the physical device's LUID, and +the tier installs it as the resident plan's `moe_gpu_os_memory` source. The plan's usable +bytes are then the tier's cap, or the ceiling minus what every other process holds minus a +1 GiB reserve, whichever is smaller; the headroom share is zero. The reserve is not slack: the +process's dedicated footprint runs about 0.7 GB past the plan's own tally (arena slab tails, +descriptor pools, pipelines, the transfer queue's staging - a 14.0 GB plan held 14.66 GB), and +a decline names what the other processes hold. Measured untagged on that card with a 0.63 GB desktop: plans of 13.35, 14.0 and 14.6 GB +(the card at 14.7, 15.3 and 15.9 GB dedicated) all decoded the 27B at full rate; with 2 GiB +more held by another process the same 13.35 GB plan lost 1.7 GB to system memory and decoded +at 4.35 tok/s, and with 4 GiB more the device returned all-zero logits with no error. The +manager never demotes the idle other process; ours is the victim, so the room left is the only +honest input. A pinned `VRAM_MB` takes no headroom and reads no room: the number is the user's +contract, and the plan fills it. Every rate in this section is a `benchmarks/lcpp_bench.das` +tg128 reading (`-jit --for-debug-purposes`, `DASLLAMA_IMAGE=0`, the rig `plans/qwen_arc_board.md` +states) on the zen2 box's RTX 5060 Ti 16 GB, driver 616.56. + +**Where no OS answers, the auto arm's headroom is the larger of 2 GiB and 27% of the tier's +cap.** That share keeps a 16 GB card's plan near 10.7 GB and leaves an 8 GB card's plan where +the fixed term already put it; it declines every 27B file on a 16 GB card, and the ladder +behind it (the 9B hybrid at 6.8, 3.4 and 49.8 tok/s across 12.5, 12 and 11 GB plans) was taken +with every allocation tagged priority 1.0, the arm that made the planes pageable. Linux's Mesa +drivers fill `heapBudget` from the kernel's system-wide accounting and the NVIDIA driver fails +an allocation past the card out loud, so the share is a coarse stand-in there, not a measured +knee. + +**Two guards stand whatever the plan decided.** The resident decode panics on an all-zero +logits row (the first four tokens and every 256th are scanned) naming the over-commit, and at +its eighth token it reads the demotion tell and measures the token's wall against the weights +streamed at 40 GB/s, the PCIe class; either past its line logs the remedy once. Every resident +prefill reads the tell the same way. The tell is the adapter's shared usage - system memory the +GPU maps - grown past the desktop's baseline at tier init plus every live host-visible buffer +of ours (`make_host_buf`'s tally) by more than 512 MB: a demoted plane lands exactly there +(the 27B beside a 2 GiB hog read 3.07 GB shared against 0.67 GB of its own host buffers), while +the dedicated-usage counters can sit a few MB over the soft ceiling with nothing demoted. + +**Device allocations carry no memory-priority tag.** The tier enables +`VK_EXT_memory_priority` and `VK_EXT_pageable_device_local_memory` on the device (the boost +creator does, whenever the card reports them) but chains no priority into an allocation +unless `DASLLAMA_VK_MEMPRIO=1` asks. On the NVIDIA WDDM driver the priority-1.0 tag is what +makes an allocation pageable: the 27B UD-IQ4_XS at a 13.35 GB plan on the 16 GB card held +12.26 GB dedicated with 1.7 GB demoted and decoded at 4.2 tok/s tagged, and held 14.66 GB +dedicated at 23.5 tok/s untagged, same run shape, same desktop (llama.cpp, which tags only +behind its own opt-in, sat at 13.7 GB dedicated and 24.1). The tag is an A/B arm. An OPTIONAL plane rides only the room left under the budget at THIS context - what remains of `budget_bytes - headroom_bytes` after weights, KV and scratch; the reserved headroom itself diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index 22a6098ea9..b6d925d939 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -52,7 +52,15 @@ axis to any kq superblock format: it drives the prefill's own (format, column) l l and m columns with the kq batch tile as the control row, over random block bytes at that format's device block size, and it runs the four-wide decode's two arms (the twin served, then stripped through `vkd_pipes_rebuild`) interleaved in one process, two rounds each, so a format's -`DECVEC` verdict comes from one instrument. A new arm joins one of the three. +`DECVEC` verdict comes from one instrument. The `khrx` arm is the second axis for the KHR kq +tile: the staged k4 tile with its weight stage as the variable (a constant fill, the decode on +the plane element, the plane's words read in place), then the structural variants (unstaged B +fragments, a 64-deep k step, a separate edge-store slab), with the shipped KHR class and the +sdot4 tile as control rows. The `mmqx` arm is the first axis for the integer tile: the sdot4 +k4 tile against register-block prototypes over the same planes. Both sweeps read every +product-computing arm back against the shipped class, and time the served graph's shape: +sixteen dispatches per submit over two alternating outputs with a fresh hazard each, the arms +interleaved round by round, an arm's figure its best round. A new arm joins one of the three. **A measured number proves its kernel provenance through `tune_gate()` (`performance/profile_common.das`), one arm per world it can run in.** Four worlds, because @@ -65,9 +73,10 @@ before its first timed rep. Two rig shapes fall outside "measuring entry point" property itself, ledgered here: a kernel A/B lab dispatches its variants through its own arms (never the `[tune]` selection), and `lcpp_bench.das`'s `--tok` cell dispatches no kernels at all - neither can measure a fallback silently. A kernel A/B lab is also outside the -in-process reference check: `harness/vk_gemm_probe.das` dispatches the shipped, suite-gated -kernels on timing fixtures, compares no arm's output, and marks every row `timing-only`; its -rows never enter a record store, and a decision it seeds is confirmed by the e2e board rows. +in-process reference check: it dispatches shipped kernels and prototypes on timing fixtures, +compares an arm's output against its own reference arm where the outputs are comparable and +marks the rest `timing-only`; its rows never enter a record store, and a decision it seeds is +confirmed by the e2e board rows. **A binary-stale sidecar still serves its `runtime` section; a foreign one serves nothing.** The staleness rule kills measured kernel WINNERS - a rebuild can change the bodies they were @@ -134,9 +143,8 @@ stages, tokenizer build) - the rails do not apply. A timing that is part of an A the facade's `TtsTimings`, the per-stage walls a synthesis returns to its caller and the server logs per request - is a deliverable of the same kind, not instrumentation, and the one-rail follow-up (`followup_general.md` row 72) keeps it that way while it retires the -duplicate rails. A clock whose value feeds logic is control -flow, not instrumentation; it is marked `// clock: control` so the sweep and any future lint -leave it alone. +duplicate rails. A clock whose value feeds logic is control flow, not instrumentation; it is +marked `// clock: control` so the sweep and any future lint leave it alone. The override-announce rule (REVIEW.md) draws its boundary here: a knob or setter whose purpose is timing still counts as an override when it moves computed numerics - two GEMM forms of the @@ -144,7 +152,6 @@ same math differ in float terms - while one that changes only WHEN work happens a CLI flag is never an override (it is the run's own command line, visible where the run is launched). - ### Re-stamping inside the content-addressed archive A sidecar archived as `records/.tune..json` is content-addressed: its filename diff --git a/modules/dasLLAMA/ENVIRONMENT.md b/modules/dasLLAMA/ENVIRONMENT.md index 56aca21eb8..4e1a350930 100644 --- a/modules/dasLLAMA/ENVIRONMENT.md +++ b/modules/dasLLAMA/ENVIRONMENT.md @@ -129,7 +129,7 @@ Vulkan GPU backend. Present only where the dasVulkan package is installed. | `DASLLAMA_VK_DECVEC` | flag | on | Run the cm2 tiles' four-wide decode callback where the device has VK_NV_cooperative_matrix_decode_vector; 0 strips it and serves the scalar callback - the same-build A/B and the fallback probe. | | `DASLLAMA_VK_IMPORT` | flag | on | Stream mirrors import the mapped .dlim (VK_EXT_external_memory_host) instead of pinned copies; =0 restores the copy path. | | `DASLLAMA_TRIM` | flag | off | Serve from P3-trimmed vulkan images (big CPU weight families dropped; folded into the flavor identity). | -| `DASLLAMA_VK_MEMPRIO` | flag | on | Tag allocations high-priority (VK_EXT_memory_priority) so the driver demotes desktop memory, not ours. | +| `DASLLAMA_VK_MEMPRIO` | flag | off | Tag every device allocation priority 1.0 (VK_EXT_memory_priority). Off by default: on the NVIDIA WDDM driver the tag makes the tagged weights the pageable set, and a resident model then decodes at PCIe speed once anything is demoted. | | `DASLLAMA_VK_FA` | flag | on | Vulkan flash attention: the decode fa kernel pick AND the cm2 prefill fa tile; 0 falls back to the chunked/scalar paths. | | `DASLLAMA_VK_KV_MERGE` | flag | on | Merged k|v prefill GEMM - one dispatch over the adjacent k+v arena planes; 0 pins the split k + v dispatches for a same-build A/B. | | `DASLLAMA_VK_FFN_SLICE` | flag | on | The last layer's FFN runs on the window's last 32 rows only (the classifier reads one); 0 runs it over the whole window for a same-build A/B. | @@ -138,8 +138,8 @@ Vulkan GPU backend. Present only where the dasVulkan package is installed. | `DASLLAMA_VK_FULLSG` | flag | off | Pin REQUIRE_FULL_SUBGROUPS on every class pipeline (instrument; measured slower than plain pipelines on the mm_a gate shape, so those are the default). | | `DASLLAMA_VK_REBAR` | flag | on | Use a ReBAR device-local host-visible heap when one larger than 1GB is present. | | `DASLLAMA_VK_KV32` | number | 0 | Arm the resident driver with f32 KV mirrors instead of the f16 default (A/B instrument; only sessions of the armed codec are served). | -| `DASLLAMA_CM2_TILE` | number | 0 | cm2 prefill tile pick: 0 = occupancy heuristic, 128 = force the m tile, 256 = force the l tile (A/B instrument). | -| `DASLLAMA_CM2_SPLITK` | number | 0 | cm2 split-k: 0 = occupancy heuristic, 1 = off, N = force N k-chunks (A/B instrument; shrinks if N strands an empty tail). | +| `DASLLAMA_CM2_TILE` | number | 0 | cm2 prefill tile pick: 0 = occupancy heuristic, 128 = force the m tile, 256 = force the l tile (A/B instrument). Inert on the KHR arm (DASLLAMA_COOPMAT=mm, or a device without NV_coopmat2), whose kq tile has one geometry. | +| `DASLLAMA_CM2_SPLITK` | number | 0 | cm2 split-k: 0 = occupancy heuristic, 1 = off, N = force N k-chunks (A/B instrument; shrinks if N strands an empty tail). Inert on the KHR arm, whose kq tile carries no split-k scratch. | | `DASLLAMA_VK_SPV_OVERRIDE` | path | unset | Directory of .spv files served instead of the emitted words at pipeline creation (offline spirv-opt / hand-patched A/B instrument). | | `DASLLAMA_VK_SPV_DUMP` | path | unset | Directory to write each kernel's emitted words as .spv at pipeline creation (the override instrument's capture half). | | `DASLLAMA_VK_HAZARD_PARANOID` | flag | off | Barrier at every dispatch (correctness bisect). | diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index ad14fbdd43..391af28fd6 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -162,8 +162,9 @@ trusting its numbers. ## 6. Vulkan - `dasllama_vulkan_classes.das`, `dasllama_vulkan_common.das` The tier reads the CPU planes verbatim (`stack_plane_bytes` -> `arena_block_bytes` -> -`kq_qsb(vk_kq_schema_id(fmt))` x `KQ_DEV_SSB`), so a format whose plane pair already has the -20 B decoded scale row needs no upload work - only the id bridge and the kernels. IQ4_XS took: +`kq_qsb(vk_kq_schema_id(fmt))` x `kq_dev_ssb(fmt)` - 20 B for a decoded row, 8 B for the codebook +formats' two-word row the gather packs), so a format whose plane pair already has the 20 B +decoded scale row needs no upload work - only the id bridge and the kernels. IQ4_XS took: 1. `vk_kq_schema_id` (`dasllama_vulkan_common.das`): the `int(KqFmt)` -> kernel-id arm (`6 -> 44`). This is the third id space of QUIRK 5 at its Vulkan seam; without the arm the @@ -172,8 +173,8 @@ The tier reads the CPU planes verbatim (`stack_plane_bytes` -> `arena_block_byte (`wq4[wsb * 8 + blk]`), each nibble word decoded through `iq4_word` (a `fixed_array` LUT local - the SPIR-V emitter lowers a `let` fixed array to a Function-storage variable and indexes it) into SIGNED lanes for `sdot4` (OpSDot, signed x signed - the block-sum trick of - q40/k4 does not apply and is not needed), scale `d * sc` with `sc` the signed byte off word - 1..2 of the 5-word row (`unpack8` sign-extends, the k6 spelling). + q40/k4 does not apply and is not needed), scale `d * sc` with `sc` decoded by `iq4xs_sc` off + the two-word device row (`scales_h` above d in word 0, the `scales_l` nibbles in word 1). 3. `KqBatchIq4xs : KqBatchBase` - `stage_w` decodes the staged words through `iq4_word` (k4's staging otherwise), `stage_ws` fills ONE plane with `d * sc`, `blk_fma` is `xscl * ws * idot` (q40's without the `- 8 * bsum`). @@ -185,6 +186,12 @@ The tier reads the CPU planes verbatim (`stack_plane_bytes` -> `arena_block_byte `iq4_word`) - `iq4xs_gemv_float_oracle`, a float dequant straight off the plane bytes that the class oracle must match. +A grid format adds one more: its table joins the family's grid buffer (`kq_grid_dev` - a +`KQ_GRID_` word offset, `KQ_GRID_WORDS` / `KQ_GRID_BYTES` grown, the accessor called +once per word into the host image) and the GEMV's `run` stages `gridb[KQ_GRID_ + idx]` +into its `@workgroup` table - never the `*_grid_word` accessor, which the batch and cm2 tiles +keep (`REVIEW_GPU_VULKAN.md`, `ARCHITECTURE_GPU_VULKAN.md` sec.2.2ab). + Not done, by ruling: a cm2 decode-in-load tile (`[spirv_decode] def decode_iq4xs`). The f16 feed admits q8/k4/k6 only (`pf_f16_feed`), q40 and k5 have no cm2 tile either, and `followup_vulkan.md` item 24 rules that new formats land on the one class template, not as @@ -195,7 +202,7 @@ three more hand-stamped bodies. IQ4_XS prefill rides the kq batch tile like q40 On an NV_coopmat2 device the f16 feed serves every kq format through ONE tile template (`KqCm2BatchT`): a new format is a format template authoring `[spirv_decode] def decode` over the DEVICE forms (quants as the gather lays them out - k4/k5 re-paired k/k+16, q40/iq4xs/k3 -verbatim; scales the 20 B `KQ_DEV_SSB` row) plus three eight-line width stamps, arms in the +verbatim; scales the `kq_dev_ssb(fmt)` row - 20 B decoded, or the codebook formats' two words) plus three eight-line width stamps, arms in the `cm2_cls_ensure/set/enc` ladders, and `pf_f16_feed` admits it via `kq_sb` automatically. A codebook format raises the `IQLUT` axis - a gated `@workgroup` f16 table staged ahead of the tile loop (llama.cpp's `init_iq_shmem` form); never select codes out of a register vector per @@ -213,6 +220,16 @@ device-form CPU oracle an l/m/s cell in `tests/test_vulkan_kernels.das`. Payoff on the 1B: iq4xs pp512 5161 -> 15334, k3 5174 -> 14031 (0.90x / 0.80x llama.cpp's Vulkan, from 0.30x). +The KHR instantiation rides the same decode (`ARCHITECTURE_GPU_VULKAN_GEMM.md` sec.2.2l, the mm-mode +paragraph): `KhrBatch : Cm2T` with `override KHR = true`, `override BN = 128u`, the +four cm2 typedefs the uncalled tensor body still names (`BT`, `ACC`, `ACCW`, `FLO` - copy k4's), +a `[vk_dispatch(name = "kq_batch__khr_cls", ...)]`, an arm in each of `khr_cls_ensure/set/enc` +(`dasllama_vulkan_prefill.das`), and the format's kernel cell runs its fourth arm (`ml == 3`, tile +128) wherever the device has KHR coopmat at subgroup 32 - on the 5060 Ti the same run covers the +cm2 l/m/s tiles and the KHR tile. No new decode, no new oracle: the KHR arm calls the format's +`decode`/`decode_v4` directly on the plane element (`decode_v4(wq[i], ...)` - never on a `let` +copy of it, which runs at a third of the rate) and the `f16_gemm_oracle` already holds it. + ## 7. Metal - `dasllama_metal_kernels.das`, `_common`, `_prefill`, `_shapes`, `dasllama_layout.das` Done for IQ4_XS over ssh on the M1 (the tier compiles MSL and runs only there). The tier is @@ -961,7 +978,8 @@ ksigns table anywhere - the disk carries explicit signs), scale = f16 d x (1 + 2 sub-scale); signed reconstruction, so no `xbsp` term. Disk block 110 B: f16 d, 64 qs, 8 qh, 32 signs, 4 packed scale nibbles. Plane pair: quants [qs 64][qh 8][signs 32] VERBATIM (104 B - the k3 "disk is already the device form" answer), scale row = the iq4xs/k4 20 B shape with -(1 + 2s) decoded at transcode, so every k4-row consumer serves unchanged. Ids: `KqFmt.iq3s` +(1 + 2s) decoded at transcode, so every k4-row consumer serves unchanged; on Vulkan the device +row is the two-word codebook form (`iq3_sc`). Ids: `KqFmt.iq3s` = 8, kernel id 33, stream code 33. The codebook question a grid format adds: the 2 KB table ships as `iq3s_grid()` - the per-call-local worker-safe form `IQ4NL_LUT` documented, just 512 entries now - plus the main-context `IQ3S_GRID`; the repack is 26 uniform 4-byte columns (no @@ -1130,7 +1148,8 @@ Shape: 256-superblock, codebook nibble (`kvalues_iq4nl`, signed, no offset), sca 6-bit sub-scale - 32. Disk block 136 B: f16 d, u16 scales_h, 4 scales_l bytes, 128 nibbles paired k/k+16 per 32-block. Plane pair: quants verbatim (the q40 tiling), scale row 20 B decoded at transcode ([f16 d][2 pad][8 x int8 (ls-32)][8 pad] - the k4 decoded-row shape, so -`repack_k4_grp`'s scale addressing is reused verbatim in `repack_iq4xs_grp`). Dot fold: +`repack_k4_grp`'s scale addressing is reused verbatim in `repack_iq4xs_grp`; the Vulkan gather +packs it back to the disk's two words, `iq4xs_sc` decodes them). Dot fold: `sum_blk sc * sum_k LUT[q] * x`, then `* (d * xs)` per superblock; no `xbsp` term. Ids: `KqFmt.iq4xs` = 6, kernel id 44, stream code 44. Gates green on the interpreter binary AND under `-jit`: `test_kqformat` 14/14, `test_kquant` 114/114 (iq4xs arms: transcode+dequant diff --git a/modules/dasLLAMA/README.md b/modules/dasLLAMA/README.md index f8750c8fce..6d079d5944 100644 --- a/modules/dasLLAMA/README.md +++ b/modules/dasLLAMA/README.md @@ -101,7 +101,8 @@ modules/dasLLAMA/ ARCHITECTURE_ENGINE.md # companion: the engine, format, load, CPU-tier, support and serving charters ARCHITECTURE_GPU.md # companion: the GPU backend role table, the backend asymmetries, the refuted kernel shapes ARCHITECTURE_GPU_PREFILL.md # companion: the Metal prefill driver's GEMM ladder - ARCHITECTURE_GPU_VULKAN.md # companion: the Vulkan resident driver - window chain, cm2 tiles, class pipelines, expert chain + ARCHITECTURE_GPU_VULKAN.md # companion: the Vulkan resident driver - window chain, Q8 requant store, GEMV grid codebooks + ARCHITECTURE_GPU_VULKAN_GEMM.md # companion: the Vulkan GEMM tiles - cm2 decode lanes, tile pick, class pipelines, expert chain ARCHITECTURE_GPU_VULKAN_RESIDENCY.md # companion: the Vulkan residency plan and the GPU-slot marks swap ARCHITECTURE_MEDIA.md # companion: the encoder-tower, audio, ASR and vision charters ARCHITECTURE_IMAGE.md # companion: the prepared-image (.dlim) rail diff --git a/modules/dasLLAMA/REVIEW.das b/modules/dasLLAMA/REVIEW.das index c8fd821ef2..c1b744e4bb 100644 --- a/modules/dasLLAMA/REVIEW.das +++ b/modules/dasLLAMA/REVIEW.das @@ -544,8 +544,8 @@ let private IMAGE_FILE = "modules/dasLLAMA/dasllama/dasllama_image.das" // every *_prepare mint, the layout helpers and the layout constants' declaration lines, hashed // in file order. A closure change with IMAGE_VERSION unmoved is red; the finding prints the // value to re-stamp with. -let private IMAGE_LAYOUT_STAMP_VERSION = 33 -let private IMAGE_LAYOUT_STAMP_HASH = 0x49b8e09188e9d74ful +let private IMAGE_LAYOUT_STAMP_VERSION = 35 +let private IMAGE_LAYOUT_STAMP_HASH = 0x4e552ab5bff69e17ul // The helpers that decide WHERE bytes land: the page pad, the plane and total sizing, the // writer's append / zero-fill / header patch, and the header's scalar stores. Changing one diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index b2f1bc2485..be4b71483f 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -115,8 +115,8 @@ timing two candidates against each other. A reading taken across two processes, commits, says which way the wall-clock time moved, not which implementation to adopt. **A change to an allocation reached from a load, bake, or convert path (judge a shared helper -at each call site) that trades footprint for speed ships the measured pair - peak footprint and -wall-clock - in `PERF_LEDGER.md` with the decision it settles.** +at each call site) that trades footprint against wall-clock, either way, ships the measured +pair - peak footprint and wall-clock - in `PERF_LEDGER.md` with the decision it settles.** **A new call to an f32 matmul (`matmul_batch`, `mm_blob_b`, per-head `gemm_f32` / `gemm_f32_jo`, or an f32 GPU mm) outside a correctness-comparison path (one whose only job is @@ -130,8 +130,8 @@ without first proving both stdin and stdout are terminals is a defect - emit the a `@sidecar` event instead.** A supervised or piped boot must never block on input. **A print or log of an elapsed interval whose site is in an engine file (`dasllama/`), outside a -cold one-shot load, bake, map or tokenizer-build progress log and the first-start race report -(`ARCHITECTURE_MEASUREMENT.md` sec.2.42a), is a defect** - instrumentation goes through the +cold one-shot load, bake, map or tokenizer-build progress log and the fat exe's first-start +kernel race report, is a defect** - instrumentation goes through the profiling rails (`profile_tag` / `profile_marker`, `prof_add`, `asr_prof_add`, the Vulkan tier's `vk_prof()`-gated ledgers), `ARCHITECTURE_MEASUREMENT.md` sec.2.10. @@ -145,8 +145,10 @@ quantum - one batch of prompt tokens the prefill path processes in a single pass by an annotated region entry** - `[hot_path]`, any of the `[no_alloc]` / `[no_env]` / `[no_io]` contracts, or `[cold_path]` on its only reaching entry. Covered means an annotated entry reaches it: an annotation binds every function the entry calls, so an interior function -carries nothing of its own; an entry no annotated entry reaches carries the annotation itself, -and a function reached only through a registered function value is reached by none. A region +carries nothing of its own except a `[cold_path]` that exempts a rarely-taken branch (a guard +that logs once) from the entry's contracts; an entry no annotated entry reaches carries the +annotation itself, and a function reached only through a registered function value is reached +by none. A region entry is the outermost such function (a kernel `*_encode` / `*_decode`, a step driver, the CPU decoder's `forward_*` entries); a loop reached only from a load, stage, bake, or convert path is not one. A driver that calls the `forward_*` entries and is reached only by a measurement - a @@ -155,10 +157,11 @@ only the functions below it that carry no annotation of their own. **A renamed per-token function is not new: its annotation moves with the name in the same change.** -**A change to code or data of `encode`/`bpe_encode` or anything they reach in -`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das` / `dasllama/dasllama_pretok.das` -ships before/after `--tok` rows (this folder's `benchmarks/lcpp_bench.das`) for a model using -the affected tokenizer.** +**A change to `encode`/`bpe_encode`, or to a function they call, in `dasllama/dasllama_spm.das` / +`dasllama/dasllama_bpe.das` / `dasllama/dasllama_pretok.das`, ships before/after `--tok` rows +(this folder's `benchmarks/lcpp_bench.das`) for a model using the affected tokenizer.** A +change confined to the load path - a metadata default the encode reads as a value - is the +tokenizer-suite rule's below, not this one's. **A tokenizer wall-clock time that grows faster than linearly with input size is a defect** - the `--tok` rows cover at least two input sizes so the growth is readable. @@ -256,9 +259,9 @@ a struct the renderer emits but the registry does not is caught by **Hand-editing `dasllama/dasllama_unicode.das`'s RANGES/WS tables is a defect - regenerate them by retranscoding `$LCPP/src/unicode-data.cpp` (the reference checkout) instead.** -**A diff that adds a file under `dasllama/`, moves code between files, or changes what a file -owns lands the sec.1 edit that keeps the charters true - in an `ARCHITECTURE_*.md` companion, -never `ARCHITECTURE.md` - in the same change.** A diff that adds a file to any +**A diff the placement routing line above routes, or one that changes what a file owns, lands +the sec.1 edit that keeps the charters true - in an `ARCHITECTURE_*.md` companion, never +`ARCHITECTURE.md` - in the same change.** A diff that adds a file to any folder where another file has its own sec.1 charter line lands the new file's charter line too. A module-root doc file - a ledger, a plan - has no charter line and needs no charter edit. diff --git a/modules/dasLLAMA/REVIEW_GPU.md b/modules/dasLLAMA/REVIEW_GPU.md index c95f0c54f5..95d400540d 100644 --- a/modules/dasLLAMA/REVIEW_GPU.md +++ b/modules/dasLLAMA/REVIEW_GPU.md @@ -1,7 +1,7 @@ # dasLLAMA GPU Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: the `ARCHITECTURE_GPU*.md` companions that `ARCHITECTURE.md` indexes. +docs: `ARCHITECTURE_GPU.md`, `ARCHITECTURE_GPU_VULKAN.md`. **Routed from `REVIEW.md`: a diff that checklist routes here applies this list together with it.** @@ -34,7 +34,7 @@ readiness, whether this window's rope tables are staged, is asked by `prefill_de already knows its answer as it picks the pipeline, is a defect - stamp the guard instead.** Stamped means the guard is carried by a `@template_constant` - a `static_if` block, or a value select on the constant. The instance stamped without the guard shows no guard in its generated -`*_msl` global. +`*_msl` global or its SPIR-V dump. **A `[metal_dispatch]` kernel whose main loop steps one fixed-size chunk at a time and never checks for a partial last chunk declares each alignment it assumes on a value the builder @@ -131,14 +131,18 @@ twin, or shifts a shared field to a different binding number, is a defect - even twin ignores that field.** Kernel twins are kernel classes whose bodies differ on one stamp axis - one compile-time choice, such as single/batch, format, or single-pass/chunked. -**A copy-pasted kernel twin, or a kernel split into hand instances where a `static_if` on a -`@template_constant` serves, is a defect - kernel twins stamp one `class template`, whatever -the stamp axis is.** Body divergence is carried by a `@template_constant`, or by an +**A copy-pasted kernel twin - one of two kernel classes whose bodies differ on one compile-time +choice - or a kernel split into hand instances where a `static_if` on a `@template_constant` +serves, is a defect - kernel twins stamp one `class template`, whatever that choice is.** Body divergence is carried by a `@template_constant`, or by an overridden method spliced flat at emission. -**A kernel class template that binds a real buffer to a field the stamp's own body never -reads - a dummy bind that exists only to fill the slot - is a defect: gate that field with -`@template_gate` so the stamps whose bodies do not read it do not carry it.** +**A stamped kernel family - a class template's stamps, or a base shell's `[vk_dispatch]` / +`[metal_dispatch]` leaves - that binds a real buffer to a field a stamp's own body never reads +- a dummy bind that exists only to fill the slot - is a defect: gate the field with +`@template_gate` where a template constant decides it, and where the family shares one set +layout on purpose, name that case in `ARCHITECTURE_GPU.md` (Metal) or +`ARCHITECTURE_GPU_VULKAN.md` (Vulkan).** A field the body reads +under a run-time flag is read, and its unread arm binds a placeholder the kernel never touches. **A diff that forks a kernel class out of a shared template shows that the bodies no longer differ on the compile-time choice the template carried, and names that choice in the @@ -146,24 +150,27 @@ surviving template's comment.** **A `[metal_dispatch]` / `[vk_dispatch]` field whose memory is load-once - a model plane, or an `upload_region` upload never written after arming - is a defect unless it carries -`@role = "weight"`, even when the kernel compiles and passes parity.** +`@role = "weight"`, even when the kernel compiles and passes parity.** A field the kernel reads +under a run-time flag takes the role of its read arm; the placeholder its unread arm binds is +never read, so its lifetime does not decide the role. -**`@role = "weight"` on per-encode data - a pooled buffer the host refills each encode - is a -defect; a per-encode field either omits `@role` or names the access its body performs.** -`weight` tells the generated builder the buffer needs no per-encode hazard tracking. +**`@role = "weight"` on per-encode data the kernel reads - a pooled buffer the host refills +each encode - is a defect; a per-encode field either omits `@role` or names the access its body +performs.** `weight` tells the generated builder the buffer needs no per-encode hazard tracking. **A diff that adds a Metal kernel class under `dasllama/` - a `[metal_kernel]` def, or a new instance of a template carrying one - either adds a census row to `tests/test_kernel_coverage.das` that dispatches it, or names it in that file's -`CENSUS_NEVER_DISPATCHED` with the reason no row can reach it.** The two lists together are the -file's coverage claim. +`CENSUS_NEVER_DISPATCHED` with the reason no row can reach it.** **A diff that adds a Vulkan kernel class under `dasllama/` - a `[vk_dispatch]` declaration, or a -new instance of a template carrying one - adds a census row to `tests/test_kernel_coverage.das` -that dispatches it; a class no census model reaches gets a census model that does.** -`CENSUS_NEVER_DISPATCHED` covers Metal classes only. +new instance of a template carrying one - adds a row to the Vulkan serving census in +`tests/test_kernel_coverage.das` that dispatches it: a census model that reaches the class, or +an arm that forces the device mode the class is gated on; a class no census model reaches gets +a census model that does.** A Vulkan class never joins `CENSUS_NEVER_DISPATCHED`, which takes +Metal classes only. -**Every field of a new kernel class declared in `dasllama/` carries at least one of the +**Every `@ssbo` field of a new kernel class declared in `dasllama/` carries at least one of the annotations its `[metal_dispatch]` / `[vk_dispatch]` builder reads - `@binding`, `@role`, `@off`, `@default`.** A field carrying none of them is dropped from the bind list with no error. @@ -211,11 +218,10 @@ own init/release pair. **A diff that adds or removes a Metal-only or Vulkan-only hook, role, served path, or backend-only capability - a hook in sec.1.5's per-driver registered-hook or borrowed-kernel -lists included, a seat of the `dasllama_gpu_tier` cooperation SPI excluded (the closed list's -standing entry sends those to the tier's role row) - lands its own entry in -`ARCHITECTURE_GPU.md` sec.1.5's closed asymmetry list in the same change, even when that list -already carries an asymmetry of the same class, and even when the diff also extends the file's -sec.1.5 role row.** One backend serving the same path faster or slower is not such a change. +lists included - lands its own entry in `ARCHITECTURE_GPU.md` sec.1.5's closed asymmetry list +in the same change, even when that list already carries an asymmetry of the same class.** One +backend serving the same path faster or slower is not such a change; a seat of the +`dasllama_gpu_tier` cooperation SPI is sec.1.5's tier role row's, not this list's. **A change to code that a served GPU decode or prefill path executes ships GPU-vs-CPU parity on one q8 and one kq (K-quant) model the changed path serves.** That code is anything a @@ -228,8 +234,7 @@ the call routes through; never the bake paths, never a comment. **Parity evidence counts only when it comes from `harness/parity.das`, `benchmarks/lcpp_bench.das --parity` (`performance/model_specs.das`'s fixed model list), or an in-suite parity instrument run through `tests/run.das` that feeds both sides the same fixed -tokens and compares the logits against a fixed tolerance - Metal's -`tests/test_metal_*_parity.das`, Vulkan's `tests/test_gpu_resident_hybrid.das`.** +tokens and compares the logits against a fixed tolerance.** **Parity evidence counts only when its backend was armed: the Metal arm ran with `--ngl`; the Vulkan arm ran with `DASLLAMA_GPU=1` - never `--ngl` - and its log shows the tier that serves diff --git a/modules/dasLLAMA/REVIEW_GPU_RACE.md b/modules/dasLLAMA/REVIEW_GPU_RACE.md index 102c9f5b1a..abf8a8e7ac 100644 --- a/modules/dasLLAMA/REVIEW_GPU_RACE.md +++ b/modules/dasLLAMA/REVIEW_GPU_RACE.md @@ -1,21 +1,23 @@ # dasLLAMA GPU Race Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -doc: `ARCHITECTURE_GPU.md`. Planned work: `followup_metal.md`. +docs: `ARCHITECTURE_GPU.md`, `ARCHITECTURE_GPU_VULKAN_GEMM.md`, `ARCHITECTURE_MEASUREMENT.md`. +Planned work: `followup_metal.md` for Metal, `followup_vulkan.md` for Vulkan. **Routed here by another checklist: a diff that checklist routes here applies this list together with it.** **A hand-binding arm - a race or knockout timing arm (a race times two implementations of one computation on one queue and compares their outputs; a knockout attributes cost across stages -instead of selecting between implementations) that lists its bindings by number instead of -dispatching through the `enc_*` builder - that binds a field at a number the class does not -declare for that field is a defect.** A mis-numbered arm dispatches, reads -the wrong buffer, and its timing selects the wrong kernel silently. +instead of selecting between implementations) that mirrors a class's binding order by hand - a +literal bind number, or a positional buffer and size array - instead of naming the class's +fields - that binds a field at a position the class does not declare for that field is a +defect.** A mis-bound arm dispatches, reads the wrong buffer, and its timing selects the wrong +kernel silently. **A hand-binding arm outside `dasllama/`, or one whose pipeline source or threadgroup-memory size arrives as a function parameter rather than a literal global, states in the PR that its -bind numbers and kargs word layout were verified by hand against the class declaration.** +binding order and push-constant layout were verified by hand against the class declaration.** The `REVIEW.das` gate `check_race_bind_numbers` cannot read those arms, and an arm nothing checked is where a mis-numbered bind reaches the board. @@ -37,46 +39,51 @@ file. small enough to sit in cache ranks the kernels by an effect production never sees, and the race then picks the slower kernel. -**A timing arm whose ranking is recorded as a decision - a runtime crown (the winner the box -profile records and the served graph dispatches), a tune-sidecar row (the winner a kernel's -`*.tune.json` sidecar records), or an `ARCHITECTURE_GPU.md` sec.2.2b entry - times its kernel at -every width that kernel is dispatched at: every verify width, every row count from 2 to -`MTP_MAX_ROWS - 1` (`dasllama/dasllama_common.das`), when the batched decode driver -(`dasllama/dasllama__decode.das`) dispatches it per row count; the tile's width when it is -dispatched at one fixed width per tile; each power of two it spans when the instrument runs on a -power-of-two batch grid.** A ranking timed at one width alone is applied at widths it was never -ranked at. +**A timing arm whose ranking a checked-in document, box profile or sidecar records as decided - +a runtime crown (the winner the box profile records and the served graph dispatches), a +tune-sidecar row (the winner a kernel's `*.tune.json` sidecar records), an architecture-doc +entry - times its kernel at every width that kernel is dispatched at: every verify width, every +row count from 2 to `MTP_MAX_ROWS - 1` (`dasllama/dasllama_common.das`), when the batched decode +driver (`dasllama/dasllama__decode.das`) dispatches it per row count; the tile's width when +it is dispatched at one fixed width per tile; each power of two it spans when the instrument +runs on a power-of-two batch grid; the served window row counts when it is a prefill tile over a +variable window.** A ranking timed at one width alone is applied at widths it was never ranked +at. **An `ARCHITECTURE_GPU.md` sec.2.2b entry for a kernel ranked on a power-of-two batch grid names that grid.** -**A kernel A/B race arm that mints a runtime crown or a tune-sidecar row binds a DIFFERENT -output buffer for consecutive dispatches of its chain, never one shared output.** One shared -output serializes the chain on its write-after-read hazard while the served graph overlaps -consecutive dispatches, so the race ranks the arms on a shape production never runs. +**A kernel A/B race arm whose ranking a checked-in document, box profile or sidecar records as +decided binds a DIFFERENT output buffer for consecutive dispatches of its chain, never one +shared output, and every arm of the race handles the hazard between its dispatches the same +way.** One shared output serializes the chain on its write-after-read hazard while the served +graph overlaps consecutive dispatches, so the race ranks the arms on a shape production never +runs; an arm serialized by a barrier races an arm that overlaps. **Weakening the burn phase of `race_pair_ms` (`dasllama/dasllama_metal_common.das`) - the GPU work it runs on both arms before the first timed round - is a defect.** -**A race arm that mints a runtime crown or a tune-sidecar row and does not go through -`race_pair_ms` burns GPU work on its own arms before its first timed round.** The arms -alternate, so a first round on a cold GPU clock charges the clock's ramp-up to whichever arm -ran first. +**A race arm whose ranking a checked-in document, box profile or sidecar records as decided +and that does not go through `race_pair_ms` burns GPU work on its own arms before its first +timed round.** The arms alternate, so a first round on a cold GPU clock charges the clock's +ramp-up to whichever arm ran first. -**A timed encoder in a race arm that mints a runtime crown or a tune-sidecar row issues its -dispatches back to back.** An encoder that leaves gaps between its dispatches times an idle -clock. +**A timed encoder in a race arm whose ranking a checked-in document, box profile or sidecar +records as decided issues its dispatches back to back.** An encoder that leaves gaps between +its dispatches times an idle clock. **A diff that ports an A/B lab's winning variant into a kernel deletes, in the same change, that variant's class and any `*_variants.das` code that exists only for it; an arm that -survives the port dispatches the shipped kernel class's generated source, or is ledgered as -a retained reference in `ARCHITECTURE_GPU.md` sec.2.2b.** An A/B lab is a timing script -whose output selects between implementations of the same compute; a decided arm that -outlives its decision degrades into an unmaintained duplicate of the kernel it seeded. +survives the port dispatches the shipped kernel class's generated source, or is ledgered as a +retained reference in the architecture doc that owns the kernel's tier (`ARCHITECTURE_GPU.md` +sec.2.2b for Metal, `ARCHITECTURE_MEASUREMENT.md`'s probe axes for Vulkan).** An A/B lab is a +timing script whose output selects between implementations of the same compute; a decided arm +that outlives its decision degrades into an unmaintained duplicate of the kernel it seeded. **A diff that leaves an A/B lab with no undecided arm deletes the lab's driver and its -remaining arm in the same change.** +remaining arm in the same change, unless the architecture doc that owns the kernel's tier +ledgers the lab as a retained instrument.** **A diff that moves a shipped kernel's staging shape or its grid or threadgroup geometry -resyncs or deletes, in the same change, every arm `ARCHITECTURE_GPU.md` sec.2.2b ledgers as a +resyncs or deletes, in the same change, every arm the tier's architecture doc ledgers as a retained reference that reimplements that kernel.** diff --git a/modules/dasLLAMA/REVIEW_GPU_VULKAN.md b/modules/dasLLAMA/REVIEW_GPU_VULKAN.md index 7c998697e8..eb49a6f61a 100644 --- a/modules/dasLLAMA/REVIEW_GPU_VULKAN.md +++ b/modules/dasLLAMA/REVIEW_GPU_VULKAN.md @@ -1,8 +1,9 @@ # dasLLAMA Vulkan Tier Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: `ARCHITECTURE_GPU_VULKAN.md`, `ARCHITECTURE_GPU_VULKAN_RESIDENCY.md` and -`ARCHITECTURE_GPU_VULKAN_DECODE.md`. Planned work: `followup_vulkan.md`. +docs: `ARCHITECTURE_GPU_VULKAN.md`, `ARCHITECTURE_GPU_VULKAN_GEMM.md`, +`ARCHITECTURE_GPU_VULKAN_RESIDENCY.md` and `ARCHITECTURE_GPU_VULKAN_DECODE.md`. Planned work: +`followup_vulkan.md`. **Routed from `REVIEW_GPU.md`: a diff that checklist routes here applies this list together with `REVIEW_GPU.md`'s and `REVIEW.md`'s.** @@ -10,8 +11,9 @@ with `REVIEW_GPU.md`'s and `REVIEW.md`'s.** **A hand-written Vulkan pipeline build is a defect - a Vulkan pipeline is created only by a `[vk_dispatch]`-generated `ensure_*`.** -**A diff that adds a Vulkan pipeline tears it down in `vk_drop_model_state`, in the same -change.** +**A diff that adds a Vulkan dispatch family adds its model-owned device buffers, descriptor-set +caches and `*_ready` latch to `vk_drop_model_state`'s sweep, in the same change.** Pipelines +are device-lifetime state that survives the drop and rebuilds lazily. **Never size a buffer bound as one SSBO (shader storage buffer) range above `vk_max_storage_range()` - check the size at the site that computes it, not at the site that @@ -55,16 +57,31 @@ template cover both codecs, or a single-codec kernel has a sibling that serves t behind an arming gate that keys on `kv16`. The whole-model driver serves both codecs, so a codec no kernel covers silently drops that codec's GPU path. -**A diff that adds a cm2 tile instance for a kq superblock format - one `kq_sb` -(`dasllama/dasllama_kqformat.das`) accepts - changes such a format's cm2 decode body, or -changes its four-wide twin (`decode_v4`) or its `DECV4` or `DECVEC` constant puts that -format's `cm2:` probe rows (`harness/vk_gemm_probe.das`), both the -`DASLLAMA_VK_DECVEC=1` and the `=0` rows, in the PR body.** A cm2 tile is the +**A diff that changes what a kq superblock format's cm2 tile emits - its instance set, its +decode body or four-wide twin (`decode_v4`), its `DECV4` or `DECVEC` constant, or the shared +`KqCm2BatchT` body - puts that format's `cm2:` probe rows (`harness/vk_gemm_probe.das`), +both the `DASLLAMA_VK_DECVEC=1` and the `=0` rows, in the PR body, or the claim that the +format's emitted kernels are byte-identical to master's.** A cm2 tile is the NV_cooperative_matrix2 GEMM class stamped per weight format and token-column width (the class's `BN`) in `dasllama/dasllama_vulkan_classes.das`. -**A cm2 tile format instance whose `DASLLAMA_VK_DECVEC=1` probe row is slower than its `=0` row -ships one of two fixes in the same change: a hand-written `decode_v4` under +**A `kq_sb` format (`dasllama/dasllama_kqformat.das`) that joins the cm2 template - a +`Cm2T` format template in +`dasllama/dasllama_vulkan_classes.das` - ships its KHR instantiation (`KhrBatch`, the +`kq_batch__khr_cls` dispatch) and its arm in each of `khr_cls_ensure`, `khr_cls_set` and +`khr_cls_enc` (`dasllama/dasllama_vulkan_prefill.das`) in the same change, and that format's +kernel cell in `tests/test_vulkan_kernels.das` runs its KHR arm.** `pf_f16_feed` admits every +`kq_sb` format in mm mode, so a format with no KHR class reaches the ladders' fall-through +`verify` - a panic on a KHR-only card that no cm2 box reproduces without `DASLLAMA_COOPMAT=mm`. + +**A kernel body that calls a `[spirv_decode]` method directly passes the plane element itself +(`decode(wq[i], ...)`), never a local copy of it (`let blk = wq[i]` then `decode(blk, ...)`).** +Both compile: on the element the emitter passes the index and the callee chains through the +plane; on a copy it loads and spills the whole block per call, and the KHR kq tile measured +well under half its rate that way (`ARCHITECTURE_GPU_VULKAN_GEMM.md` sec.2.2l). + +**A diff that puts a format's `cm2:` probe rows in the PR body whose `DASLLAMA_VK_DECVEC=1` +row is slower than its `=0` row ships one of two fixes in the same change: a hand-written `decode_v4` under `override DECV4 = true` on that format's class (`dasllama/dasllama_vulkan_classes.das`), re-measured so its `=1` row now beats its `=0` row; or `override DECV4 = false` and `override DECVEC = false` together, which puts the format back on the scalar callback.** With @@ -79,8 +96,21 @@ moved: attention's `rdq_role_names` with `g_rdq_role`, recurrent's `RDQ_DN_NAMES `rdq_sample` indexes a fixed count per layer, so one extra or missing timestamp reports every later stamp under the wrong role name. +**A decode GEMV class - a `KqGemvBase` leaf in `dasllama/dasllama_vulkan_classes.das` - that +stages a codebook into `@workgroup` memory reads it from the family's grid buffer (`gridb`, +binding 6, filled by `kq_grid_dev` at the format's `KQ_GRID_` offset), never from a +`*_grid_word` accessor.** The accessor is a constant composite the driver reads lane-serially +per index, and a two-row workgroup pays that read on every row pair it walks +(`ARCHITECTURE_GPU_VULKAN.md` sec.2.2ab). + **A diff that changes how many GPU timestamps the resident prefill's window command records - a `pfq_ts` call in `pf_run` or in any function `pf_run` reaches, all in `dasllama/dasllama_vulkan_prefill.das` - updates `pf_roles_per_layer` and that file's `pf_prof_report` in the same change.** Both index a fixed count per layer, so one extra or missing timestamp reports every later stamp under the wrong role name. + +**A diff that changes `AR_MAX_DIM` (`dasllama/dasllama_vulkan_common.das`) changes the `row` +`@workgroup` slab of `ArBase` (`dasllama/dasllama_vulkan_classes.das`) and the `c.dim` cap of +the Vulkan servability gate (`attn_dec_shape_ok`, `dasllama/dasllama_blocks.das`) to the same +number, in the same change.** The add+rms kernels stage a whole row in that slab, so a slab +shorter than the cap writes past its end. diff --git a/modules/dasLLAMA/REVIEW_IMAGE.md b/modules/dasLLAMA/REVIEW_IMAGE.md index 06158d13b5..5d7f661e38 100644 --- a/modules/dasLLAMA/REVIEW_IMAGE.md +++ b/modules/dasLLAMA/REVIEW_IMAGE.md @@ -1,7 +1,7 @@ # dasLLAMA image rail Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -doc: `ARCHITECTURE_IMAGE.md`. +doc: `ARCHITECTURE_IMAGE.md`. Planned work: `followup_general.md`. **Routed from `REVIEW.md`: a diff that checklist routes here applies this list together with `REVIEW.md`.** @@ -16,13 +16,19 @@ image rail does not carry is a ledgered lane in `ARCHITECTURE_IMAGE.md`, never a **A weight carrier becomes live only through `build_image` and `parse_image` in `dasllama/dasllama_image.das`: reading weights into a live carrier, or releasing an image -backing, anywhere else is a defect - and a second mint path, per family, per format, or per -backend, is a defect even where its output is identical.** +backing, anywhere else is a defect.** -**A decoder mint never holds the whole model.** A decoder mint is the mint of an LLM decoder -model, not of a tower or embedder carrier. It sizes the image before writing the first byte and -writes each plane as it is produced. A mint that is slower in exchange for a lower peak is -correct. +**A second mint path - per family, per format, or per backend - is a defect even where its +output is identical.** + +**A decoder mint never materializes the whole image in memory: it sizes the image before +writing the first byte and writes each plane as it is produced.** A decoder mint is the mint of +an LLM decoder model, not of a tower or embedder carrier. A mint that is slower in exchange for +a lower peak is correct. + +**A decoder mint never holds two copies of the carrier: the planar model it streamed from is +released before the written image is mapped, and a declined save serves the streamed build +from memory instead of reloading.** **A staged carrier mint (`cache_via_image_staged`) meeting a source file at or past 1 GiB either refuses it or streams it the way a decoder mint does.** A refusal names that file. The diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 8607d84980..c034331d30 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -3,12 +3,9 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE_MEASUREMENT.md`. Planned work: `PERF_LEDGER.md`. -Figure rules here bind every surface the diff under review writes a figure on - this module's -code comments and checked-in docs, checked-in data whose contents a run produced (a captured -response fixture, a record), the PR body, and any file another checklist routed to this one. A -number on a page under `site/` or `site-dasllama/` (repo root) answers to that folder's -checklist instead, and a number on a page a tool under `utils/` (repo root) ships answers to -that tool's folder checklist. +Figure rules here bind every surface the diff under review writes a figure on, wherever it +sits - code comments, checked-in docs and plans, checked-in data a run produced, the PR body - +except a page a `site*/` or `utils/` folder checklist owns (repo root), which answers there. **A `PERF_LEDGER.md` entry never states a tok/s figure or a turn wall that the `-jit` script produced - a `-jit` A/B pair enters as its ratio, with the arms' absolute rates left in the @@ -33,16 +30,17 @@ board row. two processes or two commits, and `out-of-process` when the wall was measured from outside the benchmark process.** -**A `--for-debug-purposes` reading never overturns a committed board row - the board answers -to a re-minted board cell.** A `--for-debug-purposes` row is the `-jit` script's own output. +**A checked-in document that states a `--for-debug-purposes` reading for a model and arm a +committed board row covers cites that row and marks the reading debug-jit; the board itself +changes only through a re-minted board cell.** A `--for-debug-purposes` row is the `-jit` +script's own output. **A `PERF_LEDGER.md` entry carrying a figure from another project names the source and the report it came from and tags it `external`.** -**A diff never rests an adoption decision on a figure from another project - the decision -rests on a self-measured board cell.** A board cell is one `performance/gen_bench_records.das` -spawns, or a manual cell of this folder's `benchmarks/lcpp_bench.das` that `PROFILE.md` -documents in a section. +**A diff never rests an adoption decision about what the engine serves on a figure from +another project - that decision rests on a self-measured board cell.** A selection between +kernel forms settles on its own instrument's run (the next rule). **A diff that adds an entry to `PERF_LEDGER.md` never records a selection timing - a timing that picks a winner between candidate kernel forms.** That timing settles its adoption @@ -61,32 +59,35 @@ exe's stand-in is `tune_gate`'s fat-world report (`performance/profile_common.da **A diff that claims to make an already-served path faster, from an author whose box mints that path, re-mints a board row (`performance/records/.json`) that exercises that path, in the same change, and names that row in the PR body.** A box mints a path when -`performance/gen_bench_records.das` mints a row for it on that box rather than refusing or -skipping it. Where no row exercises the path, the diff mints one. The board is the module's -public memory of what serving costs; a kernel win that never lands there is invisible to the -next regression check. +`performance/gen_bench_records.das` mints a row for it on that box - a leg its `stored_row_leg` +(`performance/profile_common.das`) admits - rather than refusing or skipping it; a backend +route the record rig has no leg for takes the stand-in clause instead. Where no row exercises +the path, the diff mints one. The board is the module's public memory of what serving costs; a +kernel win that never lands there is invisible to the next regression check. **A timing figure of a served turn as a whole - tok/s, latency, a whole-turn model or engine comparison, the 512-token prefill (pp512) and 128-token decode (tg128) rates - that this module writes down as a measurement is a defect without either a board cell behind it or a provenance line, covering its passage or its section, naming harness, flags, box, and the exe or script -that ran it.** The board cell states its quant mode and stamps box and engine provenance, so a -number can never silently describe a format nobody serves or a kernel set nobody ships. A -figure labeled as a prediction is not a reading, and this rule does not reach it. +that ran it - or a citation of the passage whose provenance line covers it.** The board cell +states its quant mode and stamps box and engine provenance, so a number can never silently +describe a format nobody serves or a kernel set nobody ships. A figure labeled as a prediction +is not a reading, and this rule does not reach it. **A number this module writes down that is not a whole served-turn reading - any figure a run or a build produced, timing or not - names the harness, the flags and the box that produced it.** A figure a committed board cell or ruler record produced names the record and row instead. The naming sits in the figure's own sentence, in a table heading that covers the table's rows, in a section-level provenance line that covers the paragraphs under it, or in a citation of the -`PERF_LEDGER.md` entry whose provenance line covers it. +passage whose provenance line covers it. **A diff that adds a race to a shipped exe's startup races on synthetic inputs only - never -loading a model, never spawning a child process - and never races a `[tune]` kernel family, a -race whose winner needs a recompiled clone; a GPU pso twin race, which only sets a runtime -knob, is what a first start may do** (`ARCHITECTURE_MEASUREMENT.md` sec.2.42a). A shipped exe -carries no vehicle model and no harness script, so a model or a child there is a hang or a -silent skip, and a `[tune]` winner there would name a clone the baked exe does not carry. +loading a model, never spawning a child process.** A shipped exe carries no vehicle model and +no harness script, so a model or a child there is a hang or a silent skip. + +**A shipped exe's startup race never races a `[tune]` kernel family; a GPU pso twin race, which +only sets a runtime knob, is what a first start may do** (`ARCHITECTURE_MEASUREMENT.md` +sec.2.42a). A `[tune]` winner needs a recompiled clone the baked exe does not carry. **A diff never adds a confirm - an end-to-end A/B served on a vehicle model in a spawned child - outside `harness/`.** diff --git a/modules/dasLLAMA/REVIEW_PLACEMENT.md b/modules/dasLLAMA/REVIEW_PLACEMENT.md index ce605a87b7..f81198f50b 100644 --- a/modules/dasLLAMA/REVIEW_PLACEMENT.md +++ b/modules/dasLLAMA/REVIEW_PLACEMENT.md @@ -11,11 +11,14 @@ it.** charters own the per-file list; a rule naming what KIND of code lands in which file is the checklist's own. +**A function whose KIND the file's sec.1 charter line seats in another file lands in that +file, or the charter line changes in the same diff.** + **A tensor format conversion lands in `dasllama/dasllama_convert.das`.** -**A disk-order -> compute-order transform lands per scope: a transform a kernel's layout needs -in `dasllama/dasllama_repack.das`, a transform run while the model loads in -`dasllama/dasllama_layout.das`.** +**A disk-order -> compute-order transform lands by its consumer: a transform into the layout +a CPU row core reads in `dasllama/dasllama_repack.das`, a transform into the layout a GPU plane +or gather reads in `dasllama/dasllama_layout.das`.** **A CPU KV-cache store, read, score dot, or V-accumulate OVER CACHE BYTES - a codec primitive that knows the K/V element format - lands in `dasllama/dasllama_kv_codec.das`, its format @@ -27,19 +30,19 @@ backend file (`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das`).** **A kernel body - the arithmetic loop itself, the one a `[tune]` family or a dispatch class (a class a `[metal_dispatch]` or `[vk_dispatch]` declares) picks one variant of - lands in its -owner's backend file.** A GPU kernel body lands in the file where its pipeline state object -(PSO) is compiled and released. A CPU-tier kernel body lands in that tier's -`dasllama/dasllama_math_.das`. A kernel body never lands in `dasllama/dasllama_math.das` -or in a file whose job is declaring kernels and routing dispatch. A class stamped from a -template declared elsewhere is not a kernel body: it compiles and releases its own PSO where it -is stamped. +owner's backend file.** A GPU kernel body lands in its backend's kernel home - +`dasllama/dasllama_metal_kernels.das`, `dasllama/dasllama_vulkan_classes.das` - never in a +driver, seat or math file. A CPU-tier kernel body lands in that tier's +`dasllama/dasllama_math_.das`, never in `dasllama/dasllama_math.das`. A class stamped from +a template declared elsewhere is not a kernel body: it compiles its own PSO where it is stamped. **A quirk of one family - one model architecture's file, or one backend driver's - lands in that file, never sideways into a sibling.** **A piece two files need lands in their nearest shared file (its own file when none exists) - -never a second copy.** A predicate, a constant, or a helper spelled twice drifts on the first -edit to one copy. A piece two folders outside each other both need lands in the folder that +never a second copy.** A predicate, a constant, or a helper spelled once in each of two files +drifts on the first edit to one copy; an enum-and-int twin of one predicate inside one file is +the tier's idiom, not a copy. A piece two folders outside each other both need lands in the folder that owns the concern; one landing under `dasllama/` that code outside `modules/dasLLAMA/` drives lands as a public entry module - one `dasllama/dasllama_lint.das` licenses a consumer to require directly. diff --git a/modules/dasLLAMA/REVIEW_UPSTREAM.md b/modules/dasLLAMA/REVIEW_UPSTREAM.md index 312fc92235..ce10c4994a 100644 --- a/modules/dasLLAMA/REVIEW_UPSTREAM.md +++ b/modules/dasLLAMA/REVIEW_UPSTREAM.md @@ -1,24 +1,27 @@ -# dasLLAMA Code Review Checklist - the reference build in checked-in text +# dasLLAMA Upstream-Naming Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE_MEASUREMENT.md`. **Routed from `REVIEW.md`: a diff that checklist routes here applies this list together with it.** The reference build is the third-party engine this module measures itself against - the -checkout `benchmarks/setup_lcpp_ref.das` pins. +checkout `benchmarks/setup_lcpp_ref.das` pins. The text this list binds is checked-in text +under `modules/dasLLAMA/`; a repo-root `plans/` document a dasLLAMA change writes reaches it +through `plans/REVIEW.md`. -**Checked-in text under `modules/dasLLAMA/` - docs, comments, and string data, any language - -that is not locating, patching, or reproducing work against the reference build describes an -upstream mechanism in our own terms: no "lifted/ported verbatim from", and no name belonging -to the reference build - symbol, header, constant, binary, project or organization - write -"the reference exe" or "upstream" instead.** A symbol the file carrying that text calls or -holds as a value is its own name, not attribution. +**Text whose job is not work performed ON the reference build or measured AGAINST it - +locating a site in it, patching it, running it, regenerating from it - describes an upstream +mechanism in our own terms: no "lifted/ported verbatim from", and no name belonging to the +reference build - symbol, header, constant, binary, project or organization - write "the +reference exe" or "upstream" instead.** Mirroring the reference build's behavior in our own +code is not that job. A symbol this module's code calls or holds as a value is its own name, +not attribution. -**Prose whose job is to locate, patch, or reproduce work against the reference build names -that build's binaries and symbols outright, and keeps that naming inside the sentences doing -that job.** The job decides, not the artifact kind - a regeneration path, an env-knob row, a -command line in a methodology or how-to document, a ledger row whose subject is a reading of -the reference build (the compared row, the command that reproduces it), and a source patch -applied TO the reference build all qualify. A paragraph that mixes a reading of the reference -build with a proposal of our own keeps them in separate sentences; a row that cites upstream -while proposing our own work is a proposal, not a reading, so it names no upstream symbol. +**Text whose job is work performed on the reference build or measured against it names that +build's names - symbol, header, constant, binary, project or organization - outright, and keeps +the naming inside the sentences doing that job.** The job decides, not the artifact kind; a +knob or command needed to reproduce a compared reading stays licensed inside a planned-work row. + +**A paragraph that mixes a reading of the reference build with a proposal of our own keeps +them in separate sentences.** The proposal sentence names no upstream name; the reading +sentences beside it name what they read, kernel and file included. diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index ca3d0dee21..444e6fc125 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -20,31 +20,28 @@ not one the engine selected. Without the gate or the stamp the instrument measur kernels silently. **A diff that adds or changes a race alternates its arms within one process - one timed round -per arm, best-of across rounds.** A race is an instrument that runs both implementations -itself, in its own process, and compares them; an A/B arm is one run of an instrument under a -named lever, compared against a paired run of the same instrument. One instrument is one or the -other, never both. - -**A diff that adds or changes a race arm proves the arm's output on its report line, by what the -arm computes:** an arm producing no comparable -output carries the literal token `timing-only`; an arm whose result is bit-identical to the -baseline's prints the bit-exact compare over the sampled region - the set of output elements -the run compares - on the report's "bit-exact vs ..." line; every other arm prints a -bounded-difference compare (against the baseline arm or the CPU reference) plus the bound it -passed. How the arm orders its sums, and whether its multiply-adds fuse, decide bit-identity - -not the declared precision. +per arm, best-of across rounds.** A race arm times two implementations of the same computation +in one process and compares them; an A/B arm is one run of an instrument under a named lever, +compared against a paired run of the same instrument. An instrument is reviewed arm by arm. + +**A diff that adds or changes a race arm that computes a comparable output proves that output +on its report line:** an arm whose result is bit-identical to the baseline's prints the +bit-exact compare over the sampled region - the set of output elements the run compares - on +the report's "bit-exact vs ..." line; every other arm prints a bounded-difference compare +(against the baseline arm or the CPU reference) plus the bound it passed. How the arm orders +its sums, and whether its multiply-adds fuse, decide bit-identity - not the declared precision. **A diff that adds or changes a race also checks the race's baseline arm against a CPU reference.** The baseline arm is the arm running the implementation already in use. The reference check runs in the same process, on the same output elements the arms are judged on. Two arms can agree and both be wrong; only the reference makes the winner right. -**A knockout or sweep arm - one that attributes cost across stages instead of selecting -between two implementations - carries the literal token `timing-only` on its report line.** +**An arm that compares no output - a knockout, a sweep across formats or stages, an arm whose +output is not comparable - carries the literal token `timing-only` on its report line.** -**An instrument all of whose arms attribute carries the literal text `ATTRIBUTION SWEEP` in its -file header comment, on a line that also names what its arms attribute.** Without the text a -reader takes the sweep's arms for an adoption decision it never made. +**An instrument with no race arm carries the literal text `ATTRIBUTION SWEEP` in its file +header comment, on a line that also names what its arms attribute.** Without the text a reader +takes the sweep's arms for an adoption decision it never made. **A new instrument that puts its own clock around a served turn is a defect: add a board cell to `../performance/gen_bench_records.das`, or a `lcpp_bench.das` cell with its own diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 06bf167997..93f97aa73d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -522,11 +522,11 @@ def private attn_dec_hydrate_codec(var s : Session; l, pos, woq, kvd, seq_len : [arch(at="../ARCHITECTURE_GPU_VULKAN_DECODE.md#decode-attention-block")] def private attn_dec_shape_ok(t : Model; s : Session; l : int64) : bool { let c = t.config - if (!moe_gpu_attn_on_gpu(l) || t.kv_src[l] != l || s.kv_pool != null || c.attn_qkv_bias || c.attn_out_bias + if (!moe_gpu_attn_on_gpu(l) || t.kv_src[l] != l || s.kv_pool != null || c.attn_out_bias || c.v_norm || c.attn_sinks || c.attn_logit_softcap != 0.0 || c.q_gated || layer_is_sliding(c, l) || s.kv_dtype_k == KVDtype.tq4 || s.kv_dtype_v == KVDtype.tq4 || (c.rope_dim > 0l && c.rope_dim != layer_head_size(c, l)) - || c.dim > 4096l || layer_head_size(c, l) > 256l // the chain's add-rms stages a row of at most AR_MAX_DIM, the attention a head of at most 256 + || c.dim > 8192l || layer_head_size(c, l) > 256l // the chain's add-rms stages a row of at most AR_MAX_DIM (8192), the attention a head of at most 256 || layer_kv_dim(c, l) != layer_kv_dim(c, 0l) || layer_head_size(c, l) != layer_head_size(c, 0l)) { // the block holds ONE geometry return false } @@ -598,14 +598,17 @@ def private attention_gpu_decode(t : Model; var s : Session; l : int64; pos : in unsafe { let rq = c.qk_norm ? addr(t.fblob[t.rms_q_offs[l]]) : null let rk = c.qk_norm ? addr(t.fblob[t.rms_k_offs[l]]) : null + let bq = c.attn_qkv_bias ? addr(t.bq[l * qd]) : null + let bk = c.attn_qkv_bias ? addr(t.bk[l * kv_dim]) : null + let bv = c.attn_qkv_bias ? addr(t.bv[l * kv_dim]) : null if (q8) { matmul_moe_gpu_attn_dec(s.xb2, s.k, s.v, t.wq_offs[l], t.wk_offs[l], t.wv_offs[l], t.wo_offs[l], int(fq), int(fmt_at(t.wk_fmt, l)), int(fmt_at(t.wv_fmt, l)), int(fmt_at(t.wo_fmt, l)), - s.xq, s.xs, g_dat_cossin, rq, rk, pos, dim, qd, kv_dim, head_size, n_heads, kv_mul, scale, c.norm_eps, c.rope_neox, c.qk_norm) + s.xq, s.xs, g_dat_cossin, rq, rk, bq, bk, bv, pos, dim, qd, kv_dim, head_size, n_heads, kv_mul, scale, c.norm_eps, c.rope_neox, c.qk_norm) } else { matmul_moe_gpu_attn_dec(s.xb2, s.k, s.v, t.wq_offs[l], t.wk_offs[l], t.wv_offs[l], t.wo_offs[l], int(fq), int(fmt_at(t.wk_fmt, l)), int(fmt_at(t.wv_fmt, l)), int(fmt_at(t.wo_fmt, l)), - s.kxq, s.kxs, g_dat_cossin, rq, rk, pos, dim, qd, kv_dim, head_size, n_heads, kv_mul, scale, c.norm_eps, c.rope_neox, c.qk_norm) + s.kxq, s.kxs, g_dat_cossin, rq, rk, bq, bk, bv, pos, dim, qd, kv_dim, head_size, n_heads, kv_mul, scale, c.norm_eps, c.rope_neox, c.qk_norm) } } prof_add("attn_gpu", ts_gpu) @@ -630,6 +633,10 @@ def private attn_dec_store_kv(t : Model; var s : Session; l, pos : int64) { let kv_dim = layer_kv_dim(c, l) let head_size = layer_head_size(c, l) let ts_kv = prof_ticks() + if (c.attn_qkv_bias) { // the block returns the projection rows before the bias; the cache row carries it, as on the CPU path + add_bias(s.k, 0l, t.bk, l * kv_dim, kv_dim) + add_bias(s.v, 0l, t.bv, l * kv_dim, kv_dim) + } if (c.qk_norm) { rms_batch(s.k, s.k, t, t.rms_k_offs[l], head_size, layer_n_kv_heads(c, l)) } @@ -704,6 +711,12 @@ def private span_fill_layers(t : Model; var a : MoeSpanDec; l0, nl : int64) { a.f2 |> resize(nl) a.rmsq |> resize(nl) a.rmsk |> resize(nl) + a.bq |> reserve(nl) + a.bq |> resize(nl) + a.bk |> reserve(nl) + a.bk |> resize(nl) + a.bv |> reserve(nl) + a.bv |> resize(nl) unsafe { for (li in range64(nl)) { let l = l0 + li @@ -723,6 +736,9 @@ def private span_fill_layers(t : Model; var a : MoeSpanDec; l0, nl : int64) { a.f2[li] = int(fmt_at(t.we2_fmt, l)) a.rmsq[li] = c.qk_norm ? addr(t.fblob[t.rms_q_offs[l]]) : null a.rmsk[li] = c.qk_norm ? addr(t.fblob[t.rms_k_offs[l]]) : null + a.bq[li] = c.attn_qkv_bias ? addr(t.bq[l * a.qd]) : null + a.bk[li] = c.attn_qkv_bias ? addr(t.bk[l * a.kvd]) : null + a.bv[li] = c.attn_qkv_bias ? addr(t.bv[l * a.kvd]) : null } a.router = addr(t.fblob[t.router_off + l0 * c.n_expert * dim]) a.att_norm = addr(t.fblob[t.rms_att_off + l0 * dim]) @@ -1038,9 +1054,7 @@ def attention_gpu_prefill(t : Model; var s : Session; l : int64; npos : int64; g let kv_dim = layer_kv_dim(c, l) let head_size = layer_head_size(c, l) let n_heads = c.n_heads - let n_kv_heads = layer_n_kv_heads(c, l) let qd = n_heads * head_size - let kv_mul = n_heads / n_kv_heads let scale = c.attn_scale > 0.0 ? c.attn_scale : 1.0 / sqrt(float(head_size)) let rot = c.rope_dim > 0l ? c.rope_dim : head_size let fq = fmt_at(t.wq_fmt, l) @@ -1061,25 +1075,26 @@ def attention_gpu_prefill(t : Model; var s : Session; l : int64; npos : int64; g prof_add("at_norm", ts_norm0) let ts_gpu = prof_ticks() unsafe { + let rq = c.qk_norm ? addr(t.fblob[t.rms_q_offs[l]]) : null + let rk = c.qk_norm ? addr(t.fblob[t.rms_k_offs[l]]) : null + let bq = c.attn_qkv_bias ? addr(t.bq[l * qd]) : null + let bk = c.attn_qkv_bias ? addr(t.bk[l * kv_dim]) : null + let bv = c.attn_qkv_bias ? addr(t.bv[l * kv_dim]) : null if (kq_cls) { matmul_moe_gpu_attn(s.xb2_b, s.k_b, s.v_b, t.wq_offs[l], t.wk_offs[l], t.wv_offs[l], t.wo_offs[l], int(fq), int(fmt_at(t.wk_fmt, l)), int(fmt_at(t.wv_fmt, l)), int(fmt_at(t.wo_fmt, l)), - s.kxqb, s.kxsb, - c.qk_norm ? addr(t.fblob[t.rms_q_offs[l]]) : null, - c.qk_norm ? addr(t.fblob[t.rms_k_offs[l]]) : null, + s.kxqb, s.kxsb, rq, rk, bq, bk, bv, addr(s.rope_cos[0]), addr(s.rope_sin[0]), - npos, dim, qd, kv_dim, head_size, rot, n_heads, kv_mul, + npos, dim, qd, kv_dim, head_size, rot, n_heads, scale, c.norm_eps, gated, c.qk_norm) } else { matmul_moe_gpu_attn(s.xb2_b, s.k_b, s.v_b, t.wq_offs[l], t.wk_offs[l], t.wv_offs[l], t.wo_offs[l], int(fq), int(fmt_at(t.wk_fmt, l)), int(fmt_at(t.wv_fmt, l)), int(fmt_at(t.wo_fmt, l)), - s.xqb, s.xsb, - c.qk_norm ? addr(t.fblob[t.rms_q_offs[l]]) : null, - c.qk_norm ? addr(t.fblob[t.rms_k_offs[l]]) : null, + s.xqb, s.xsb, rq, rk, bq, bk, bv, addr(s.rope_cos[0]), addr(s.rope_sin[0]), - npos, dim, qd, kv_dim, head_size, rot, n_heads, kv_mul, + npos, dim, qd, kv_dim, head_size, rot, n_heads, scale, c.norm_eps, gated, c.qk_norm) } } diff --git a/modules/dasLLAMA/dasllama/dasllama_bpe.das b/modules/dasLLAMA/dasllama/dasllama_bpe.das index 701bc87f7f..233c03a479 100644 --- a/modules/dasLLAMA/dasllama/dasllama_bpe.das +++ b/modules/dasLLAMA/dasllama/dasllama_bpe.das @@ -32,11 +32,18 @@ struct BpeTokenizer { // SPM-style BPE (Gemma-4): raw-UTF-8 merges (no GPT-2 byte alphabet), '▁' space escape, // newline-run pre-split. No field default — zero-init `var tk : BpeTokenizer` must stay legal. spm_space : bool - // whether encode(add_special) prepends BOS — the GGUF add_bos_token key, else a per-pre - // default (gpt-4o adds none). Same no-field-default rule as spm_space. add_bos : bool } +//! Whether a BPE vocab whose GGUF omits `tokenizer.ggml.add_bos_token` prepends BOS - upstream's +//! per-pre default; the list is the pres that add one, every other pre adds none. +def bpe_pre_adds_bos(pre : string) : bool { + for (p in ["llama3", "llama-v3", "llama-bpe", "falcon3", "falcon-h1", "pixtral", "midm-2.0", "lfm2", "jina-v5-nano", "tekken", "chameleon"]) { + return true if (pre == p) + } + return false +} + //! Load the BPE vocab / merges / token-types from a llama-arch GGUF's tokenizer metadata. def load_bpe_tokenizer_gguf(path : string) : BpeTokenizer { var tk : BpeTokenizer @@ -65,11 +72,9 @@ def load_bpe_tokenizer_gguf(path : string) : BpeTokenizer { // Qwen2.5-Omni GGUFs omit bos_token_id entirely (qwen2 never adds BOS) tk.bos_id = gguf_has(m, "tokenizer.ggml.bos_token_id") ? gguf_int(m, bytes, "tokenizer.ggml.bos_token_id") : -1l tk.eos_id = gguf_int(m, bytes, "tokenizer.ggml.eos_token_id") - // the metadata key wins; absent, the per-pre default applies (gpt-4o adds no BOS, - // the llama-bpe family keeps it) tk.add_bos = (gguf_has(m, "tokenizer.ggml.add_bos_token") ? gguf_int(m, bytes, "tokenizer.ggml.add_bos_token") != 0l - : tk.pre != "gpt-4o") + : bpe_pre_adds_bos(tk.pre)) if (tk.pre == "gpt-4o") { // these gpt-oss control tokens force-render (attr -> USER_DEFINED) so channel // markers stay visible in decoded text (type 4 escapes the type-3 mute rule) diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 913b77286a..285c90aceb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -1316,6 +1316,14 @@ def cls_kq(t : Model) : bool => t.kquant_native && t.config.shared_weights && t. def fmt_at(a : array; l : int64) : KqFmt => empty(a) ? KqFmt.q8 : a[l] // Exact reserve then size — no pow2 slack, and the reserved resize never trips max_unreserved_size. +//! A byte count against its baseline for a report line: "+12 MB, +3.4%" (a zero baseline prints n/a for the percent). +def mb_delta(now, base : int64) : string { + let d = now - base + let sign = d < 0l ? "-" : "+" + let pct = base > 0l ? "{sign}{100.0lf * double(abs(d)) / double(base):.1f}%" : "n/a" + return "{sign}{abs(d) >> 20l} MB, {pct}" +} + def reserve_resize(var a : array; n : int | int64) { a |> reserve(n) a |> resize(n) diff --git a/modules/dasLLAMA/dasllama/dasllama_env.das b/modules/dasLLAMA/dasllama/dasllama_env.das index 61788b6cac..7aee4ab33b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_env.das +++ b/modules/dasLLAMA/dasllama/dasllama_env.das @@ -346,8 +346,8 @@ struct public VulkanEnv { @clarg_doc = "Serve from P3-trimmed vulkan images (big CPU weight families dropped; folded into the flavor identity)." trim : bool = false - @clarg_doc = "Tag allocations high-priority (VK_EXT_memory_priority) so the driver demotes desktop memory, not ours." - vk_memprio : bool = true + @clarg_doc = "Tag every device allocation priority 1.0 (VK_EXT_memory_priority). Off by default: on the NVIDIA WDDM driver the tag makes the tagged weights the pageable set, and a resident model then decodes at PCIe speed once anything is demoted." + vk_memprio : bool = false @clarg_doc = "Vulkan flash attention: the decode fa kernel pick AND the cm2 prefill fa tile; 0 falls back to the chunked/scalar paths." vk_fa : bool = true @@ -373,10 +373,10 @@ struct public VulkanEnv { @clarg_doc = "Arm the resident driver with f32 KV mirrors instead of the f16 default (A/B instrument; only sessions of the armed codec are served)." vk_kv32 : int64 = 0l - @clarg_doc = "cm2 prefill tile pick: 0 = occupancy heuristic, 128 = force the m tile, 256 = force the l tile (A/B instrument)." + @clarg_doc = "cm2 prefill tile pick: 0 = occupancy heuristic, 128 = force the m tile, 256 = force the l tile (A/B instrument). Inert on the KHR arm (DASLLAMA_COOPMAT=mm, or a device without NV_coopmat2), whose kq tile has one geometry." cm2_tile : int64 = 0l - @clarg_doc = "cm2 split-k: 0 = occupancy heuristic, 1 = off, N = force N k-chunks (A/B instrument; shrinks if N strands an empty tail)." + @clarg_doc = "cm2 split-k: 0 = occupancy heuristic, 1 = off, N = force N k-chunks (A/B instrument; shrinks if N strands an empty tail). Inert on the KHR arm, whose kq tile carries no split-k scratch." cm2_splitk : int64 = 0l @clarg_path diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 16ee216ab5..7f63d1910d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -5,7 +5,8 @@ options _dasllama_internal = true module dasllama_gemm_schema shared public require dasllama/dasllama_lint public -require dasllama/dasllama_kqformat public // the format taxonomy + typed stride truth these int-id forms delegate to +require dasllama/dasllama_kqformat public +require daslib/enum_trait // the format taxonomy + typed stride truth these int-id forms delegate to // The one place the Q8 interleaved-weight layout is defined (gemm_generator_plan.md part 3): // the [llvm_code] tile generator derives its strides from q8q8_repack_type at emit time, and @@ -48,11 +49,26 @@ let Q8_BLOCK_ELEMS = 32l let Q8_QPB = 32l // quant bytes per block let Q8_SPB = 2l // scale bytes per block (f16 halfword) -//! kq superblock element count, and the DEVICE scale-row stride: the gathers decode every kq -//! format's scales to 20B rows on device (kq_ssb below is the DISK/plane stride, per format). +//! kq superblock element count, and the DEVICE scale-row stride the gathers decode a kq format's +//! scales to (kq_ssb below is the DISK/plane stride, per format); kq_dev_ssb is the per-format form. let KQ_SUPERBLOCK_ELEMS = 256l let KQ_DEV_SSB = 20l +//! The codebook formats whose device scale row is two words: word 0 = f16 d with the format's +//! native high bits above it, word 1 = eight 4-bit fields. +def kq_two_word_row(f : KqFmt) : bool => f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq2xxs + +//! The device scale-row stride of a kq format: 8 B for the two-word-row formats, KQ_DEV_SSB for the rest. +def kq_dev_ssb(f : KqFmt) : int64 => kq_two_word_row(f) ? 8l : KQ_DEV_SSB + +//! kq_dev_ssb over the tier's int(KqFmt) stack-format space. +def kq_dev_ssb_i(fmt : int) : int64 { + for (f in type) { + return kq_dev_ssb(f) if (int(f) == fmt) + } + return KQ_DEV_SSB +} + //! q5_1 per-32-block plane strides: 16B nibbles + 4B qh quants, f16 d + f16 m scales. let Q51_QPB = 20l let Q51_SPB = 4l diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index d033a9d8ab..dcd2728eca 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -63,6 +63,64 @@ let GGML_TYPE_IQ4_XS = 23 let GGML_TYPE_BF16 = 30 let GGML_TYPE_MXFP4 = 39 +//! Bytes `n` elements of a ggml type take on disk (block bytes x blocks); 0 for a type the loader +//! does not know, so a census sums what it can and names the rest. +def ggml_type_bytes(gt : int; n : int64) : int64 { + if (gt == GGML_TYPE_F32) return n * 4l + if (gt == GGML_TYPE_F16 || gt == GGML_TYPE_BF16) return n * 2l + if (gt == GGML_TYPE_Q4_0 || gt == GGML_TYPE_IQ4_NL) return n / 32l * 18l + if (gt == GGML_TYPE_Q5_0) return n / 32l * 22l + if (gt == GGML_TYPE_Q5_1) return n / 32l * 24l + if (gt == GGML_TYPE_Q8_0) return n / 32l * 34l + if (gt == GGML_TYPE_MXFP4) return n / 32l * 17l + if (gt == GGML_TYPE_Q2_K) return n / 256l * 84l + if (gt == GGML_TYPE_Q3_K || gt == GGML_TYPE_IQ3_S) return n / 256l * 110l + if (gt == GGML_TYPE_Q4_K) return n / 256l * 144l + if (gt == GGML_TYPE_Q5_K) return n / 256l * 176l + if (gt == GGML_TYPE_Q6_K) return n / 256l * 210l + if (gt == GGML_TYPE_IQ2_XXS) return n / 256l * 66l + if (gt == GGML_TYPE_IQ2_XS) return n / 256l * 74l + if (gt == GGML_TYPE_IQ2_S) return n / 256l * 82l + if (gt == GGML_TYPE_IQ3_XXS) return n / 256l * 98l + if (gt == GGML_TYPE_IQ4_XS) return n / 256l * 136l + return 0l +} + +//! The ggml type a kq format's plane converts from; -1 for a format no ggml type spells. +def kq_ggml_type(f : KqFmt) : int { + if (f == KqFmt.q8) return GGML_TYPE_Q8_0 + if (f == KqFmt.q40) return GGML_TYPE_Q4_0 + if (f == KqFmt.iq4nl) return GGML_TYPE_IQ4_NL + if (f == KqFmt.q51) return GGML_TYPE_Q5_1 + if (f == KqFmt.k2) return GGML_TYPE_Q2_K + if (f == KqFmt.k3) return GGML_TYPE_Q3_K + if (f == KqFmt.iq3s) return GGML_TYPE_IQ3_S + if (f == KqFmt.k4) return GGML_TYPE_Q4_K + if (f == KqFmt.k5) return GGML_TYPE_Q5_K + if (f == KqFmt.k6) return GGML_TYPE_Q6_K + if (f == KqFmt.iq4xs) return GGML_TYPE_IQ4_XS + if (f == KqFmt.iq3xxs) return GGML_TYPE_IQ3_XXS + if (f == KqFmt.iq2s) return GGML_TYPE_IQ2_S + if (f == KqFmt.iq2xs) return GGML_TYPE_IQ2_XS + if (f == KqFmt.iq2xxs) return GGML_TYPE_IQ2_XXS + return -1 +} + +//! Bytes `n` weights of a kq format take on disk (the ggml block sizes) - what a device plane +//! is measured against; q8 counts its Q8_0 blocks. +def kq_disk_bytes(f : KqFmt; n : int64) : int64 => ggml_type_bytes(kq_ggml_type(f), n) + +let private GGML_TYPE_NAMES <- { + GGML_TYPE_F32 => "F32", GGML_TYPE_F16 => "F16", GGML_TYPE_BF16 => "BF16", + GGML_TYPE_Q4_0 => "Q4_0", GGML_TYPE_Q5_0 => "Q5_0", GGML_TYPE_Q5_1 => "Q5_1", GGML_TYPE_Q8_0 => "Q8_0", GGML_TYPE_MXFP4 => "MXFP4", + GGML_TYPE_Q2_K => "Q2_K", GGML_TYPE_Q3_K => "Q3_K", GGML_TYPE_Q4_K => "Q4_K", GGML_TYPE_Q5_K => "Q5_K", GGML_TYPE_Q6_K => "Q6_K", + GGML_TYPE_IQ2_XXS => "IQ2_XXS", GGML_TYPE_IQ2_XS => "IQ2_XS", GGML_TYPE_IQ2_S => "IQ2_S", + GGML_TYPE_IQ3_XXS => "IQ3_XXS", GGML_TYPE_IQ3_S => "IQ3_S", GGML_TYPE_IQ4_NL => "IQ4_NL", GGML_TYPE_IQ4_XS => "IQ4_XS" +} + +//! The ggml type's upstream spelling for logs ("Q4_K", "IQ3_S"); "type N" for one the loader does not know. +def ggml_type_name(gt : int) : string => GGML_TYPE_NAMES?[gt] ?? "type {gt}" + // MXFP4 e2m1 magnitudes DOUBLED (nibble -> ±{0,.5,1,1.5,2,3,4,6}×2, OCP MX spec); the E8M0 // block scale below is halved to compensate. let private MXFP4_KVALUES = fixed_array(0.0, 1.0, 2.0, 3.0, 4.0, 6.0, 8.0, 12.0, diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 2e4741c530..9d5b924df1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -17,6 +17,7 @@ require dasllama/dasllama_kqformat require dasllama/dasllama_math require dasllama/dasllama_rope // build_rope_row_packed — the one source for every resident rope row require dasllama/dasllama_gemm_schema // Q8_BLOCK_ELEMS / KQ_SUPERBLOCK_ELEMS — device block sizing +require dasllama/dasllama_gguf // kq_disk_bytes - the file bytes a device plane is measured against require daslib/f16_cvt // f32_to_f16 — the trim pass re-packs the emb scale plane require dasllama/dasllama_par require dasllama/dasllama_env @@ -414,8 +415,10 @@ struct ResidentPlan { weight_bytes : int64 kv_bytes : int64 scratch_bytes : int64 //!< the driver's own device scratch (prefill window + decode smalls) - headroom_bytes : int64 //!< desktop slack the auto arm leaves unfilled (0 when VRAM is pinned) + headroom_bytes : int64 //!< desktop slack the auto arm leaves unfilled (0 when VRAM is pinned, or when the OS budget answers) budget_bytes : int64 + usable_bytes : int64 //!< what the plan may fill: the budget minus the headroom, capped by the room the OS reports when it answers and no pin overrides + others_bytes : int64 //!< dedicated video memory every other process holds, when the OS answers (0 otherwise) - named in a decline emb_f32_bytes : int64 //!< the raw f32 embd table the gather would upload - counted only when it rides the headroom left after weights + KV (0 = the CPU embed serves) fits : bool reason : string //!< "" when it fits; otherwise why, with a remedy where one exists @@ -426,6 +429,7 @@ let private RDEC_VRAM_HEADROOM = 2_147_483_648l // auto-arm slack past the ti // filling weights + KV to the cap brim measured a // 12x WDDM-demotion collapse on a live desktop let private RDEC_VRAM_HEADROOM_PCT = 27l +let private RDEC_OS_RESERVE = 1_073_741_824l let private RDEC_MISC_BYTES = 67_108_864l // decode smalls + batch act planes + metas // the driver's device allocations past weights + KV — the prefill window buffers dominate. @@ -440,6 +444,40 @@ def private rdec_scratch_bytes(t : Model) : int64 { + 8l * hid + 4l * layer_head_size(c, 0l)) + c.vocab_size * 4l + RDEC_MISC_BYTES } +//! The room a plan may fill: the tier's cap less its headroom, capped by the OS's room when the OS +//! answers - its per-process budget less what the other processes hold on the adapter and the reserve. +//! A pinned cap takes no headroom and reads no room. +[arch(at="../ARCHITECTURE_GPU_VULKAN_RESIDENCY.md#resident-plan")] +def plan_room(budget_bytes : int64; pinned : bool; os : tuple) : tuple { + let answered = !pinned && os.budget > 0l + let headroom = (pinned || answered) ? 0l : max(RDEC_VRAM_HEADROOM, budget_bytes * RDEC_VRAM_HEADROOM_PCT / 100l) + var usable = budget_bytes - headroom + var others = 0l + if (answered) { + others = max(os.adapter_dedicated - os.usage, 0l) + usable = min(usable, os.budget - others - RDEC_OS_RESERVE) + } + return (headroom = headroom, usable = usable, others = others) +} + +def private fill_plan_room(var p : ResidentPlan) { + let pinned = gpu_want_vram_mb() > 0l + let room = plan_room(p.budget_bytes, pinned, pinned ? (budget = 0l, usage = 0l, adapter_dedicated = 0l, adapter_shared = 0l) : moe_gpu_os_memory()) + p.headroom_bytes = room.headroom + p.usable_bytes = room.usable + p.others_bytes = room.others +} + +//! 0 = no cap known, or no attention layer. +[arch(at="../ARCHITECTURE_GPU_VULKAN_RESIDENCY.md#resident-plan")] +def resident_binding_ctx(c : Config; dt : KVDtype; bcap : int64) : int64 { + var side = 0l + for (l in range64(c.n_layers)) { + side += kv_row_bytes(dt, layer_kv_dim(c, l)) + } + return (bcap > 0l && side > 0l) ? bcap / side : 0l +} + [arch(at="../ARCHITECTURE_GPU_VULKAN_RESIDENCY.md#resident-plan")] def resident_plan(t : Model; seq_cap : int64; kdt, vdt : KVDtype) : ResidentPlan { let c = t.config @@ -460,7 +498,7 @@ def resident_plan(t : Model; seq_cap : int64; kdt, vdt : KVDtype) : ResidentPlan } p.kv_bytes = seq_cap * (krow + vrow) p.scratch_bytes = rdec_scratch_bytes(t) - p.headroom_bytes = gpu_want_vram_mb() > 0l ? 0l : max(RDEC_VRAM_HEADROOM, p.budget_bytes * RDEC_VRAM_HEADROOM_PCT / 100l) + fill_plan_room(p) // every 2D plane a dense forward reads: the attention quad, the FFN triple, the classifier for (l in range64(c.n_layers)) { let qd = layer_qd(c, l) @@ -494,7 +532,7 @@ def resident_plan(t : Model; seq_cap : int64; kdt, vdt : KVDtype) : ResidentPlan let cls_fmt = c.shared_weights ? t.emb_fmt : t.wcls_fmt p.weight_bytes += moe_gpu_plane_bytes(c.dim, c.vocab_size, int(cls_fmt)) let total = p.weight_bytes + p.kv_bytes + p.scratch_bytes - let usable = p.budget_bytes - p.headroom_bytes + let usable = p.usable_bytes let emb_want = g_env_vulkan.vk_gpu_embed && rdec_prefill_ids_installed() && !t.cls_q8 && !cls_kq(t) && !t.planes_trimmed let emb_bytes = emb_want ? rdec_emb_f32_bytes(c.vocab_size, c.dim) : 0l p.emb_f32_bytes = total + emb_bytes <= usable ? emb_bytes : 0l @@ -504,9 +542,10 @@ def resident_plan(t : Model; seq_cap : int64; kdt, vdt : KVDtype) : ResidentPlan // a shorter context is the remedy that actually works — weights are fixed, KV is not let kv_fit = max(usable - p.weight_bytes - p.scratch_bytes, 0l) let cap_fit = (krow + vrow) > 0l ? kv_fit / (krow + vrow) : 0l + let room = p.others_bytes > 0l ? " - other processes hold {p.others_bytes >> 20l} MB of the card" : "" p.reason = (cap_fit >= 512l - ? "needs {total >> 20l} MB of {usable >> 20l} MB at ctx {seq_cap} - would fit at ctx {cap_fit}" - : "needs {total >> 20l} MB of {usable >> 20l} MB (weights alone {p.weight_bytes >> 20l} MB)") + ? "needs {total >> 20l} MB of {usable >> 20l} MB at ctx {seq_cap} - would fit at ctx {cap_fit}{room}" + : "needs {total >> 20l} MB of {usable >> 20l} MB (weights alone {p.weight_bytes >> 20l} MB){room}") } return p } @@ -611,7 +650,7 @@ def private name_list_add(var names : string&; on : bool; name : string) { //! it serves every feature the config carries. The list is the decline reason a user reads. def resident_unserved_features(c : Config) : string { var names = "" - name_list_add(names, c.attn_qkv_bias, "attention qkv bias") + name_list_add(names, c.attn_qkv_bias && !rdec_bias_installed(), "attention qkv bias") name_list_add(names, c.attn_out_bias, "attention output bias") name_list_add(names, c.v_norm, "v-norm") name_list_add(names, c.attn_sinks, "attention sinks") @@ -640,7 +679,10 @@ def resident_layer_decline(t : Model; l, hid0, hs, kvd : int64) : string { if (rdec_quant_class(fqkv) != rdec_quant_class(fz)) { return "layer {l}'s deltanet qkv and z planes mix activation-quant forms ({int(fqkv)}/{int(fz)}) - one x feed serves both" } - if (fmt_at(t.dnout_fmt, l) != KqFmt.q8) return "layer {l}'s deltanet out plane is kq-tagged (the step's o row feeds a q8 GEMV)" + let fout = fmt_at(t.dnout_fmt, l) + if (!rdec_dn_fmt_ok(fout, c.ssm_d_inner)) { + return "layer {l}'s deltanet out plane carries a format ({int(fout)}) the resident GEMVs do not serve at width {c.ssm_d_inner}" + } if (!t.dn_ba_f32 && (t.dnbeta_offs[l] < 0l || t.dnalpha_offs[l] < 0l)) return "layer {l} lacks its beta/alpha planes" if (!rdec_dn_installed()) return "layer {l} is recurrent and the tier installed no deltanet seats for the whole-model driver" if (layer_hidden(t, l) != hid0) return "layer {l} FFN width {layer_hidden(t, l)} differs from layer 0's {hid0}" @@ -673,11 +715,65 @@ def private resident_upload_dn_ba(t : Model; n_recurrent : int64) { delete rows } +[arch(at="../ARCHITECTURE_GPU_VULKAN_DECODE.md#hybrid-token-command")] +def private resident_upload_bias(t : Model; qd, kvd : int64) { + let c = t.config + let stride = qd + 2l * kvd + var rows : array + rows |> reserve_resize(c.n_layers * stride) + for (l in range64(c.n_layers)) { + continue if (layer_is_recurrent(c, l)) + copy_floats(t.bq, l * qd, rows, l * stride, qd) + copy_floats(t.bk, l * kvd, rows, l * stride + qd, kvd) + copy_floats(t.bv, l * kvd, rows, l * stride + qd + kvd, kvd) + } + rdec_upload_bias(rows) + delete rows +} + +struct private ImageRow { + name : string + plane_count : int64 + file_bytes : int64 + device_bytes : int64 +} + +//! The device-side twin of the load report: every plane the whole-model driver uploads, per +//! format, as bytes in the file's own blocks against bytes in the device planes, biggest overhead +//! first, then the KV mirror and scratch the plan adds at this context. +def private log_resident_image(planes : array>; plan : ResidentPlan; seq_cap : int64) { + var rows : table + var file_total = 0l + var device_total = 0l + for (pl in planes) { + let fi = int(pl.f) + var r = rows[fi] + r.name = "{pl.f}" + r.plane_count++ + r.file_bytes += kq_disk_bytes(pl.f, pl.n * pl.rows) + r.device_bytes += moe_gpu_plane_bytes(pl.n, pl.rows, fi) + rows[fi] = r + file_total += kq_disk_bytes(pl.f, pl.n * pl.rows) + device_total += moe_gpu_plane_bytes(pl.n, pl.rows, fi) + } + var sorted : array + for (v in values(rows)) { + sorted |> emplace(ImageRow(name = clone_string(v.name), plane_count = v.plane_count, file_bytes = v.file_bytes, device_bytes = v.device_bytes)) + } + sorted |> sort() $(a, b) => a.device_bytes - a.file_bytes > b.device_bytes - b.file_bytes + to_log(LOG_INFO, "dasLLAMA: resident image: {file_total >> 20l} MB in the file's blocks -> {device_total >> 20l} MB on device ({mb_delta(device_total, file_total)}) over {long_length(planes)} planes; KV mirror {plan.kv_bytes >> 20l} MB at ctx {seq_cap}, scratch {plan.scratch_bytes >> 20l} MB\n") + for (r in sorted) { + to_log(LOG_INFO, "dasLLAMA: resident image: {r.name} x{r.plane_count}: {r.file_bytes >> 20l} MB -> {r.device_bytes >> 20l} MB ({mb_delta(r.device_bytes, r.file_bytes)})\n") + } + delete sorted + delete rows +} + //! Reserve arenas + place every dense plane + wire the resident decode driver. Returns false (and //! leaves the driver unarmed) if the model is not a servable dense shape or does not fit at seq_cap; //! every decline logs its reason through `rdec_decline`. [arch(at="../ARCHITECTURE_GPU.md#gpu-backends")] -def resident_upload(t : Model; seq_cap : int64; rkdt : KVDtype) : bool { // nolint:STYLE037,STYLE038 — all-or-nothing arm: one reject per unsupported family feature, no seam +def resident_upload(t : Model; seq_cap : int64; rkdt : KVDtype; plan : ResidentPlan) : bool { // nolint:STYLE037,STYLE038 — all-or-nothing arm: one reject per unsupported family feature, no seam g_rdec_active = false g_rdec_emb_gpu = false g_rdec_pass_said |> clear() @@ -729,7 +825,7 @@ def resident_upload(t : Model; seq_cap : int64; rkdt : KVDtype) : bool { // no && c.shared_weights && t.cls_q8 && cls_fmt == KqFmt.q8) let emb_f32_want = (g_env_vulkan.vk_gpu_embed && rdec_prefill_ids_installed() && !t.emb_q8 && !cls_kq(t) && !t.planes_trimmed // an untied q8 table has no f32 copy to upload: the CPU embed loop serves it - && resident_plan(t, seq_cap, rkdt, rkdt).emb_f32_bytes > 0l) // the plan counted it at this context + && plan.emb_f32_bytes > 0l) // the plan counted it at this context var planes : array> planes |> reserve(c.n_layers * 8l + 1l) for (l in range64(c.n_layers)) { @@ -763,6 +859,7 @@ def resident_upload(t : Model; seq_cap : int64; rkdt : KVDtype) : bool { // no let fi = int(pl.f) need[fi] = (need?[fi] ?? 0l) + rdec_blocks_for(pl.n, pl.rows, pl.f) } + log_resident_image(planes, plan, seq_cap) delete planes var reserve_ok = true for (fmt, blocks in keys(need), values(need)) { @@ -783,9 +880,13 @@ def resident_upload(t : Model; seq_cap : int64; rkdt : KVDtype) : bool { // no rkdt == KVDtype.f16, c.q_gated, rot, dng)) { return rdec_decline("the device prepare declined (a kernel class or the mirror range - the tier's line above)") } + rdec_note_weight_bytes(plan.weight_bytes) if (n_recurrent > 0l && dng.ba_f32) { resident_upload_dn_ba(t, n_recurrent) } + if (c.attn_qkv_bias) { + resident_upload_bias(t, qd, kvd) + } // place + wire each layer. One quantized X image feeds each consumer group (qkv shares one, // gate/up another), so a group must agree on its quant form — mixed groups decline var mir_idx = 0l @@ -1389,7 +1490,6 @@ def private walk_budget_say(rail : string; l, layers : int64) { //! The attention features the per-op attention-quad chain has no arm for, comma-separated; "". def attn_chain_unserved_features(c : Config) : string { var names = "" - name_list_add(names, c.attn_qkv_bias, "attention qkv bias") name_list_add(names, c.v_norm, "v-norm") name_list_add(names, c.attn_sinks, "attention sinks") name_list_add(names, c.attn_logit_softcap != 0.0, "attention logit softcap") @@ -1475,26 +1575,22 @@ def moe_gpu_upload_resident(t : Model) { // nolint:STYLE037,STYLE038 — seven // the mirror codec: f16 rows by default (native fa feed, ~2x plannable ctx); // DASLLAMA_VK_KV32=1 arms f32 mirrors instead (A/B; then only f32 sessions serve) let rkdt = g_env_vulkan.vk_kv32 != 0l ? KVDtype.f32 : KVDtype.f16 - // each side of the KV mirror binds as ONE SSBO range — the device's max binding - // range caps the armable ctx (past it every mirror read is undefined, not slow) - let bcap = moe_gpu_binding_cap() - let mrow = t.config.n_layers * kv_row_bytes(rkdt, layer_kv_dim(t.config, 0l)) - if (bcap > 0l && mrow > 0l) { - seq_cap = min(seq_cap, bcap / mrow) + let bctx = resident_binding_ctx(t.config, rkdt, moe_gpu_binding_cap()) + if (bctx > 0l) { + seq_cap = min(seq_cap, bctx) } var plan = resident_plan(t, seq_cap, rkdt, rkdt) if (!plan.fits && plan.weight_bytes > 0l && plan.kv_bytes > 0l) { // weights are fixed, the KV mirror is not: retry at the context that fits (the plan's // own remedy), as long as it stays a usable window. The driver guards decode past the cap. - let fit_ctx = (max(plan.budget_bytes - plan.headroom_bytes - plan.weight_bytes - - plan.scratch_bytes, 0l)) / (plan.kv_bytes / seq_cap) + let fit_ctx = (max(plan.usable_bytes - plan.weight_bytes - plan.scratch_bytes, 0l)) / (plan.kv_bytes / seq_cap) if (fit_ctx >= rdec_min_ctx()) { seq_cap = fit_ctx plan = resident_plan(t, seq_cap, rkdt, rkdt) } } if (plan.fits) { - if (resident_upload(t, seq_cap, rkdt)) { + if (resident_upload(t, seq_cap, rkdt, plan)) { return } } else { @@ -1702,9 +1798,8 @@ def moe_gpu_upload_resident(t : Model) { // nolint:STYLE037,STYLE038 — seven vulkan_bake_role(VkBakeRole.attn) if (at_want) { let c = t.config - // NEOX rope only (partial rotary implies NEOX per the loader); the chain has no bias / - // v-norm / sinks / softcap / post-norm arms - let arch_ok = (!c.attn_qkv_bias && !c.v_norm && !c.attn_sinks && c.attn_logit_softcap == 0.0 + // partial rotary implies NEOX per the loader, so the rope test admits it unroped + let arch_ok = (!c.v_norm && !c.attn_sinks && c.attn_logit_softcap == 0.0 && !c.pre_post_norm && !c.attn_out_bias && (c.rope_neox || (c.rope_dim > 0l && c.rope_dim != c.head_size))) var at_marked = 0l @@ -1746,7 +1841,7 @@ def moe_gpu_upload_resident(t : Model) { // nolint:STYLE037,STYLE038 — seven walk_skip(at_skips, l, "q/k/v/o formats {int(fq)}/{int(fk)}/{int(fv)}/{int(fo)} not tier-servable as one chain") continue } - if (!(hs == 128l || hs == 256l) || qd > 8192l || kvd > 1024l + if (!(hs == 128l || hs == 256l) || qd > 8192l || kvd > 2048l || c.n_heads > 64l || rot % 2l != 0l || rot > hs || kvd % 32l != 0l || qd % (fo != KqFmt.q8 ? 256l : 32l) != 0l || dim % 32l != 0l) { walk_skip(at_skips, l, "geometry outside the chain (hs {hs}, qd {qd}, kvd {kvd}, heads {c.n_heads}, rot {rot})") diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_tier.das b/modules/dasLLAMA/dasllama/dasllama_gpu_tier.das index fd45e49023..1b8c0d0226 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_tier.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_tier.das @@ -48,9 +48,10 @@ typedef MoeGpuAttnFn = function<(var yp : float?; var kp : float?; var vp : floa fq : int; fk : int; fv : int; fo : int; xqp : int8 const?; xsp : float const?; rmsqp : float const?; rmskp : float const?; + bqp : float const?; bkp : float const?; bvp : float const?; cosp : float const?; sinp : float const?; npos : int64; dim : int64; qd : int64; kv_dim : int64; - hs : int64; rot : int64; n_heads : int64; kv_mul : int64; + hs : int64; rot : int64; n_heads : int64; scale : float; eps : float; gated : bool; qknorm : bool) : void> [unused_argument(goutp, offs1, offs3, offs2, nregions, xqp, xsp, wp, invp, n, nfe, nrows, npos, f1, f3, f2, is_gelu)] @@ -87,13 +88,14 @@ def private moe_gpu_unset_dn(var yp : float?; woq, woz, woo : int64; panic("dasLLAMA: GPU deltanet dispatch hit without an installed GPU tier") } -[unused_argument(yp, kp, vp, woq, wok, wov, woo, fq, fk, fv, fo, xqp, xsp, rmsqp, rmskp, cosp, sinp, npos, dim, qd, kv_dim, hs, rot, n_heads, kv_mul, scale, eps, gated, qknorm)] +[unused_argument(yp, kp, vp, woq, wok, wov, woo, fq, fk, fv, fo, xqp, xsp, rmsqp, rmskp, bqp, bkp, bvp, cosp, sinp, npos, dim, qd, kv_dim, hs, rot, n_heads, scale, eps, gated, qknorm)] def private moe_gpu_unset_attn(var yp : float?; var kp : float?; var vp : float?; woq, wok, wov, woo : int64; fq, fk, fv, fo : int; xqp : int8 const?; xsp : float const?; rmsqp : float const?; rmskp : float const?; + bqp : float const?; bkp : float const?; bvp : float const?; cosp : float const?; sinp : float const?; - npos, dim, qd, kv_dim, hs, rot, n_heads, kv_mul : int64; + npos, dim, qd, kv_dim, hs, rot, n_heads : int64; scale : float; eps : float; gated : bool; qknorm : bool) { panic("dasLLAMA: GPU attention dispatch hit without an installed GPU tier") } @@ -135,6 +137,7 @@ typedef RdecNormsFn = function<(norms : array) : void> typedef RdecSetLayerFn = function<(l : int64; mir_idx : int64; bq : int64; bk : int64; bv : int64; bo : int64; b1 : int64; b3 : int64; b2 : int64; fq : int; fk : int; fv : int; fo : int; f1 : int; f3 : int; f2 : int) : void> typedef RdecSetDnLayerFn = function<(l : int64; bqkv : int64; bz : int64; bout : int64; bbeta : int64; balpha : int64; b1 : int64; b3 : int64; b2 : int64; ba_off : int64; fqkv : int; fz : int; fout : int; f1 : int; f3 : int; f2 : int; convw : float const?; wnormp : float const?; ap : float const?; dtp : float const?) : void> typedef RdecUploadDnBaFn = function<(rows : array) : void> +typedef RdecUploadBiasFn = function<(rows : array) : void> typedef RdecDnOwnFn = function<(l : int64; stp : float?; histp : float?) : void> typedef RdecDnPrefillOwnFn = function<(l : int64; stp : float?; histp : float?) : void> typedef RdecSetClsFn = function<(cls_block : int64; cls_fmt : int) : void> @@ -199,6 +202,23 @@ var g_rdec_dn_own = @@rdec_unset_dn_own var g_rdec_dn_prefill_own = @@rdec_unset_dn_prefill_own var g_rdec_dn_installed = false +[unused_argument(rows)] +def private rdec_unset_upload_bias(rows : array) { + panic("dasLLAMA: resident q/k/v bias upload hit without an installed bias seat") +} +var g_rdec_upload_bias = @@rdec_unset_upload_bias +var g_rdec_bias_installed = false + +//! Install the resident driver's q/k/v projection-bias seat (qwen2) - separately, so a tier without +//! it declines a biased model by name instead of serving it unbiased. +def public install_moe_gpu_resident_bias(upload_bias : RdecUploadBiasFn) { + g_rdec_upload_bias = upload_bias + g_rdec_bias_installed = true +} + +def public rdec_bias_installed() : bool => g_rdec_bias_installed +def public rdec_upload_bias(rows : array) { invoke(g_rdec_upload_bias, rows) } + //! Install the resident driver's hybrid (deltanet) seats - separately, so a tier without them //! declines recurrent layers by name instead of serving a model it cannot step. def public install_moe_gpu_resident_dn(set_dn_layer : RdecSetDnLayerFn; upload_dn_ba : RdecUploadDnBaFn; dn_own : RdecDnOwnFn; dn_prefill_own : RdecDnPrefillOwnFn) { @@ -883,6 +903,11 @@ var g_moe_gpu_vram_report = @@moe_gpu_no_vram var g_moe_gpu_device_report = @@moe_gpu_no_device var g_moe_gpu_budget_report = @@moe_gpu_no_budget var g_moe_gpu_plane_bytes_fn = @@moe_gpu_no_plane_bytes +def private moe_gpu_no_os_memory() : tuple => (budget = 0l, usage = 0l, adapter_dedicated = 0l, adapter_shared = 0l) +[unused_argument(bytes)] +def private rdec_no_note_weight_bytes(bytes : int64) {} +var g_moe_gpu_os_memory = @@moe_gpu_no_os_memory +var g_rdec_note_weight_bytes = @@rdec_no_note_weight_bytes def set_gpu_tier_want_(want : GpuTierWant) { g_gpu_want = want @@ -1039,6 +1064,23 @@ def public set_moe_gpu_budget_hooks(budget : function<() : int64>; plane : funct //! Device-local bytes the armed tier may spend on resident weights (0 = no tier). def moe_gpu_weight_budget_() : int64 => invoke(g_moe_gpu_budget_report) +//! The OS's video memory numbers, in bytes (all 0 = no OS answer): this process's dedicated ceiling, +//! what it holds, what every process holds on the adapter, the system memory the adapter maps - the +//! resident plan's auto arm sizes against the room left where a backend installs the query. +def moe_gpu_os_memory() : tuple => invoke(g_moe_gpu_os_memory) + +def public install_moe_gpu_os_memory(f : function>) { + g_moe_gpu_os_memory = f +} + +//! Tell the resident driver how many weight bytes its plan put on the device (its warm-up +//! self-check reads the decode wall against them). +def rdec_note_weight_bytes(bytes : int64) { invoke(g_rdec_note_weight_bytes, bytes) } + +def public install_rdec_note_weight_bytes(f : function<(bytes : int64) : void>) { + g_rdec_note_weight_bytes = f +} + def private moe_gpu_no_binding_cap : int64 => 0l var private g_moe_gpu_binding_cap = @@moe_gpu_no_binding_cap @@ -1233,6 +1275,9 @@ struct MoeSpanDec { f2 : array rmsq : array // the qk-norm rows (null when off) rmsk : array + bq : array // the q/k/v projection bias rows (null when the model has none) + bk : array + bv : array router : float const? // [nl x ne x dim] f32 rows, layer-major att_norm : float const? // [nl x dim] ffn_norm : float const? // [nl x dim] @@ -1387,7 +1432,7 @@ def public matmul_moe_gpu_ffn_combined_xf_join(var gout : array) { // ===== the decode attention block: one submit per layer over the resident q/k/v/o quad ===== // The tier keeps a per-layer f16 K/V mirror the prefill chain fills; a served token returns the // wo row plus its RAW k and v rows (f32, before norm and rope) so the host re-derives its cache row. -typedef MoeGpuAttnDecFn = function<(var xb2p : float?; var kp : float?; var vp : float?; woq : int64; wok : int64; wov : int64; woo : int64; fq : int; fk : int; fv : int; fo : int; xqp : int8 const?; xsp : float const?; cosp : float const?; rmsqp : float const?; rmskp : float const?; pos : int64; dim : int64; qd : int64; kvd : int64; hs : int64; n_heads : int64; kv_mul : int64; scale : float; eps : float; neox : bool; qk_norm : bool) : void> +typedef MoeGpuAttnDecFn = function<(var xb2p : float?; var kp : float?; var vp : float?; woq : int64; wok : int64; wov : int64; woo : int64; fq : int; fk : int; fv : int; fo : int; xqp : int8 const?; xsp : float const?; cosp : float const?; rmsqp : float const?; rmskp : float const?; bqp : float const?; bkp : float const?; bvp : float const?; pos : int64; dim : int64; qd : int64; kvd : int64; hs : int64; n_heads : int64; kv_mul : int64; scale : float; eps : float; neox : bool; qk_norm : bool) : void> typedef MoeGpuAttnDecOkFn = function<(woq : int64; pos : int64; gen : int64) : int> typedef MoeGpuAttnDecClaimFn = function<(fresh : bool) : int64> typedef MoeGpuAttnDecHydrateFn = function<(woq : int64; kp : uint8 const?; vp : uint8 const?; npos : int64; kvd : int64; f16 : bool) : void> @@ -1397,9 +1442,10 @@ let DAT_STALE_GEN = 1 // the session's generation is not the mirrors' - claim let DAT_HYDRATE = 2 // this generation, but the layer's rows are not [0, pos) - refill from the host cache let DAT_NEVER = 3 // no block, or pos past the mirror cap -[unused_argument(xb2p, kp, vp, woq, wok, wov, woo, fq, fk, fv, fo, xqp, xsp, cosp, rmsqp, rmskp, pos, dim, qd, kvd, hs, n_heads, kv_mul, scale, eps, neox, qk_norm)] +[unused_argument(xb2p, kp, vp, woq, wok, wov, woo, fq, fk, fv, fo, xqp, xsp, cosp, rmsqp, rmskp, bqp, bkp, bvp, pos, dim, qd, kvd, hs, n_heads, kv_mul, scale, eps, neox, qk_norm)] def private moe_gpu_unset_attn_dec(var xb2p : float?; var kp : float?; var vp : float?; woq, wok, wov, woo : int64; fq, fk, fv, fo : int; xqp : int8 const?; xsp : float const?; cosp : float const?; rmsqp : float const?; rmskp : float const?; + bqp : float const?; bkp : float const?; bvp : float const?; pos, dim, qd, kvd, hs, n_heads, kv_mul : int64; scale, eps : float; neox, qk_norm : bool) { panic("dasLLAMA: GPU decode attention hit without an installed block") } @@ -1448,17 +1494,18 @@ def public moe_gpu_attn_dec_ok(woq, pos, gen : int64) : int { return invoke(g_moe_gpu_attn_dec_ok, woq, pos, gen) } -//! One layer's decode attention on the GPU (q/k/v projections, qk-norm+rope into the layer's -//! mirror, attention over it, wo). `xb2` gets the wo row; `k`/`v` the RAW projection rows - -//! the host cache's K takes its own norm + rope from them. +//! One layer's decode attention on the GPU (q/k/v projections, bias, qk-norm+rope into the layer's +//! mirror, attention over it, wo). `xb2` gets the wo row; `k`/`v` the RAW projection rows (before the +//! bias) - the host cache row takes its own bias + norm + rope. Bias rows are this layer's, or null. def public matmul_moe_gpu_attn_dec(var xb2 : array; var k : array; var v : array; woq, wok, wov, woo : int64; fq, fk, fv, fo : int; xq : array; xs : array; cossin : array; rmsq : float const?; rmsk : float const?; + bqp : float const?; bkp : float const?; bvp : float const?; pos, dim, qd, kvd, hs, n_heads, kv_mul : int64; scale, eps : float; neox, qk_norm : bool) { unsafe { invoke(g_moe_gpu_attn_dec, addr(xb2[0]), addr(k[0]), addr(v[0]), woq, wok, wov, woo, fq, fk, fv, fo, - addr(xq[0]), addr(xs[0]), addr(cossin[0]), rmsq, rmsk, + addr(xq[0]), addr(xs[0]), addr(cossin[0]), rmsq, rmsk, bqp, bkp, bvp, pos, dim, qd, kvd, hs, n_heads, kv_mul, scale, eps, neox, qk_norm) } } @@ -1488,22 +1535,23 @@ def public matmul_moe_gpu_dn(var y : array; woq, woz, woo : int64; } } -//! One full-attention layer's whole softmax attention block on the GPU tier — see MoeGpuAttnFn -//! for the contract. y gets npos x dim wo rows; k/v get npos x kv_dim roped-k / raw-v rows. -//! cos/sin are npos x rot/2 NEOX rope rows; rms weight rows may be null when qknorm is off. +//! One full-attention layer's whole softmax attention block on the GPU tier (MoeGpuAttnFn). y gets +//! npos x dim wo rows; k/v get npos x kv_dim roped-k / biased-v rows; cos/sin are npos x rot/2 NEOX +//! rows; rms rows null when qknorm is off, bias rows null without the projection bias (32-arg cap: no kv_mul). def public matmul_moe_gpu_attn(var y : array; var k : array; var v : array; woq, wok, wov, woo : int64; fq, fk, fv, fo : int; xq : array; xs : array; rmsqp : float const?; rmskp : float const?; + bqp : float const?; bkp : float const?; bvp : float const?; cosp : float const?; sinp : float const?; - npos, dim, qd, kv_dim, hs, rot, n_heads, kv_mul : int64; + npos, dim, qd, kv_dim, hs, rot, n_heads : int64; scale : float; eps : float; gated : bool; qknorm : bool) { unsafe { invoke(g_moe_gpu_attn, addr(y[0]), addr(k[0]), addr(v[0]), woq, wok, wov, woo, fq, fk, fv, fo, addr(xq[0]), addr(xs[0]), - rmsqp, rmskp, cosp, sinp, - npos, dim, qd, kv_dim, hs, rot, n_heads, kv_mul, scale, eps, gated, qknorm) + rmsqp, rmskp, bqp, bkp, bvp, cosp, sinp, + npos, dim, qd, kv_dim, hs, rot, n_heads, scale, eps, gated, qknorm) } } @@ -1634,6 +1682,15 @@ def private restore_gpu_tier_fn_defaults { // nolint:STYLE037,STYLE038 — flat if (g_moe_gpu_budget_report == null) { g_moe_gpu_budget_report = @@moe_gpu_no_budget } + if (g_moe_gpu_os_memory == null) { + g_moe_gpu_os_memory = @@moe_gpu_no_os_memory + } + if (g_rdec_note_weight_bytes == null) { + g_rdec_note_weight_bytes = @@rdec_no_note_weight_bytes + } + if (g_rdec_upload_bias == null) { + g_rdec_upload_bias = @@rdec_unset_upload_bias + } if (g_moe_gpu_binding_cap == null) { g_moe_gpu_binding_cap = @@moe_gpu_no_binding_cap } diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index ae6c997a64..17e5aff3ed 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 33 // an untied Q8 load keeps its Q8_0 token table as quants (emb_q8): the fp32 copy left fblob, a linear q8 copy joined qblob +let IMAGE_VERSION = 35 // the vulkan flavor's iq4xs/iq3s/iq3xxs/iq2xxs scale planes are two-word rows; a BPE vocab without add_bos takes upstream's per-pre default //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -1650,6 +1650,7 @@ def load_model_cached(path : string; mode : QuantMode = QuantMode.fp32; var jobs : array var streamed = Model() var served = false + var saved = false let ts_s = ref_time_ticks() let save = image_save_enabled() let ran = load_gguf_streaming(path, mode, mt, jobs, want_metal) $(gm : GGUFMeta; gbytes : array) { @@ -1657,13 +1658,18 @@ def load_model_cached(path : string; mode : QuantMode = QuantMode.fp32; if (want_metal && !mt.metal_blob) { return false } - served = (save ? save_model_image_streaming(mt, gm, gbytes, jobs, s_img, s_tag, s_quant) - : image_from_model_streaming(mt, streamed, gm, gbytes, jobs, "", s_tag, s_quant)) + if (save) { + saved = save_model_image_streaming(mt, gm, gbytes, jobs, s_img, s_tag, s_quant) + if (!saved) { + to_log(LOG_WARNING, "dasLLAMA: the image save for '{path}' declined - serving the streamed build from memory\n") + } + } + served = saved || image_from_model_streaming(mt, streamed, gm, gbytes, jobs, "", s_tag, s_quant) return served } delete jobs delete mt // the streamed planes never became model planes; the rest goes now - if (served && save) { + if (saved) { to_log(LOG_INFO, "dasLLAMA: prepared image streamed in {get_time_usec(ts_s) / 1000} ms - {s_img}\n") if (!load_image(s_img, streamed, s_tag, s_quant)) { panic("dasLLAMA: '{s_img}' was just streamed out and will not map back - see the decline above") diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index a16580079a..56f6eed6a9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -918,19 +918,44 @@ def q8_region_to_metal_blob(qp : int8 const?; sp : float const?; n, rows : int64 } } +//! The codebook formats' two-word device scale row: word 0 = the plane row's f16 d with iq4xs's +//! scales_h above it, word 1 = the eight 4-bit fields (scales_l, or s) packed from the row's eight +//! decoded sub-scale bytes (iq4xs's ls - 32 or the iq3 pair's 1 + 2s). +def private pack_iq_dev_row(fmt : KqFmt; var out : uint8?; d : uint8 const?; sc : uint8 const?; sstride : int64) { + unsafe { + out[0] = d[0] + out[1] = d[1] + var hi = 0u + var lo = 0u + for (ib in range64(8l)) { + let v = int(int8(sc[ib * sstride])) + let f = uint(fmt == KqFmt.iq4xs ? v + 32 : (v - 1) >> 1) + lo |= (f & 15u) << uint(4l * ib) + hi |= ((f >> 4u) & 3u) << uint(2l * ib) + } + out[2] = uint8(hi & 0xFFu) + out[3] = uint8(hi >> 8u) + for (b in range64(4l)) { + out[4l + b] = uint8((lo >> uint(8l * b)) & 0xFFu) + } + } +} + // Gather one PREPARED K-quant weight stack into the GPU tier's device planes — quant payloads -// row-major per superblock in the k/k+16 nibble pairing, plus DECODED 20B scale rows. Reads -// whatever the load prepared (grp interleave or disk-order); device bytes are identical either way. +// row-major per superblock in the k/k+16 nibble pairing, plus the device scale rows (decoded 20 B, or +// the codebook formats' two words); grp interleave or disk-order in, identical device bytes out. [arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice_rows : int64; repacked : bool; mr : int64; var wq : array; var ws : array) { let nsb = n / KQ_SUPERBLOCK_ELEMS let qsb = kq_qsb(fmt) // quant plane stride per superblock (disk == device for quants) - let dssb = kq_ssb(fmt) // disk/plane scale stride; device rows decode to KQ_DEV_SSB + let dssb = kq_ssb(fmt) // disk/plane scale stride; device rows decode to kq_dev_ssb(fmt) + let dev_ssb = kq_dev_ssb(fmt) + let two_word_row = kq_two_word_row(fmt) assert(n % KQ_SUPERBLOCK_ELEMS == 0l, "GPU MoE kq gather: row length must be a superblock multiple") assert(rows * nsb * qsb <= 2147483647l, "GPU MoE kq stack exceeds the int32 gather-buffer rail") wq |> grow_resize(rows * nsb * qsb) - ws |> grow_resize(rows * nsb * KQ_DEV_SSB) + ws |> grow_resize(rows * nsb * dev_ssb) let nslices = int(rows / slice_rows) let qrow = nsb * qsb let srow = nsb * dssb @@ -964,7 +989,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice let r = ri % mr for (sbi in range64(nsb)) { var dq = wqp + rr * qrow + sbi * qsb - var dsc = wsp + (rr * nsb + sbi) * KQ_DEV_SSB + var dsc = wsp + (rr * nsb + sbi) * dev_ssb if (grouped) { let gq = qp + sliceQ + g * mr * qrow + sbi * qsb * mr let gs = sp + sliceS + g * mr * srow + sbi * dssb * mr @@ -1055,6 +1080,8 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (idx in range64(16l)) { dsc[4l + idx] = gs[idx * mr + r] } + } elif (two_word_row) { // the k4-shaped grp row (d at 2r, the eight sub-scale bytes mr-strided from 4mr) packs to two words + pack_iq_dev_row(fmt, dsc, gs + 2l * r, gs + 4l * mr + r, mr) } else { dsc[0] = gs[2l * r] dsc[1] = gs[2l * r + 1l] @@ -1095,7 +1122,9 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { // rows already in the decoded device form + if (two_word_row) { // the plane row (d, then the eight decoded sub-scale bytes at 4) packs to two words + pack_iq_dev_row(fmt, dsc, rs, rs + 4l, 1l) + } elif (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.k3 || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 6e66bea6bf..e048cd0c25 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -601,6 +601,89 @@ def conv_profile_report() { delete rows } +//! The form a converted tensor is served in: a kq plane, the q8 rail's quants and scales, the +//! q4_0 rail's nibbles and scales, or f32 rows. +enum private ConvServed { + kq + q8 + q40 + f32 +} + +struct private ConvBytes { + tensors : int64 + disk : int64 + elems : int64 + fmt : KqFmt + served : ConvServed +} + +struct private ConvReportRow { + name : string + tensors : int64 + disk_bytes : int64 + served_bytes : int64 +} + +var private g_conv_bytes : table +var private g_load_demotions = 0l + +def private conv_reset() { + g_conv_bytes |> clear() + g_load_demotions = 0l +} + +def private conv_account(m : GGUFMeta; name, kind : string; served : ConvServed; fmt : KqFmt; n : int64) { + let gt = gguf_tensor_type(m, name) + let key = "{ggml_type_name(gt)} -> {kind}" + var b = g_conv_bytes[key] + b.tensors++ + b.disk += ggml_type_bytes(gt, n) + b.elems += n + b.fmt = fmt + b.served = served + g_conv_bytes[key] = b +} + +//! Bytes the served form of `n` converted elements takes: kq planes by their stride sizes, q8 as +//! the quants plus the scale plane the load leaves (f16 after the wscale pass, else f32), q4_0 as +//! nibbles plus f32 scales, everything else as f32. +def private served_bytes(t : Model; served : ConvServed; fmt : KqFmt; n : int64) : int64 { + if (served == ConvServed.kq) { + return n / kq_elems(fmt) * (kq_qsb(fmt) + kq_ssb(fmt)) + } + if (served == ConvServed.q8) { + return n + n / 32l * (t.wscale_f16 ? 2l : 4l) + } + if (served == ConvServed.q40) { + return n / 2l + n / 32l * 4l + } + return n * 4l +} + +//! The load report: the file's bytes, then every (disk type -> conversion kind) as bytes on disk +//! against bytes served with the delta, biggest overhead first, and the q8-rail demotion count. +def private log_conversion_report(t : Model; file_bytes : int64) { + if (empty(g_conv_bytes)) { + return + } + var rows : array + var disk_total = 0l + var served_total = 0l + for (k, v in keys(g_conv_bytes), values(g_conv_bytes)) { + let s = served_bytes(t, v.served, v.fmt, v.elems) + rows |> emplace(ConvReportRow(name = clone_string(k), tensors = v.tensors, disk_bytes = v.disk, served_bytes = s)) + disk_total += v.disk + served_total += s + } + rows |> sort() $(a, b) => a.served_bytes - a.disk_bytes > b.served_bytes - b.disk_bytes + to_log(LOG_INFO, "dasLLAMA load: {file_bytes >> 20l} MB on disk; converted weights {disk_total >> 20l} MB -> {served_total >> 20l} MB served ({mb_delta(served_total, disk_total)}); {g_load_demotions} q8-rail demotion(s)\n") + for (r in rows) { + to_log(LOG_INFO, "dasLLAMA load: {r.name}: {r.disk_bytes >> 20l} MB -> {r.served_bytes >> 20l} MB ({mb_delta(r.served_bytes, r.disk_bytes)}) over {r.tensors} tensor(s)\n") + } + delete rows +} + // ===== streaming plan/fill (phase D of the .dlim arc) ===== //! One recorded big-tensor conversion: load_big's call args, replayable — the fill pass @@ -821,9 +904,11 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie temp_q |> grow_resize(int(sh.qb)) let t0 = g_conv_prof ? ref_time_ticks() : 0l var kind = "" + var served = ConvServed.q8 if (kq_sb(j.fmt) || j.fmt == KqFmt.q51) { temp_s |> grow_resize(int(sh.sb)) kind = transcode_kq_tensor(m, bytes, j.name, j.fmt, temp_q, temp_s, 0l, j.n, j.src_off, scratch) + served = ConvServed.kq } else { // q8 family: scales are FLOATS; the epilogue converts the whole plane at the end temp_q8 |> grow_resize(int(j.n)) @@ -831,6 +916,7 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie kind = transcode_q8_tensor(m, bytes, j.name, temp_q8, temp_sf, 0l, 0l, j.n, j.src_off, scratch) } conv_note(kind, t0, j.n) + conv_account(m, j.name, kind, served, j.fmt, j.n) // land the scale half at its real offset (RAM-resident plane) unsafe { if (j.fmt == KqFmt.k4) { @@ -1230,6 +1316,19 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var gguf_read_tensor_f32(m, bytes, name, t.wblob, woff, n, src_off) } conv_note(kind, t0, n) + conv_account(m, name, kind, big_served_form(t, fmt), fmt, n) +} + +//! The form load_big's ladder serves a tensor in: a kq plane by its format, else the model's q8 or +//! q4_0 rail, else f32 rows. +def private big_served_form(t : Model; fmt : KqFmt) : ConvServed { + if (fmt != KqFmt.q8) { + return ConvServed.kq + } + if (t.quant == QuantMode.q8) { + return ConvServed.q8 + } + return t.quant == QuantMode.q4_0 ? ConvServed.q40 : ConvServed.f32 } // qwen3next-layout deltanet big weights: same planes as the qwen35 arm, but the file's v-indexed @@ -1366,6 +1465,29 @@ def private kq_fmt_of(gt : int) : KqFmt { } +//! One weight's plane tag off its disk type: the K-quant form when the dense rails serve it at +//! this row length, else the q8 rail - a demotion is logged and counted for the load report. +def private kq_tag(m : GGUFMeta; name : string; rows : int64) : KqFmt { + let disk_fmt = kq_fmt_of(gguf_tensor_type(m, name)) + let why = disk_fmt == KqFmt.q51 ? "the dense rails carry no q51 form" : "a {rows}-wide row is no multiple of 256" + return note_demotion(name, disk_fmt, kq_fmt_row_ok(disk_fmt, rows), why) +} + +//! The expert-stack twin of kq_tag (q51 serves natively off per-32 planes there). +def private kq_tag_expert(m : GGUFMeta; name : string; rows : int64) : KqFmt { + let disk_fmt = kq_fmt_of(gguf_tensor_type(m, name)) + let why = disk_fmt == KqFmt.q51 ? "a {rows}-wide row is no multiple of 32" : "a {rows}-wide row is no multiple of 256" + return note_demotion(name, disk_fmt, kq_fmt_expert_ok(disk_fmt, rows), why) +} + +def private note_demotion(name : string; disk_fmt, tag_fmt : KqFmt; why : string) : KqFmt { + if (tag_fmt != disk_fmt) { + g_load_demotions++ + to_log(LOG_WARNING, "dasLLAMA load: {name} ({disk_fmt} on disk) serves through the q8 rail - {why}\n") + } + return tag_fmt +} + // Fill the per-kind weight format tags from the file's tensor types (q8-mode loads with the // kquant_native knob on). Fused-projection arches (Phi3) split attn_qkv/ffn_up at row-aligned // slice offsets, so every slice stays superblock-aligned; one disk tensor = one tag for its slices. @@ -1387,22 +1509,22 @@ def private detect_kq_formats(var t : Model; m : GGUFMeta) { // nolint:STYLE03 // q40 tags additionally demote per tensor when the row length isn't % 256 — Q4_0 only guarantees % 32 for (l in range64(layers)) { if (t.config.fused_qkv) { - let fqkv = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.attn_qkv.weight")), t.config.dim) + let fqkv = kq_tag(m, "blk.{l}.attn_qkv.weight", t.config.dim) t.wq_fmt[l] = fqkv t.wk_fmt[l] = fqkv t.wv_fmt[l] = fqkv - let fgu = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ffn_up.weight")), t.config.dim) + let fgu = kq_tag(m, "blk.{l}.ffn_up.weight", t.config.dim) t.w1_fmt[l] = fgu t.w3_fmt[l] = fgu } else { - t.wq_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.attn_q.weight")), t.config.dim) - t.wk_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.attn_k.weight")), t.config.dim) - t.wv_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.attn_v.weight")), t.config.dim) - t.w1_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ffn_gate.weight")), t.config.dim) - t.w3_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ffn_up.weight")), t.config.dim) - } - t.wo_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.attn_output.weight")), layer_qd(t.config, l)) - t.w2_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ffn_down.weight")), layer_hidden(t, l)) + t.wq_fmt[l] = kq_tag(m, "blk.{l}.attn_q.weight", t.config.dim) + t.wk_fmt[l] = kq_tag(m, "blk.{l}.attn_k.weight", t.config.dim) + t.wv_fmt[l] = kq_tag(m, "blk.{l}.attn_v.weight", t.config.dim) + t.w1_fmt[l] = kq_tag(m, "blk.{l}.ffn_gate.weight", t.config.dim) + t.w3_fmt[l] = kq_tag(m, "blk.{l}.ffn_up.weight", t.config.dim) + } + t.wo_fmt[l] = kq_tag(m, "blk.{l}.attn_output.weight", layer_qd(t.config, l)) + t.w2_fmt[l] = kq_tag(m, "blk.{l}.ffn_down.weight", layer_hidden(t, l)) any ||= t.wq_fmt[l] != KqFmt.q8 || t.wk_fmt[l] != KqFmt.q8 || t.wv_fmt[l] != KqFmt.q8 any ||= t.wo_fmt[l] != KqFmt.q8 || t.w1_fmt[l] != KqFmt.q8 || t.w2_fmt[l] != KqFmt.q8 any ||= t.w3_fmt[l] != KqFmt.q8 @@ -1416,26 +1538,26 @@ def private detect_kq_formats(var t : Model; m : GGUFMeta) { // nolint:STYLE03 let fused_gu = t.config.moe_dense_shexp // gemma4: experts ship fused as ffn_gate_up_exps for (l in range64(layers)) { if (fused_gu) { - let f = kq_fmt_expert_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ffn_gate_up_exps.weight")), t.config.dim) + let f = kq_tag_expert(m, "blk.{l}.ffn_gate_up_exps.weight", t.config.dim) t.we1_fmt[l] = f t.we3_fmt[l] = f } else { - t.we1_fmt[l] = kq_fmt_expert_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ffn_gate_exps.weight")), t.config.dim) - t.we3_fmt[l] = kq_fmt_expert_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ffn_up_exps.weight")), t.config.dim) + t.we1_fmt[l] = kq_tag_expert(m, "blk.{l}.ffn_gate_exps.weight", t.config.dim) + t.we3_fmt[l] = kq_tag_expert(m, "blk.{l}.ffn_up_exps.weight", t.config.dim) } - t.we2_fmt[l] = kq_fmt_expert_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ffn_down_exps.weight")), t.config.n_ff_exp) + t.we2_fmt[l] = kq_tag_expert(m, "blk.{l}.ffn_down_exps.weight", t.config.n_ff_exp) any ||= t.we1_fmt[l] != KqFmt.q8 || t.we2_fmt[l] != KqFmt.q8 || t.we3_fmt[l] != KqFmt.q8 } } if (t.config.shared_weights) { - t.emb_fmt = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "token_embd.weight")), t.config.dim) + t.emb_fmt = kq_tag(m, "token_embd.weight", t.config.dim) any ||= t.emb_fmt != KqFmt.q8 } else { - t.wcls_fmt = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "output.weight")), t.config.dim) + t.wcls_fmt = kq_tag(m, "output.weight", t.config.dim) any ||= t.wcls_fmt != KqFmt.q8 } if (t.config.n_layer_nextn > 0l) { - t.mtp_ehproj_fmt = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{t.config.n_layers}.nextn.eh_proj.weight")), 2l * t.config.dim) + t.mtp_ehproj_fmt = kq_tag(m, "blk.{t.config.n_layers}.nextn.eh_proj.weight", 2l * t.config.dim) any ||= t.mtp_ehproj_fmt != KqFmt.q8 } let dn_native_driver = gpu_want_dn_native() && t.config.n_expert == 0l @@ -1447,9 +1569,9 @@ def private detect_kq_formats(var t : Model; m : GGUFMeta) { // nolint:STYLE03 t.dnout_fmt |> resize(layers) for (l in range64(layers)) { if (layer_is_recurrent(t.config, l)) { - t.dnqkv_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.attn_qkv.weight")), t.config.dim) - t.dngate_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.attn_gate.weight")), t.config.dim) - t.dnout_fmt[l] = kq_fmt_row_ok(kq_fmt_of(gguf_tensor_type(m, "blk.{l}.ssm_out.weight")), t.config.ssm_d_inner) + t.dnqkv_fmt[l] = kq_tag(m, "blk.{l}.attn_qkv.weight", t.config.dim) + t.dngate_fmt[l] = kq_tag(m, "blk.{l}.attn_gate.weight", t.config.dim) + t.dnout_fmt[l] = kq_tag(m, "blk.{l}.ssm_out.weight", t.config.ssm_d_inner) any ||= t.dnqkv_fmt[l] != KqFmt.q8 || t.dngate_fmt[l] != KqFmt.q8 || t.dnout_fmt[l] != KqFmt.q8 } } @@ -1510,6 +1632,7 @@ def load_gguf_streaming(path : string; mode : QuantMode; var t : Model; var jobs g_stream_want_blob = want_blob bump_weights_epoch() t.quant = mode + conv_reset() if (g_env_engine.conv_prof) { set_conv_profile(true) } @@ -1589,6 +1712,7 @@ def private load_gguf_impl(path : string; var mode : QuantMode) : Model { bump_weights_epoch() var t = Model() t.quant = mode + conv_reset() if (g_env_engine.conv_prof) { set_conv_profile(true) } @@ -2345,6 +2469,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | // weights below if (!t.emb_q8 && !cls_kq(t)) { gguf_read_tensor_f32(m, bytes, "token_embd.weight", t.fblob, t.tok_emb_off, vocab * dim) + conv_account(m, "token_embd.weight", "f32 token table", ConvServed.f32, KqFmt.q8, vocab * dim) } // gpt-oss names its pre-FFN norm post_attention_norm (same graph role as ffn_norm) let ffn_norm_name = t.config.post_attn_is_ffn_norm ? "post_attention_norm" : "ffn_norm" @@ -2679,6 +2804,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | } } to_log(LOG_INFO, "dasLLAMA: load stage wscale: {get_time_usec(ts_stage) / 1000} ms, total {get_time_usec(ts_load) / 1000} ms\n") + log_conversion_report(t, long_length(bytes)) log_load_report(t) if (!g_fill_plan_mode) { // streaming converts only — no serving, no gathers over unmade planes ple_check_table(t, "gguf load") diff --git a/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das b/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das index b80769484d..a1b1f3e7a8 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_vulkan.das @@ -81,6 +81,9 @@ def install_vk_moe_hooks { set_moe_gpu_attn_dec_hooks(@@vk_moe_attn_dec, @@vk_moe_attn_dec_ok, @@vk_moe_attn_dec_claim, @@vk_moe_attn_dec_hydrate) set_moe_gpu_ffn_tail_hooks(@@vk_moe_ffn_tail, @@vk_moe_ffn_tail_pending) set_moe_gpu_span_dec_hook(@@vk_moe_span_dec) + install_moe_gpu_os_memory(@@vk_os_memory_now) + install_rdec_note_weight_bytes(@@vk_rdec_note_weight_bytes) + install_moe_gpu_resident_bias(@@vk_rdec_upload_bias) } // the model drop: no chain may be in flight, the decode-side counters reset, then the tier's teardown diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index f3a86f05b5..f5544cfc90 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -12,6 +12,7 @@ require dasllama/dasllama_vulkan_common require dasllama/dasllama_vulkan_dispatch require dasllama/dasllama_kqformat require daslib/typemacro_boost // [template_struct_instance] — the KV-codec kernel templates +require daslib/math_bits // uint_bits_to_float: the KHR tile's f32 store bounce read back through its uint staging array require math // The Vulkan KERNEL home: every kernel is a [spirv_kernel]/[vk_dispatch] class — families are @@ -97,7 +98,7 @@ class ArBase : RmsWgBase { @ssbo @binding = 1 aa : array // the add partner rows @ssbo @binding = 2 wn : array // the norm weight row @push_constant pa : ArArgs - @workgroup row : float[4096] // AR_MAX_DIM — staged so the post-reduce reread is local + @workgroup row : float[8192] // AR_MAX_DIM — staged so the post-reduce reread is local // the residual add + row stash + sum-of-squares pass over one row at `base` def accum_row(base : uint) : float { @@ -137,54 +138,17 @@ class ClsArAddRms : ArBase { // ar_add_rms + Q8_0 requant in ONE dispatch: the normed row quantizes out of the row stash, so // xb never reaches memory. Verbatim reduce/amax/rounding => bit-identical to the split pair. -[vk_dispatch(name = "cls_ar_rq", grid = "wgs", params = "wgs : int64")] -class ArAddRmsRq : ArBase { - @ssbo @binding = 3 outq : array // the quantized bytes - @ssbo @binding = 4 outs : array // per-32-block scales - - [spirv_kernel(local_size_x = 256, name = "cls_ar_rq_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] - def run { - let tid = gl_LocalInvocationID.x - let ss = accum_row(0u) - let inv = rms_inv(ss) - // quantize the normed row instead of storing it: 8 lanes per 32-block, 32 blocks per pass - let nblk = pa.dim / 32u - let lane8 = tid % 8u - var b = tid / 8u - while (b < nblk) { - let base = b * 32u + lane8 * 4u - let v0 = wn[pa.woff + base] * (row[base] * inv) - let v1 = wn[pa.woff + base + 1u] * (row[base + 1u] * inv) - let v2 = wn[pa.woff + base + 2u] * (row[base + 2u] * inv) - let v3 = wn[pa.woff + base + 3u] * (row[base + 3u] * inv) - let m = q8_amax3(max(max(abs(v0), abs(v1)), max(abs(v2), abs(v3)))) - let d = m / 127.0 - let id = d != 0.0 ? 1.0 / d : 0.0 - if (lane8 == 0u) { - outs[b] = d - } - outq[base] = q8_quant1(v0, id) - outq[base + 1u] = q8_quant1(v1, id) - outq[base + 2u] = q8_quant1(v2, id) - outq[base + 3u] = q8_quant1(v3, id) - b += 32u - } - } -} - -[vk_dispatch(name = "cls_ar_rq_b", grid = "nrows", params = "nrows : int64")] -class ClsArAddRmsRqB : ArBase { +class ArRqBase : ArBase { @ssbo @binding = 3 outq : array // the quantized bytes, row-major blocks @ssbo @binding = 4 outs : array // per-32-block scales - [spirv_kernel(local_size_x = 256, name = "cls_ar_rq_b_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] - def run { + // quantize the normed row instead of storing it: 8 lanes per 32-block, 32 blocks per pass + [arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#q8-requant-byte-store")] + def rq_row(rbase, qb0 : uint) { let tid = gl_LocalInvocationID.x - let rbase = gl_WorkGroupID.x * pa.dim let ss = accum_row(rbase) let inv = rms_inv(ss) let nblk = pa.dim / 32u - let qb0 = gl_WorkGroupID.x * nblk let lane8 = tid % 8u var b = tid / 8u while (b < nblk) { @@ -209,6 +173,22 @@ class ClsArAddRmsRqB : ArBase { } } +[vk_dispatch(name = "cls_ar_rq", grid = "wgs", params = "wgs : int64")] +class ArAddRmsRq : ArRqBase { + [spirv_kernel(local_size_x = 256, name = "cls_ar_rq_spv")] + def run { + rq_row(0u, 0u) + } +} + +[vk_dispatch(name = "cls_ar_rq_b", grid = "nrows", params = "nrows : int64")] +class ClsArAddRmsRqB : ArRqBase { + [spirv_kernel(local_size_x = 256, name = "cls_ar_rq_b_spv")] + def run { + rq_row(gl_WorkGroupID.x * pa.dim, gl_WorkGroupID.x * (pa.dim / 32u)) + } +} + [vk_dispatch(name = "cls_ar_f16_b", grid = "nrows", params = "nrows : int64")] class ClsArAddRmsF16B : ArBase { @ssbo @binding = 3 outh : array // the normed rows, f16 @@ -841,7 +821,7 @@ class MoeGatherF16 { @ssbo @binding = 2 xg : array // the gathered rows, bucket order, window-relative @push_constant pa : GatherArgs - [spirv_kernel(local_size_x = 256, name = "moe_gather_f16_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-expert-chain")] + [spirv_kernel(local_size_x = 256, name = "moe_gather_f16_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-expert-chain")] def run { let r = gl_WorkGroupID.x let b = slot_idx[r] @@ -880,7 +860,7 @@ class DnConv { @ssbo @binding = 1 outr : array // conv output rows @ssbo @binding = 2 smalls : array // taps + history + any small planes (push offsets) @push_constant pa : DnConvArgs - @workgroup row : float[8192] // the whole position row, silu'd + @workgroup row : float[8192] // the position row's q/k region, silu'd: staged for the per-head norm (2 kd <= DN_MAX_QK); the v region streams through @workgroup nsc : float[64] // per-block L2 scales [spirv_kernel(local_size_x = 256, name = "dn_conv_cls_spv")] @@ -906,7 +886,12 @@ class DnConv { acc += xv * smalls[pa.taps_off + ch * pa.dconv + k] k++ } - row[ch] = acc / (1.0 + exp(-acc)) + let sv = acc / (1.0 + exp(-acc)) + if (ch < 2u * pa.kd) { + row[ch] = sv + } else { + outr[p * pa.cd + ch] = sv + } ch += 256u } barrier() @@ -924,9 +909,8 @@ class DnConv { } barrier() ch = tid - while (ch < pa.cd) { - let s = ch < 2u * pa.kd ? nsc[ch / pa.ds] : 1.0 - outr[p * pa.cd + ch] = row[ch] * s + while (ch < 2u * pa.kd) { + outr[p * pa.cd + ch] = row[ch] * nsc[ch / pa.ds] ch += 256u } } @@ -978,11 +962,12 @@ struct DnStepArgs { hist_off : uint eps : float qscale : float // the conv-ring parity rides tok[0].parity + out_f32 : uint } // the decode-step FUSED chain (one workgroup per v-head): conv preamble -> chunk-length-1 delta -// rule IN PLACE on the persistent state -> gated out-norm -> the head's Q8_0 requant. The conv -// ring is parity-double-buffered inside the smalls member. +// rule IN PLACE on the persistent state -> gated out-norm -> the head's o row. The conv ring is +// parity-double-buffered inside the smalls member. [vk_dispatch(name = "dn_step_cls", grid = "nvh", params = "nvh : int64")] class DnStepFused { @ssbo @binding = 0 proj : array // the fresh projection row (+ z rows at cd) @@ -991,6 +976,7 @@ class DnStepFused { @ssbo @binding = 3 outs : array // the head's Q8_0 scales @ssbo @binding = 4 outq : array // ... and quant words @ssbo @binding = 5 tok : array // the shared per-token block (element 0) + @ssbo @binding = 6 outf : array // the head's o row as f32 @push_constant pa : DnStepArgs @workgroup dq : float[128] @workgroup dk : float[128] @@ -1115,8 +1101,13 @@ class DnStepFused { e += 256u } barrier() - // the head's Q8_0 blocks (ds % 32 asserted host-side) - if (tid < ds / 32u) { + if (pa.out_f32 != 0u) { + e = tid + while (e < ds) { + outf[h * ds + e] = dout[e] + e += 256u + } + } elif (tid < ds / 32u) { // the head's Q8_0 blocks (ds % 32 asserted host-side) let gb = (h * ds) / 32u + tid let jb = tid * 32u var amax = 0.0 @@ -1463,16 +1454,18 @@ class DnScanP3 { // ===== prefill attention: the prep pass ===== struct AtPrepArgs { - d : uint // this pass's row width (qd for q, kvd for k) + d : uint // this pass's row width (qd for q, kvd for k/v) nh : uint // heads in this pass hs : uint - half : uint // partial-rope half (0 = no rope) + half : uint // partial-rope half (0 = no rope; the v pass runs here) flags : uint // bit 0 = gated q, bit 1 = qk-norm rms_off : uint // this pass's norm-weight row in smalls cos_off : uint // per-position cos rows in smalls sin_off : uint obase : uint // output panel's row base (0 for q, npos for k) - qsrc : uint // 1 = the q source (gating applies), 0 = k + qsrc : uint // 1 = the q source (gating applies), 0 = k/v + boff : uint // this pass's row base in the bias buffer ([q | k | v] per-head biases) + hasb : uint eps : float } @@ -1485,6 +1478,7 @@ class AtPrep { @ssbo @binding = 1 outp : array // the prepped panel rows @ssbo @binding = 2 gates : array // the gate stash (gated-q models; written only then) @ssbo @binding = 3 smalls : array // cos/sin rows + norm weights (push offsets) + @ssbo @binding = 4 bias : array // per-layer [q | k | v] projection biases (hasb), else a placeholder @push_constant pa : AtPrepArgs @workgroup row : float[8192] // one position's projection row (d <= 8192) @workgroup hsinv : float[64] // per-head inverse rms (heads <= 64) @@ -1502,10 +1496,10 @@ class AtPrep { let h = i / hs let j = i % hs if (gated != 0u) { - row[i] = src[p * 2u * pa.d + h * 2u * hs + j] + row[i] = src[p * 2u * pa.d + h * 2u * hs + j] + (pa.hasb != 0u ? bias[pa.boff + i] : 0.0) gates[p * pa.d + i] = src[p * 2u * pa.d + h * 2u * hs + hs + j] } else { - row[i] = src[p * pa.d + i] + row[i] = src[p * pa.d + i] + (pa.hasb != 0u ? bias[pa.boff + i] : 0.0) } i += 256u } @@ -1881,28 +1875,34 @@ struct FaCm2Args { qhs : uint // the q panel's per-head stride (2 x hs on a gated plane); 0 = hs } -// hs=64 (tinyllama-class). The h128 twin below differs only in the head-size-shaped tiles — -// duplication tolerated until the kernel-reification arc. +// The cm2 flash pass over the f16 K/V shadows: ONE class template over the head size. The head +// template sizes the eight coopmat typedefs: QACC/QA the q and o tiles (64 x HS), SACC/PA the +// score tiles (64 x BC), KB/VB the K^T and V steps, ONE the rowsum ones-matrix, OF the f16 store. [ |> template_struct_instance] -class template FaCm2H64T { +class template FaCm2T { @ssbo @binding = 0 qpanel : array // the prepped/roped window q rows (window-local) @ssbo @binding = 1 kh : array // f16 K shadow at absolute positions @ssbo @binding = 2 vh : array // f16 V shadow at absolute positions @ssbo @binding = 3 outp : array // attention out plane (rows x qd) @push_constant pa : FaCm2Args - @template_constant OUT16 : bool = false + @template_constant HS : uint = 64u // the head width + @template_constant BC : uint = 64u // the K/V step: 64 at hs 64, 32 above (the head-size cut: K^T, V and the softmax state at 32 columns) + @template_constant OUT16 : bool = false // the o store as f16 + @template_constant GATED : bool = false // multiply the sigmoid of the gate half of a [q | gate] head into o; qstride/qhs describe that panel and read as qd/HS when unset [spirv_kernel(local_size_x = 128), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] - def run { + def run { // nolint:STYLE038 — the verbatim flash pass, parity-locked to the shipped kernel let qtiles = (pa.rows + 63u) / 64u let h = gl_WorkGroupID.x / qtiles let q0 = (gl_WorkGroupID.x % qtiles) * 64u let kvh = h / pa.kv_mul let kvlen = pa.w0 + pa.rows + let qstride = pa.qstride == 0u ? pa.qd : pa.qstride + let qhs = pa.qhs == 0u ? HS : pa.qhs var tlq : tensorLayout2DPad tensorLayoutCreate(tlq) - tensorLayoutSetDimension(tlq, pa.rows, pa.qd) - tensorLayoutSetStride(tlq, pa.qd, 1u) + tensorLayoutSetDimension(tlq, pa.rows, qstride) + tensorLayoutSetStride(tlq, qstride, 1u) var tlk : tensorLayout2DPad tensorLayoutCreate(tlk) tensorLayoutSetDimension(tlk, kvlen, pa.kvd) @@ -1913,63 +1913,84 @@ class template FaCm2H64T { tensorLayoutSetStride(tlo, pa.qd, 1u) var tv : tensorView2Dt tensorViewCreate(tv) - var q : coopmatWgAcc_f32_64x64 - coopmatLoadTensor(q, qpanel, 0u, tlq, q0, 64u, h * 64u, 64u) + var q : QACC + coopmatLoadTensor(q, qpanel, 0u, tlq, q0, 64u, h * qhs, HS) coopmatScale(q, pa.scale) - var qf : coopmatWgA_f16_64x64 + var qf : QA coopmatConvert(qf, q) - var mrow : coopmatWgAcc_f32_64x64 + var mrow : SACC coopmatFill(mrow, -1.7e38) - var lrow : coopmatWgAcc_f32_64x64 - var o : coopmatWgAcc_f32_64x64 + var lrow : SACC + var o : QACC let kmax = pa.w0 + min(q0 + 64u, pa.rows) // causal bound (exclusive) of the tile's last row var j = 0u while (j < kmax) { - var s : coopmatWgAcc_f32_64x64 - var kt : coopmatWgB_f16_64x64 - coopmatLoadTensor(kt, kh, pa.kvbase, tlk, j, 64u, kvh * 64u, 64u, tv) + var s : SACC + var kt : KB + coopmatLoadTensor(kt, kh, pa.kvbase, tlk, j, BC, kvh * HS, HS, tv) s = coopmatMulAdd(qf, kt, s) coopmatPerElement(s, s, @@facm2_mask, pa.w0 + q0, j) - var rmax : coopmatWgAcc_f32_64x64 + var rmax : SACC coopmatReduce(rmax, s, COOPMAT_REDUCE_ROW, @@facm2_max) - var mold : coopmatWgAcc_f32_64x64 + var mold : SACC coopmatCopy(mold, mrow) coopmatPerElement(mrow, rmax, @@facm2_pick, mold) coopmatSub(s, s, mrow) coopmatPerElement(s, s, @@facm2_exp) // P = e^(S - M), in place - var em : coopmatWgAcc_f32_64x64 + var em : SACC coopmatSub(em, mold, mrow) coopmatPerElement(em, em, @@facm2_exp) // eM = e^(Mold - M) - var pf : coopmatWgA_f16_64x64 + var pf : PA coopmatConvert(pf, s) - var rsum : coopmatWgAcc_f32_64x64 - var one : coopmatWgB_f16_64x64 + var rsum : SACC + var one : ONE coopmatFill(one, 1.0) rsum = coopmatMulAdd(pf, one, rsum) // rowsum by ones-matrix multiply coopmatMulElem(lrow, em, lrow) coopmatAdd(lrow, lrow, rsum) // L = eM*L + rowsum - var emd : coopmatWgAcc_f32_64x64 + var emd : QACC coopmatReduce(emd, em, COOPMAT_REDUCE_ROW, @@facm2_smear) coopmatMulElem(o, emd, o) - var vt : coopmatWgB_f16_64x64 - coopmatLoadTensor(vt, vh, pa.kvbase, tlk, j, 64u, kvh * 64u, 64u) + var vt : VB + coopmatLoadTensor(vt, vh, pa.kvbase, tlk, j, BC, kvh * HS, HS) o = coopmatMulAdd(pf, vt, o) // O = eMdiag*O + P @ V - j += 64u + j += BC } - var ld : coopmatWgAcc_f32_64x64 + var ld : QACC coopmatReduce(ld, lrow, COOPMAT_REDUCE_ROW, @@facm2_smear) coopmatPerElement(ld, ld, @@facm2_recip) coopmatMulElem(o, ld, o) // O /= L (padding rows discard at store) + static_if (GATED) { + var g : QACC + coopmatLoadTensor(g, qpanel, 0u, tlq, q0, 64u, h * qhs + HS, HS) + coopmatPerElement(g, g, @@facm2_sigmoid) + coopmatMulElem(o, g, o) + } static_if (OUT16) { - var of : coopmatWgAcc_f16_64x64 + var of : OF coopmatConvert(of, o) - coopmatStoreTensor(of, outp, 0u, tlo, q0, 64u, h * 64u, 64u) + coopmatStoreTensor(of, outp, 0u, tlo, q0, 64u, h * HS, HS) } else { - coopmatStoreTensor(o, outp, 0u, tlo, q0, 64u, h * 64u, 64u) + coopmatStoreTensor(o, outp, 0u, tlo, q0, 64u, h * HS, HS) } } } +// hs=64 (tinyllama-class): the whole head is one K/V step +[ |> template_struct_instance] +class template FaCm2H64T : FaCm2T { + override HS = 64u + override BC = 64u + typedef QACC = coopmatWgAcc_f32_64x64 + typedef QA = coopmatWgA_f16_64x64 + typedef SACC = coopmatWgAcc_f32_64x64 + typedef PA = coopmatWgA_f16_64x64 + typedef KB = coopmatWgB_f16_64x64 + typedef VB = coopmatWgB_f16_64x64 + typedef ONE = coopmatWgB_f16_64x64 + typedef OF = coopmatWgAcc_f16_64x64 +} + [vk_dispatch(name = "fa_cm2_h64_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] class FaCm2H64 : FaCm2H64T { typedef OT = float @@ -1981,95 +2002,21 @@ class FaCm2H64F16 : FaCm2H64T { override OUT16 = true } -// hs=128 (llama-3.2-3B-class): same algorithm at Bc=32 (the head-size cut — K^T/V/softmax -// state at 32 columns, KV loop steps 32). Fast standalone (24.7 TFLOP/s, right beside the h64) and +// hs=128 (llama-3.2-3B-class) at Bc=32. Fast standalone (24.7 TFLOP/s, right beside the h64) and // WINS end-to-end when the shadows fit (3B 4287 vs 3955 no-fa) — serving is gated only on the // resident plan learning the f16 shadow cost (see the pf_facm2 gate) [ |> template_struct_instance] -class template FaCm2H128T { - @ssbo @binding = 0 qpanel : array - @ssbo @binding = 1 kh : array - @ssbo @binding = 2 vh : array - @ssbo @binding = 3 outp : array - @push_constant pa : FaCm2Args - @template_constant OUT16 : bool = false - - [spirv_kernel(local_size_x = 128), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] - def run { - let qtiles = (pa.rows + 63u) / 64u - let h = gl_WorkGroupID.x / qtiles - let q0 = (gl_WorkGroupID.x % qtiles) * 64u - let kvh = h / pa.kv_mul - let kvlen = pa.w0 + pa.rows - var tlq : tensorLayout2DPad - tensorLayoutCreate(tlq) - tensorLayoutSetDimension(tlq, pa.rows, pa.qd) - tensorLayoutSetStride(tlq, pa.qd, 1u) - var tlk : tensorLayout2DPad - tensorLayoutCreate(tlk) - tensorLayoutSetDimension(tlk, kvlen, pa.kvd) - tensorLayoutSetStride(tlk, pa.kvd, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, pa.rows, pa.qd) - tensorLayoutSetStride(tlo, pa.qd, 1u) - var tv : tensorView2Dt - tensorViewCreate(tv) - var q : coopmatWgAcc_f32_64x128 - coopmatLoadTensor(q, qpanel, 0u, tlq, q0, 64u, h * 128u, 128u) - coopmatScale(q, pa.scale) - var qf : coopmatWgA_f16_64x128 - coopmatConvert(qf, q) - var mrow : coopmatWgAcc_f32_64x32 - coopmatFill(mrow, -1.7e38) - var lrow : coopmatWgAcc_f32_64x32 - var o : coopmatWgAcc_f32_64x128 - let kmax = pa.w0 + min(q0 + 64u, pa.rows) - var j = 0u - while (j < kmax) { - var s : coopmatWgAcc_f32_64x32 - var kt : coopmatWgB_f16_128x32 - coopmatLoadTensor(kt, kh, pa.kvbase, tlk, j, 32u, kvh * 128u, 128u, tv) - s = coopmatMulAdd(qf, kt, s) - coopmatPerElement(s, s, @@facm2_mask, pa.w0 + q0, j) - var rmax : coopmatWgAcc_f32_64x32 - coopmatReduce(rmax, s, COOPMAT_REDUCE_ROW, @@facm2_max) - var mold : coopmatWgAcc_f32_64x32 - coopmatCopy(mold, mrow) - coopmatPerElement(mrow, rmax, @@facm2_pick, mold) - coopmatSub(s, s, mrow) - coopmatPerElement(s, s, @@facm2_exp) - var em : coopmatWgAcc_f32_64x32 - coopmatSub(em, mold, mrow) - coopmatPerElement(em, em, @@facm2_exp) - var pf : coopmatWgA_f16_64x32 - coopmatConvert(pf, s) - var rsum : coopmatWgAcc_f32_64x32 - var one : coopmatWgB_f16_32x32 - coopmatFill(one, 1.0) - rsum = coopmatMulAdd(pf, one, rsum) - coopmatMulElem(lrow, em, lrow) - coopmatAdd(lrow, lrow, rsum) - var emd : coopmatWgAcc_f32_64x128 - coopmatReduce(emd, em, COOPMAT_REDUCE_ROW, @@facm2_smear) - coopmatMulElem(o, emd, o) - var vt : coopmatWgB_f16_32x128 - coopmatLoadTensor(vt, vh, pa.kvbase, tlk, j, 32u, kvh * 128u, 128u) - o = coopmatMulAdd(pf, vt, o) - j += 32u - } - var ld : coopmatWgAcc_f32_64x128 - coopmatReduce(ld, lrow, COOPMAT_REDUCE_ROW, @@facm2_smear) - coopmatPerElement(ld, ld, @@facm2_recip) - coopmatMulElem(o, ld, o) - static_if (OUT16) { - var of : coopmatWgAcc_f16_64x128 - coopmatConvert(of, o) - coopmatStoreTensor(of, outp, 0u, tlo, q0, 64u, h * 128u, 128u) - } else { - coopmatStoreTensor(o, outp, 0u, tlo, q0, 64u, h * 128u, 128u) - } - } +class template FaCm2H128T : FaCm2T { + override HS = 128u + override BC = 32u + typedef QACC = coopmatWgAcc_f32_64x128 + typedef QA = coopmatWgA_f16_64x128 + typedef SACC = coopmatWgAcc_f32_64x32 + typedef PA = coopmatWgA_f16_64x32 + typedef KB = coopmatWgB_f16_128x32 + typedef VB = coopmatWgB_f16_32x128 + typedef ONE = coopmatWgB_f16_32x32 + typedef OF = coopmatWgAcc_f16_64x128 } [vk_dispatch(name = "fa_cm2_h128_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] @@ -2083,100 +2030,19 @@ class FaCm2H128F16 : FaCm2H128T { override OUT16 = true } +// hs=256 (the deltanet hybrids' attention heads) at Bc=32; the gated leaves ride the [q | gate] panel [ |> template_struct_instance] -class template FaCm2H256T { - @ssbo @binding = 0 qpanel : array - @ssbo @binding = 1 kh : array - @ssbo @binding = 2 vh : array - @ssbo @binding = 3 outp : array - @push_constant pa : FaCm2Args - @template_constant OUT16 : bool = false - @template_constant GATED : bool = false - - [spirv_kernel(local_size_x = 128), arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain")] - def run { // nolint:STYLE038 — the verbatim flash pass, parity-locked to the shipped kernel - let qtiles = (pa.rows + 63u) / 64u - let h = gl_WorkGroupID.x / qtiles - let q0 = (gl_WorkGroupID.x % qtiles) * 64u - let kvh = h / pa.kv_mul - let kvlen = pa.w0 + pa.rows - let qstride = pa.qstride == 0u ? pa.qd : pa.qstride - let qhs = pa.qhs == 0u ? 256u : pa.qhs - var tlq : tensorLayout2DPad - tensorLayoutCreate(tlq) - tensorLayoutSetDimension(tlq, pa.rows, qstride) - tensorLayoutSetStride(tlq, qstride, 1u) - var tlk : tensorLayout2DPad - tensorLayoutCreate(tlk) - tensorLayoutSetDimension(tlk, kvlen, pa.kvd) - tensorLayoutSetStride(tlk, pa.kvd, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, pa.rows, pa.qd) - tensorLayoutSetStride(tlo, pa.qd, 1u) - var tv : tensorView2Dt - tensorViewCreate(tv) - var q : coopmatWgAcc_f32_64x256 - coopmatLoadTensor(q, qpanel, 0u, tlq, q0, 64u, h * qhs, 256u) - coopmatScale(q, pa.scale) - var qf : coopmatWgA_f16_64x256 - coopmatConvert(qf, q) - var mrow : coopmatWgAcc_f32_64x32 - coopmatFill(mrow, -1.7e38) - var lrow : coopmatWgAcc_f32_64x32 - var o : coopmatWgAcc_f32_64x256 - let kmax = pa.w0 + min(q0 + 64u, pa.rows) - var j = 0u - while (j < kmax) { - var s : coopmatWgAcc_f32_64x32 - var kt : coopmatWgB_f16_256x32 - coopmatLoadTensor(kt, kh, pa.kvbase, tlk, j, 32u, kvh * 256u, 256u, tv) - s = coopmatMulAdd(qf, kt, s) - coopmatPerElement(s, s, @@facm2_mask, pa.w0 + q0, j) - var rmax : coopmatWgAcc_f32_64x32 - coopmatReduce(rmax, s, COOPMAT_REDUCE_ROW, @@facm2_max) - var mold : coopmatWgAcc_f32_64x32 - coopmatCopy(mold, mrow) - coopmatPerElement(mrow, rmax, @@facm2_pick, mold) - coopmatSub(s, s, mrow) - coopmatPerElement(s, s, @@facm2_exp) - var em : coopmatWgAcc_f32_64x32 - coopmatSub(em, mold, mrow) - coopmatPerElement(em, em, @@facm2_exp) - var pf : coopmatWgA_f16_64x32 - coopmatConvert(pf, s) - var rsum : coopmatWgAcc_f32_64x32 - var one : coopmatWgB_f16_32x32 - coopmatFill(one, 1.0) - rsum = coopmatMulAdd(pf, one, rsum) - coopmatMulElem(lrow, em, lrow) - coopmatAdd(lrow, lrow, rsum) - var emd : coopmatWgAcc_f32_64x256 - coopmatReduce(emd, em, COOPMAT_REDUCE_ROW, @@facm2_smear) - coopmatMulElem(o, emd, o) - var vt : coopmatWgB_f16_32x256 - coopmatLoadTensor(vt, vh, pa.kvbase, tlk, j, 32u, kvh * 256u, 256u) - o = coopmatMulAdd(pf, vt, o) - j += 32u - } - var ld : coopmatWgAcc_f32_64x256 - coopmatReduce(ld, lrow, COOPMAT_REDUCE_ROW, @@facm2_smear) - coopmatPerElement(ld, ld, @@facm2_recip) - coopmatMulElem(o, ld, o) - static_if (GATED) { - var g : coopmatWgAcc_f32_64x256 - coopmatLoadTensor(g, qpanel, 0u, tlq, q0, 64u, h * qhs + 256u, 256u) - coopmatPerElement(g, g, @@facm2_sigmoid) - coopmatMulElem(o, g, o) - } - static_if (OUT16) { - var of : coopmatWgAcc_f16_64x256 - coopmatConvert(of, o) - coopmatStoreTensor(of, outp, 0u, tlo, q0, 64u, h * 256u, 256u) - } else { - coopmatStoreTensor(o, outp, 0u, tlo, q0, 64u, h * 256u, 256u) - } - } +class template FaCm2H256T : FaCm2T { + override HS = 256u + override BC = 32u + typedef QACC = coopmatWgAcc_f32_64x256 + typedef QA = coopmatWgA_f16_64x256 + typedef SACC = coopmatWgAcc_f32_64x32 + typedef PA = coopmatWgA_f16_64x32 + typedef KB = coopmatWgB_f16_256x32 + typedef VB = coopmatWgB_f16_32x256 + typedef ONE = coopmatWgB_f16_32x32 + typedef OF = coopmatWgAcc_f16_64x256 } [vk_dispatch(name = "fa_cm2_h256_cls", kernel = "run", family = "fa_cm2_cls", grid = "wgs", params = "wgs : int64")] @@ -2589,30 +2455,34 @@ struct BatchArgs { ksplit : uint // cm2 split-k: 256-aligned k chunk per partial plane; 0 = no split } +// the MoE region rail every batch tile shares: sched = 4-word region records [wblk0, row0, cnt, +// wg0] at 0, then the per-wg map at pa.map_off; region_rec = this workgroup's record - x = weight +// base, y = activation row base, z = row count, w = the wg's tile index rebased to the region +class MoeCmBase { + @ssbo @binding = 2 sched : array // region records + per-wg map (bulk schedule data) + @push_constant pa : BatchArgs + + def region_rec : uint4 { + let rid = sched[pa.map_off + gl_WorkGroupID.x] + let rb = rid * 4u + return uint4(sched[rb], sched[rb + 1u], sched[rb + 2u], gl_WorkGroupID.x - sched[rb + 3u]) + } +} + // 32x32 tiles through shared so weight rows stream once per 32 rows; K advances 8 blocks per -// barrier pair. sched = 4-word region records [wblk0, row0, cnt, wg0] then the wg -> region map. +// barrier pair. [vk_dispatch(name = "q8_batch_cls", grid = "wgs", params = "wgs : int64")] -class Q8Batch { +class Q8Batch : MoeCmBase { @ssbo @binding = 0 wq : array // weight quant words @ssbo @binding = 1 wsh : array // per-block weight scales - @ssbo @binding = 2 sched : array // region records + per-wg map (bulk schedule data) @ssbo @binding = 3 xqw : array // activation quant words @ssbo @binding = 4 axs : array // per-block activation scales @ssbo @binding = 5 y : array - @push_constant pa : BatchArgs @workgroup xw4 : uint4[544] // x tile: 32 rows x 16 uint4 chunks, stride 17 (bank-spread) @workgroup ww4 : uint4[544] // w tile: 32 cols x 16 chunks, stride 17 @workgroup xsw : float[256] // x scales per (block, row) @workgroup wsw : float[256] // w scales per (block, col) - // this workgroup's region record: x = weight base, y = activation row base, z = row count, - // w = the wg's tile index rebased to the region - def region_rec : uint4 { - let rid = sched[pa.map_off + gl_WorkGroupID.x] - let rb = rid * 4u - return uint4(sched[rb], sched[rb + 1u], sched[rb + 2u], gl_WorkGroupID.x - sched[rb + 3u]) - } - [spirv_kernel(local_size_x = 256, name = "q8_batch_cls_spv")] def run { // nolint:STYLE038 — the verbatim tile pass (stage/reduce/store), parity-locked to the shipped kernel let reg = region_rec() @@ -3331,32 +3201,36 @@ def private iq4_word(n : uint) : uint { return b0 | (b1 << 8u) | (b2 << 16u) | (b3 << 24u) } -// the iq4xs scale row's sub-scale for block blk: word 1..2 of the 5-word row, SIGNED (ls - 32) -def private iq4_sc(w : uint; blk : uint) : int { - let s4 = int4(unpack8(int(w))) - return blk % 4u == 0u ? s4.x : (blk % 4u == 1u ? s4.y : (blk % 4u == 2u ? s4.z : s4.w)) +[arch(at="../ARCHITECTURE_GPU_QUANT_PLANES.md#metal-kq-split-scale-plane")] +def iq4xs_sc(w0, w1, blk : uint) : int { + let ls = ((w1 >> (4u * blk)) & 15u) | (((w0 >> (16u + 2u * blk)) & 3u) << 4u) + return int(ls) - 32 +} + +[arch(at="../ARCHITECTURE_GPU_QUANT_PLANES.md#metal-kq-split-scale-plane")] +def grid_sc(w1, blk : uint) : int { + return 1 + 2 * int((w1 >> (4u * blk)) & 15u) +} + +//! sign-apply by xor-and-add: the magnitudes are odd, so no byte carries into the next +def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 } -class KqBatchBase { +class KqBatchBase : MoeCmBase { @ssbo @binding = 0 wq : array // weight quant planes (per-format block layout) @ssbo @binding = 1 wsu : array // scale planes (5-word superblock layout) - @ssbo @binding = 2 sched : array // region records + per-wg map @ssbo @binding = 3 xqw : array // Q8_K activation words (64 per row-superblock) @ssbo @binding = 4 axs : array // one Q8_K d per (row, superblock) @ssbo @binding = 5 y : array - @push_constant pa : BatchArgs @workgroup xw4 : uint4[544] @workgroup ww4 : uint4[544] @workgroup xsw : float[256] @workgroup wsw : float[256] // first per-(block, col) scale plane @workgroup wsw2 : float[256] // second plane (the d/min pair's min side; k6's hi sub-scale) - def region_rec : uint4 { - let rid = sched[pa.map_off + gl_WorkGroupID.x] - let rb = rid * 4u - return uint4(sched[rb], sched[rb + 1u], sched[rb + 2u], gl_WorkGroupID.x - sched[rb + 3u]) - } - // x tile stage: this thread's two uint4 chunks of the Q8_K activation rows def load_x(row0, xrow, cnt, nsb, s, word, grp : uint) { for (ki in range(2)) { @@ -3581,7 +3455,8 @@ class KqBatchIq4xs : KqBatchBase { var sa = 0.0 if (wt * 32u + slane < pa.d) { let wsb = wsb0 + (wt * 32u + slane) * nsb + s - sa = unpackHalf2x16(wsu[wsb * 5u]).x * float(iq4_sc(wsu[wsb * 5u + 1u + sblk / 4u], sblk)) + let w0 = wsu[wsb * 2u] + sa = unpackHalf2x16(w0).x * float(iq4xs_sc(w0, wsu[wsb * 2u + 1u], sblk)) } wsw[sblk * 32u + slane] = sa } @@ -3600,20 +3475,27 @@ class KqBatchIq4xs : KqBatchBase { } } +// the grid formats' scale stage over the iq3 row: d x the (1 + 2s) fields, not iq4xs's ls - 32; +// iq4xs's fold (no bsum term) inherits. iq3s, iq3xxs and iq2xxs stage their grids and gather on it +class KqBatchGridBase : KqBatchIq4xs { + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + sa = unpackHalf2x16(wsu[wsb * 2u]).x * float(grid_sc(wsu[wsb * 2u + 1u], sblk)) + } + wsw[sblk * 32u + slane] = sa + } +} + // IQ3_S tile: the iq4xs staging with the GRID GATHER in stage_w - each staged uint4 is four -// grid words sign-applied (mask = ((nibble * 0x00204081) & 0x01010101) * 255; magnitudes odd -// 1..15, no cross-byte carry); scale row and blk_fma are iq4xs's verbatim (inherited). The 2 KB -// grid stages into workgroup memory once, before the tile loop. +// grid words sign-applied. The 2 KB grid stages into workgroup memory once, before the tile loop. [vk_dispatch(name = "kq_batch_iq3s_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] -class KqBatchIq3s : KqBatchIq4xs { +class KqBatchIq3s : KqBatchGridBase { @workgroup gridw : uint[512] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { for (ki in range(2)) { let k = word + uint(ki) * 8u @@ -3650,18 +3532,12 @@ class KqBatchIq3s : KqBatchIq4xs { } // IQ3_XXS tile: the iq3s staging with the halved-grid gather - bare 8-bit indices (no ninth -// bit), signs from the aux32's four 7-bit ksigns indices via the parity helper (no table); -// scale row and blk_fma are iq4xs's verbatim (inherited). The 1 KB grid stages once. +// bit), signs from the aux32's four 7-bit ksigns indices via the parity helper (no table). +// The 1 KB grid stages once. [vk_dispatch(name = "kq_batch_iq3xxs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] -class KqBatchIq3xxs : KqBatchIq4xs { +class KqBatchIq3xxs : KqBatchGridBase { @workgroup gridw : uint[256] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { for (ki in range(2)) { let k = word + uint(ki) * 8u @@ -3692,19 +3568,13 @@ class KqBatchIq3xxs : KqBatchIq4xs { } // IQ2_XXS tile: the iq3xxs shell over the two-word u64 grid - block b's grid word (column 2b) -// carries four BYTE indices, its aux32 (column 2b+1) the parity-derived ksigns; the per-32 -// strip plane and fma inherit from KqBatchIq4xs (the (2ls+1) strips read exactly under the -// signed extract, staying below 128). +// carries four BYTE indices, its aux32 (column 2b+1) the parity-derived ksigns; the iq3 strip +// stage and iq4xs's fma inherit (the (2ls+1) strips read exactly under the signed extract, +// staying below 128). [vk_dispatch(name = "kq_batch_iq2xxs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] -class KqBatchIq2xxs : KqBatchIq4xs { +class KqBatchIq2xxs : KqBatchGridBase { @workgroup gridw : uint[512] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { for (ki in range(2)) { let k = word + uint(ki) * 8u @@ -3854,8 +3724,12 @@ class KqBatchK6 : KqBatchBase { wsw2[sblk * 32u + slane] = sbv } + //! the quant offset the split-half fold subtracts through each half's block sum + def q_off : int => 32 + // the split-half FMA: each 16-weight half keeps its own dot, sum, and signed sub-scale def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let off = q_off() let kL = bu * 2u let kH = bu * 2u + 1u let xwL = xw4[grp * 17u + kL] @@ -3878,10 +3752,10 @@ class KqBatchK6 : KqBatchBase { let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) - return float4(xscl * (wsw[bu * 32u + word] * float(l0 - 32 * blo) + wsw2[bu * 32u + word] * float(h0 - 32 * bhi)), - xscl * (wsw[bu * 32u + word + 8u] * float(l1 - 32 * blo) + wsw2[bu * 32u + word + 8u] * float(h1 - 32 * bhi)), - xscl * (wsw[bu * 32u + word + 16u] * float(l2 - 32 * blo) + wsw2[bu * 32u + word + 16u] * float(h2 - 32 * bhi)), - xscl * (wsw[bu * 32u + word + 24u] * float(l3 - 32 * blo) + wsw2[bu * 32u + word + 24u] * float(h3 - 32 * bhi))) + return float4(xscl * (wsw[bu * 32u + word] * float(l0 - off * blo) + wsw2[bu * 32u + word] * float(h0 - off * bhi)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1 - off * blo) + wsw2[bu * 32u + word + 8u] * float(h1 - off * bhi)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2 - off * blo) + wsw2[bu * 32u + word + 16u] * float(h2 - off * bhi)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3 - off * blo) + wsw2[bu * 32u + word + 24u] * float(h3 - off * bhi))) } [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cls_spv")] @@ -3915,34 +3789,7 @@ class KqBatchK3 : KqBatchK6 { } } - def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { - let kL = bu * 2u - let kH = bu * 2u + 1u - let xwL = xw4[grp * 17u + kL] - let xwH = xw4[grp * 17u + kH] - let blo = sdot4(xwL.x, 0x01010101) + sdot4(xwL.y, 0x01010101) + sdot4(xwL.z, 0x01010101) + sdot4(xwL.w, 0x01010101) - let bhi = sdot4(xwH.x, 0x01010101) + sdot4(xwH.y, 0x01010101) + sdot4(xwH.z, 0x01010101) + sdot4(xwH.w, 0x01010101) - let wL0 = ww4[word * 17u + kL] - let wL1 = ww4[(word + 8u) * 17u + kL] - let wL2 = ww4[(word + 16u) * 17u + kL] - let wL3 = ww4[(word + 24u) * 17u + kL] - let wH0 = ww4[word * 17u + kH] - let wH1 = ww4[(word + 8u) * 17u + kH] - let wH2 = ww4[(word + 16u) * 17u + kH] - let wH3 = ww4[(word + 24u) * 17u + kH] - let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) - let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) - let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) - let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) - let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) - let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) - let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) - let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) - return float4(xscl * (wsw[bu * 32u + word] * float(l0 - 4 * blo) + wsw2[bu * 32u + word] * float(h0 - 4 * bhi)), - xscl * (wsw[bu * 32u + word + 8u] * float(l1 - 4 * blo) + wsw2[bu * 32u + word + 8u] * float(h1 - 4 * bhi)), - xscl * (wsw[bu * 32u + word + 16u] * float(l2 - 4 * blo) + wsw2[bu * 32u + word + 16u] * float(h2 - 4 * bhi)), - xscl * (wsw[bu * 32u + word + 24u] * float(l3 - 4 * blo) + wsw2[bu * 32u + word + 24u] * float(h3 - 4 * bhi))) - } + def override q_off : int => 4 [spirv_kernel(local_size_x = 256, name = "kq_batch_k3_cls_spv")] def override run { @@ -4034,43 +3881,9 @@ class KqBatchK2 : KqBatchK6 { } } -// IQ2_S tile: the iq3s grid gather widened to the u64 grid (a 10-bit qs|qh index picks TWO -// words - eight magnitudes) with the block's own sign bytes; the per-16 UNSIGNED strips ride -// k6's wsw/wsw2 planes with a plain split fma (no offset, no min term). The 8 KB grid stages -// once, 8 words per thread. -[vk_dispatch(name = "kq_batch_iq2s_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] -class KqBatchIq2s : KqBatchK6 { - @workgroup gridw : uint[2048] - - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - - def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { - for (ki in range(2)) { - let k = word + uint(ki) * 8u - let bu = k / 2u - let hh = k % 2u - var wv = uint4(0u) - if (wcol < pa.d) { - let wsb = wsb0 + wcol * nsb + s - let qsw = wq[wsb * 18u + bu] - let sgw = wq[wsb * 18u + 8u + bu] - let qh = (wq[wsb * 18u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u - let l0 = hh * 2u - let sg0 = (sgw >> (l0 * 8u)) & 255u - let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u - let i0 = (((qsw >> (l0 * 8u)) & 255u) | ((qh << (8u - 2u * l0)) & 0x300u)) * 2u - let i1 = (((qsw >> (l0 * 8u + 8u)) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 0x300u)) * 2u - wv = uint4(iq3s_signed(gridw[i0], sg0 & 15u), iq3s_signed(gridw[i0 + 1u], sg0 >> 4u), - iq3s_signed(gridw[i1], sg1 & 15u), iq3s_signed(gridw[i1 + 1u], sg1 >> 4u)) - } - ww4[grp * 17u + k] = wv - } - } - +// the iq2 grid formats' scale stage and fold: per-16 UNSIGNED strips on k6's wsw/wsw2 planes and a +// plain split fma (no offset, no min term); iq2s and iq2xs stage their grids and gather on it +class KqBatchIq2Base : KqBatchK6 { def override stage_ws(wsb0, wt, nsb, s, tid : uint) { let slane = tid % 32u let sblk = tid / 32u @@ -4116,6 +3929,37 @@ class KqBatchIq2s : KqBatchK6 { xscl * (wsw[bu * 32u + word + 16u] * float(l2) + wsw2[bu * 32u + word + 16u] * float(h2)), xscl * (wsw[bu * 32u + word + 24u] * float(l3) + wsw2[bu * 32u + word + 24u] * float(h3))) } +} + +// IQ2_S tile: the iq3s grid gather widened to the u64 grid (a 10-bit qs|qh index picks TWO +// words - eight magnitudes) with the block's own sign bytes. The 8 KB grid stages once, 8 words +// per thread. +[vk_dispatch(name = "kq_batch_iq2s_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq2s : KqBatchIq2Base { + @workgroup gridw : uint[2048] + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qsw = wq[wsb * 18u + bu] + let sgw = wq[wsb * 18u + 8u + bu] + let qh = (wq[wsb * 18u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + let l0 = hh * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let i0 = (((qsw >> (l0 * 8u)) & 255u) | ((qh << (8u - 2u * l0)) & 0x300u)) * 2u + let i1 = (((qsw >> (l0 * 8u + 8u)) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 0x300u)) * 2u + wv = uint4(iq3s_signed(gridw[i0], sg0 & 15u), iq3s_signed(gridw[i0 + 1u], sg0 >> 4u), + iq3s_signed(gridw[i1], sg1 & 15u), iq3s_signed(gridw[i1 + 1u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } [spirv_kernel(local_size_x = 256, name = "kq_batch_iq2s_cls_spv")] def override run { @@ -4130,17 +3974,11 @@ class KqBatchIq2s : KqBatchK6 { // IQ2_XS tile: the iq2s gather with ksigns-by-parity - each u16 qs word carries a 9-bit grid // index (two staged words) and a 7-bit KSIGNS_IQ2XS index whose sign byte ksign7 recomputes -// (no sign plane, no qh); strips and fma ride the iq2s spellings. The 4 KB grid stages once. +// (no sign plane, no qh). The 4 KB grid stages once. [vk_dispatch(name = "kq_batch_iq2xs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] -class KqBatchIq2xs : KqBatchK6 { +class KqBatchIq2xs : KqBatchIq2Base { @workgroup gridw : uint[1024] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { for (ki in range(2)) { let k = word + uint(ki) * 8u @@ -4161,52 +3999,6 @@ class KqBatchIq2xs : KqBatchK6 { } } - def override stage_ws(wsb0, wt, nsb, s, tid : uint) { - let slane = tid % 32u - let sblk = tid / 32u - var sa = 0.0 - var sbv = 0.0 - if (wt * 32u + slane < pa.d) { - let wsb = wsb0 + (wt * 32u + slane) * nsb + s - let w = wsu[wsb * 5u + 1u + sblk / 2u] - let s0 = (w >> ((sblk % 2u) * 16u)) & 255u - let s1 = (w >> ((sblk % 2u) * 16u + 8u)) & 255u - let dd = unpackHalf2x16(wsu[wsb * 5u]).x - sa = dd * float(s0) - sbv = dd * float(s1) - } - wsw[sblk * 32u + slane] = sa - wsw2[sblk * 32u + slane] = sbv - } - - // the split-half fma with UNSIGNED strips: xscl * (d*s0*l + d*s1*h) - no offset, no min - def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { - let kL = bu * 2u - let kH = bu * 2u + 1u - let xwL = xw4[grp * 17u + kL] - let xwH = xw4[grp * 17u + kH] - let wL0 = ww4[word * 17u + kL] - let wL1 = ww4[(word + 8u) * 17u + kL] - let wL2 = ww4[(word + 16u) * 17u + kL] - let wL3 = ww4[(word + 24u) * 17u + kL] - let wH0 = ww4[word * 17u + kH] - let wH1 = ww4[(word + 8u) * 17u + kH] - let wH2 = ww4[(word + 16u) * 17u + kH] - let wH3 = ww4[(word + 24u) * 17u + kH] - let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) - let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) - let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) - let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) - let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) - let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) - let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) - let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) - return float4(xscl * (wsw[bu * 32u + word] * float(l0) + wsw2[bu * 32u + word] * float(h0)), - xscl * (wsw[bu * 32u + word + 8u] * float(l1) + wsw2[bu * 32u + word + 8u] * float(h1)), - xscl * (wsw[bu * 32u + word + 16u] * float(l2) + wsw2[bu * 32u + word + 16u] * float(h2)), - xscl * (wsw[bu * 32u + word + 24u] * float(l3) + wsw2[bu * 32u + word + 24u] * float(h3))) - } - [spirv_kernel(local_size_x = 256, name = "kq_batch_iq2xs_cls_spv")] def override run { let tid = gl_LocalInvocationID.x @@ -4221,26 +4013,15 @@ class KqBatchIq2xs : KqBatchK6 { // ===== the prefill batch GEMMs — cooperative-matrix variants (DASLLAMA_COOPMAT=f16|int8|mm) ===== let private CML_SST = 20u // mm shared row stride in uints (16 f16-pair data + 4 pad — bank spread) - -// the MoE region rail the coopmat batch tiles share: sched = 4-word region records -// [wblk0, row0, cnt, wg0] at 0, then the per-wg map at pa.map_off -class MoeCmBase { - @ssbo @binding = 2 sched : array // region records + per-wg map (bulk schedule data) - @push_constant pa : BatchArgs - - def region_rec : uint4 { - let rid = sched[pa.map_off + gl_WorkGroupID.x] - let rb = rid * 4u - return uint4(sched[rb], sched[rb + 1u], sched[rb + 2u], gl_WorkGroupID.x - sched[rb + 3u]) - } -} - -// f16 tensor-core 32x32 tile: same MoE routing + Q8_0 layout as Q8Batch, but the inner GEMM rides -// the tensor cores — 128 threads / 4 subgroups, one 16x16 subtile each; edge tiles zero-fill -[vk_dispatch(name = "q8_batch_cmf16_cls", grid = "wgs", params = "wgs : int64")] -class CmF16Batch : MoeCmBase { - @ssbo @binding = 0 wq : array // weight quant words - @ssbo @binding = 1 wsh : array // per-block weight scales +let private KHR_STRIDE = 20u // the KHR kq tile's shared row stride in 32-bit words: 32 k halves as 16 f16 pairs + 4 pad (bank spread) +let private KHR_KHALF_WORDS = 8u // the 16 k halves one thread stages per step, as f16 pairs + +// The f16 tensor-core 32x32 tile shell (DASLLAMA_COOPMAT=f16): the MoE routing of Q8Batch with the +// inner GEMM on the tensor cores - 128 threads / 4 subgroups, one 16x16 subtile each, the staged +// f16 A (32 rows x 256 k) and B (256 k x 32 cols) tiles, 16 MMAs per step, the f32 scratch bounce +// for the bounds-checked store. A leaf stages its own weight format into the tiles (edge rows +// zero-fill) and hands the shell its step count and the block count its stage reads. +class CmF16Base : MoeCmBase { @ssbo @binding = 3 xqw : array // activation quant words @ssbo @binding = 4 axs : array // per-block activation scales @ssbo @binding = 5 y : array @@ -4248,8 +4029,11 @@ class CmF16Batch : MoeCmBase { @workgroup cm_bs : float16[8192] // 256 k x 32 cols, row-major stride 32 (f16 dequant B) @workgroup cm_out : float[1024] // 4 subgroups x 16x16 f32 scratch (bounds-checked write-out) - [spirv_kernel(local_size_x = 128, name = "q8_batch_cmf16_cls_spv")] - def run { // nolint:STYLE038 — the verbatim tile pass, parity-locked to the shipped kernel + // step s of the leaf's format: the activation rows into cm_as, the weight columns into cm_bs + def stage(s, wblk0, row0, cnt, xt, wt, nblk, tid : uint) { + } + + def body(nsteps, nblk : uint) { let reg = region_rec() let wblk0 = reg.x let row0 = reg.y @@ -4258,8 +4042,6 @@ class CmF16Batch : MoeCmBase { let wtiles = (pa.d + 31u) / 32u let xt = tix / wtiles let wt = tix % wtiles - let nbb = pa.n / 32u - let nsteps = (nbb + 7u) / 8u let tid = gl_LocalInvocationID.x let sg = gl_SubgroupID // 0..3 (subgroupSize 32) let row_half = sg / 2u // which 16-row half of the tile @@ -4267,49 +4049,7 @@ class CmF16Batch : MoeCmBase { var acc : coopmatAcc_f32_16x16 var s = 0u while (s < nsteps) { - let b0 = s * 8u - for (i in range(16)) { // stage A: 32x64 = 2048 uints, 16/thread; OOR -> 0 - let u = tid * 16u + uint(i) - let row = u / 64u // 0..31 local token row - let chunk = u % 64u - let blk = chunk / 8u - let w = chunk % 8u - let xrow = xt * 32u + row - let gblk = b0 + blk - var packed = 0u - var xs = 0.0 - if (xrow < cnt && gblk < nbb) { - packed = xqw[((row0 + xrow) * nbb + gblk) * 8u + w] - xs = axs[(row0 + xrow) * nbb + gblk] - } - let qv = int4(unpack8(int(packed))) - let o = row * 256u + blk * 32u + w * 4u - cm_as[o] = float16(float(qv.x) * xs) - cm_as[o + 1u] = float16(float(qv.y) * xs) - cm_as[o + 2u] = float16(float(qv.z) * xs) - cm_as[o + 3u] = float16(float(qv.w) * xs) - } - for (i in range(16)) { // stage B: 32x64 = 2048 uints, 16/thread; OOR -> 0 - let u = tid * 16u + uint(i) - let col = u / 64u // 0..31 output col - let chunk = u % 64u - let blk = chunk / 8u - let w = chunk % 8u - let wcol = wt * 32u + col - let gblk = b0 + blk - var packed = 0u - var ws = 0.0 - if (wcol < pa.d && gblk < nbb) { - packed = wq[(wblk0 + wcol * nbb + gblk) * 8u + w] - ws = float(wsh[wblk0 + wcol * nbb + gblk]) - } - let qv = int4(unpack8(int(packed))) - let k = blk * 32u + w * 4u - cm_bs[k * 32u + col] = float16(float(qv.x) * ws) - cm_bs[(k + 1u) * 32u + col] = float16(float(qv.y) * ws) - cm_bs[(k + 2u) * 32u + col] = float16(float(qv.z) * ws) - cm_bs[(k + 3u) * 32u + col] = float16(float(qv.w) * ws) - } + stage(s, wblk0, row0, cnt, xt, wt, nblk, tid) barrier() for (bb in range(8)) { // 8 blocks x 2 K-subtiles = 16 MMAs into one acc for (ks in range(2)) { @@ -4339,6 +4079,65 @@ class CmF16Batch : MoeCmBase { } } +// the Q8_0 leaf: eight 32-blocks per step, each staged uint the four quants of one block word +[vk_dispatch(name = "q8_batch_cmf16_cls", grid = "wgs", params = "wgs : int64")] +class CmF16Batch : CmF16Base { + @ssbo @binding = 0 wq : array // weight quant words + @ssbo @binding = 1 wsh : array // per-block weight scales + + def override stage(s, wblk0, row0, cnt, xt, wt, nblk, tid : uint) { + let b0 = s * 8u + for (i in range(16)) { // stage A: 32x64 = 2048 uints, 16/thread; OOR -> 0 + let u = tid * 16u + uint(i) + let row = u / 64u // 0..31 local token row + let chunk = u % 64u + let blk = chunk / 8u + let w = chunk % 8u + let xrow = xt * 32u + row + let gblk = b0 + blk + var packed = 0u + var xs = 0.0 + if (xrow < cnt && gblk < nblk) { + packed = xqw[((row0 + xrow) * nblk + gblk) * 8u + w] + xs = axs[(row0 + xrow) * nblk + gblk] + } + let qv = int4(unpack8(int(packed))) + let o = row * 256u + blk * 32u + w * 4u + cm_as[o] = float16(float(qv.x) * xs) + cm_as[o + 1u] = float16(float(qv.y) * xs) + cm_as[o + 2u] = float16(float(qv.z) * xs) + cm_as[o + 3u] = float16(float(qv.w) * xs) + } + for (i in range(16)) { // stage B: 32x64 = 2048 uints, 16/thread; OOR -> 0 + let u = tid * 16u + uint(i) + let col = u / 64u // 0..31 output col + let chunk = u % 64u + let blk = chunk / 8u + let w = chunk % 8u + let wcol = wt * 32u + col + let gblk = b0 + blk + var packed = 0u + var ws = 0.0 + if (wcol < pa.d && gblk < nblk) { + packed = wq[(wblk0 + wcol * nblk + gblk) * 8u + w] + ws = float(wsh[wblk0 + wcol * nblk + gblk]) + } + let qv = int4(unpack8(int(packed))) + let k = blk * 32u + w * 4u + cm_bs[k * 32u + col] = float16(float(qv.x) * ws) + cm_bs[(k + 1u) * 32u + col] = float16(float(qv.y) * ws) + cm_bs[(k + 2u) * 32u + col] = float16(float(qv.z) * ws) + cm_bs[(k + 3u) * 32u + col] = float16(float(qv.w) * ws) + } + } + + [spirv_kernel(local_size_x = 128, name = "q8_batch_cmf16_cls_spv")] + def run { + let nbb = pa.n / 32u + body((nbb + 7u) / 8u, nbb) + } +} + // int8 path: the tensor cores eat s8 quants directly — K-tile 32 = one Q8_0 block, one MMA per // block, then the s32 tile spills to shared for the SIMT xs[m]*ws[n] scale [vk_dispatch(name = "q8_batch_cmi8_cls", grid = "wgs", params = "wgs : int64")] @@ -4406,37 +4205,107 @@ class CmI8Batch : MoeCmBase { } } -// the mul_mm L-tile variant (DASLLAMA_COOPMAT=mm): the aligned-L geometry — BM=BN=128 -// BK=32, 4 warps 2x2 with 16 f16acc fragments each. Edge tiles bounce stores via cml_o -[vk_dispatch(name = "q8_batch_mm_cls", grid = "wgs", params = "wgs : int64")] -class MmBatch : MoeCmBase { +// The mul_mm tile (DASLLAMA_COOPMAT=mm) as ONE class template over the tile edge and the edge +// policy. The fragment sets are per-edge `static_if` arms: a cooperative-matrix handle cannot sit +// in a sized array, and each arm's MMA order is parity-locked to the shipped kernel. +[ |> template_struct_instance] +class template MmBatchT : MoeCmBase { @ssbo @binding = 0 wq : array // weight quant words @ssbo @binding = 1 wsh : array // per-block weight scales @ssbo @binding = 3 xqw : array // activation quant words @ssbo @binding = 4 axs : array // per-block activation scales @ssbo @binding = 5 y : array - @workgroup cml_a : uint[2560] // A tile: 128 weight rows x one block (32 K) as f16 pairs - @workgroup cml_b : uint[2560] // B tile: 128 token rows x one block - @workgroup cml_o : float16[1024] // 4 warps x 16x16 f16-acc store bounce (edge tiles only) + @template_constant BT : uint = 128u // the tile edge: 128 = the aligned-L geometry (BM=BN=128 BK=32, 4 warps 2x2 with 16 f16acc fragments each), 64 = the M twin (2x2 fragments each, routed in when cnt or d <= 64) + @template_constant EDGE : bool = true // rows past d / cnt exist: guarded staging, bounced stores; the aligned-L leaf runs with it off, its own pipeline for d and every cnt 128-multiples + @workgroup cml_a : uint[int(BT * CML_SST)] // A tile: BT weight rows x one block (32 K) as f16 pairs + @workgroup cml_b : uint[int(BT * CML_SST)] // B tile: BT token rows x one block + @workgroup @template_gate = EDGE cml_o : float16[1024] // 4 warps x 16x16 f16-acc store bounce (edge tiles only) + + // stage A: BT weight rows x 1 block; OOR rows -> 0 on the edge policy + def stage_a(wblk0, gm0, nbb, kb, lw, lr0 : uint) { + for [unroll] (p in range(int(BT) / 16)) { + let row = lr0 + uint(p) * 16u + let wrow = gm0 + row + var v = float4(0.0) + static_if (EDGE) { + if (wrow < pa.d) { + let ib = wblk0 + wrow * nbb + kb + v = float4(int4(unpack8(int(wq[ib * 8u + lw])))) * float(wsh[ib]) + } + } else { + let ib = wblk0 + wrow * nbb + kb + v = float4(int4(unpack8(int(wq[ib * 8u + lw])))) * float(wsh[ib]) + } + let o = row * CML_SST + lw * 2u + cml_a[o] = packHalf2x16(v.xy) + cml_a[o + 1u] = packHalf2x16(v.zw) + } + } + + // stage B: BT token rows x 1 block; OOR rows -> 0 on the edge policy + def stage_b(row0, gn0, cnt, nbb, kb, lw, lr0 : uint) { + for [unroll] (p in range(int(BT) / 16)) { + let row = lr0 + uint(p) * 16u + var v = float4(0.0) + static_if (EDGE) { + if (gn0 + row < cnt) { + let ib = (row0 + gn0 + row) * nbb + kb + v = float4(int4(unpack8(int(xqw[ib * 8u + lw])))) * axs[ib] + } + } else { + let ib = (row0 + gn0 + row) * nbb + kb + v = float4(int4(unpack8(int(xqw[ib * 8u + lw])))) * axs[ib] + } + let o = row * CML_SST + lw * 2u + cml_b[o] = packHalf2x16(v.xy) + cml_b[o + 1u] = packHalf2x16(v.zw) + } + } + + // an edge tile's fragment, already stored to this warp's cml_o slab (rowmajor, row = weight + // dim): 32 lanes widen and write 8 elements each under the row and column bounds + def bounce_out(fr, fc, dr, dc, row0, cnt : uint) { + static_if (EDGE) { + let lane = gl_SubgroupInvocationID + let ob = gl_SubgroupID * 256u + for (e in range(8)) { + let idx = lane * 8u + uint(e) + let wr = dr + fr * 16u + idx / 16u + let tc = dc + fc * 16u + idx % 16u + if (wr < pa.d && tc < cnt) { + y[(row0 + tc) * pa.d + wr] = float(cml_o[ob + idx]) + } + } + } + } - [spirv_kernel(local_size_x = 128, name = "q8_batch_mm_cls_spv")] - def run { // nolint:STYLE037,STYLE038 — the verbatim tile pass, parity-locked to the shipped kernel + [spirv_kernel(local_size_x = 128)] + def run { let reg = region_rec() let wblk0 = reg.x let row0 = reg.y let cnt = reg.z let tix = reg.w - let wtiles = (pa.d + 127u) / 128u + let wtiles = (pa.d + BT - 1u) / BT let xt = tix / wtiles // token (B) tile let wt = tix % wtiles // weight (A) tile - let gm0 = wt * 128u - let gn0 = xt * 128u + let gm0 = wt * BT + let gn0 = xt * BT let nbb = pa.n / 32u let tid = gl_LocalInvocationID.x - let warp_r = gl_SubgroupID % 2u // which 64-row half of the A tile - let warp_c = gl_SubgroupID / 2u // which 64-col half of the B tile + let warp_r = gl_SubgroupID % 2u // which half of the A tile + let warp_c = gl_SubgroupID / 2u // which half of the B tile let lw = tid % 8u // which packed word (4 qs) of the staged row's block let lr0 = tid / 8u // staging row base 0..15 + static_if (BT == 128u) { + l_tile(wblk0, row0, cnt, gm0, gn0, nbb, warp_r, warp_c, lw, lr0) + } else { + m_tile(wblk0, row0, cnt, gm0, gn0, nbb, warp_r, warp_c, lw, lr0) + } + } + + def l_tile(wblk0, row0, cnt, gm0, gn0, nbb, warp_r, warp_c, lw, lr0 : uint) { // nolint:STYLE038 — the verbatim tile pass, parity-locked to the shipped kernel + static_if (BT == 128u) { var acc00 : coopmatAcc_f16_16x16 var acc01 : coopmatAcc_f16_16x16 var acc02 : coopmatAcc_f16_16x16 @@ -4455,29 +4324,8 @@ class MmBatch : MoeCmBase { var acc33 : coopmatAcc_f16_16x16 var kb = 0u while (kb < nbb) { - for [unroll] (p in range(8)) { // stage A: 128 weight rows x 1 block; OOR rows -> 0 - let row = lr0 + uint(p) * 16u - let wrow = gm0 + row - var v = float4(0.0) - if (wrow < pa.d) { - let ib = wblk0 + wrow * nbb + kb - v = float4(int4(unpack8(int(wq[ib * 8u + lw])))) * float(wsh[ib]) - } - let o = row * CML_SST + lw * 2u - cml_a[o] = packHalf2x16(v.xy) - cml_a[o + 1u] = packHalf2x16(v.zw) - } - for [unroll] (p in range(8)) { // stage B: 128 token rows x 1 block; OOR rows -> 0 - let row = lr0 + uint(p) * 16u - var v = float4(0.0) - if (gn0 + row < cnt) { - let ib = (row0 + gn0 + row) * nbb + kb - v = float4(int4(unpack8(int(xqw[ib * 8u + lw])))) * axs[ib] - } - let o = row * CML_SST + lw * 2u - cml_b[o] = packHalf2x16(v.xy) - cml_b[o + 1u] = packHalf2x16(v.zw) - } + stage_a(wblk0, gm0, nbb, kb, lw, lr0) + stage_b(row0, gn0, cnt, nbb, kb, lw, lr0) barrier() for [unroll] (ks in range(2)) { // 2 K-subtiles x (4 A rows x 4 B cols), their MMA order let k8 = uint(ks) * 8u @@ -4526,7 +4374,8 @@ class MmBatch : MoeCmBase { // stores: y[(row0 + token) * d + wrow] — fragment weight-dim contiguous = column-major, stride d let dr = gm0 + warp_r * 64u let dc = gn0 + warp_c * 64u - if (gm0 + 128u <= pa.d && gn0 + 128u <= cnt) { + let whole = !EDGE || (gm0 + 128u <= pa.d && gn0 + 128u <= cnt) + if (whole) { var accw : coopmatAcc_f32_16x16 let yb = (row0 + dc) * pa.d + dr coopmatConvert(accw, acc00) @@ -4562,283 +4411,78 @@ class MmBatch : MoeCmBase { coopmatConvert(accw, acc33) coopmatStore(accw, y, int(yb + 48u * pa.d + 48u), int(pa.d), 1) } else { - // edge tile: every fragment bounces its f16 acc through this warp's cml_o slab (rowmajor, - // row = weight dim), then 32 lanes widen + write 8 elements each under bounds checks - let lane = gl_SubgroupInvocationID - let ob = gl_SubgroupID * 256u - for (fi in range(16)) { - let fr = uint(fi) / 4u // fragment row: weight offset fr*16 - let fc = uint(fi) % 4u // fragment col: token offset fc*16 - if (fi == 0) { - coopmatStore(acc00, cml_o, int(ob), 16, 0) - } elif (fi == 1) { - coopmatStore(acc01, cml_o, int(ob), 16, 0) - } elif (fi == 2) { - coopmatStore(acc02, cml_o, int(ob), 16, 0) - } elif (fi == 3) { - coopmatStore(acc03, cml_o, int(ob), 16, 0) - } elif (fi == 4) { - coopmatStore(acc10, cml_o, int(ob), 16, 0) - } elif (fi == 5) { - coopmatStore(acc11, cml_o, int(ob), 16, 0) - } elif (fi == 6) { - coopmatStore(acc12, cml_o, int(ob), 16, 0) - } elif (fi == 7) { - coopmatStore(acc13, cml_o, int(ob), 16, 0) - } elif (fi == 8) { - coopmatStore(acc20, cml_o, int(ob), 16, 0) - } elif (fi == 9) { - coopmatStore(acc21, cml_o, int(ob), 16, 0) - } elif (fi == 10) { - coopmatStore(acc22, cml_o, int(ob), 16, 0) - } elif (fi == 11) { - coopmatStore(acc23, cml_o, int(ob), 16, 0) - } elif (fi == 12) { - coopmatStore(acc30, cml_o, int(ob), 16, 0) - } elif (fi == 13) { - coopmatStore(acc31, cml_o, int(ob), 16, 0) - } elif (fi == 14) { - coopmatStore(acc32, cml_o, int(ob), 16, 0) - } else { - coopmatStore(acc33, cml_o, int(ob), 16, 0) - } - barrier() - for (e in range(8)) { - let idx = lane * 8u + uint(e) - let wr = dr + fr * 16u + idx / 16u - let tc = dc + fc * 16u + idx % 16u - if (wr < pa.d && tc < cnt) { - y[(row0 + tc) * pa.d + wr] = float(cml_o[ob + idx]) + static_if (EDGE) { + // edge tile: every fragment bounces its f16 acc through this warp's cml_o slab + let ob = gl_SubgroupID * 256u + for (fi in range(16)) { + let fr = uint(fi) / 4u // fragment row: weight offset fr*16 + let fc = uint(fi) % 4u // fragment col: token offset fc*16 + if (fi == 0) { + coopmatStore(acc00, cml_o, int(ob), 16, 0) + } elif (fi == 1) { + coopmatStore(acc01, cml_o, int(ob), 16, 0) + } elif (fi == 2) { + coopmatStore(acc02, cml_o, int(ob), 16, 0) + } elif (fi == 3) { + coopmatStore(acc03, cml_o, int(ob), 16, 0) + } elif (fi == 4) { + coopmatStore(acc10, cml_o, int(ob), 16, 0) + } elif (fi == 5) { + coopmatStore(acc11, cml_o, int(ob), 16, 0) + } elif (fi == 6) { + coopmatStore(acc12, cml_o, int(ob), 16, 0) + } elif (fi == 7) { + coopmatStore(acc13, cml_o, int(ob), 16, 0) + } elif (fi == 8) { + coopmatStore(acc20, cml_o, int(ob), 16, 0) + } elif (fi == 9) { + coopmatStore(acc21, cml_o, int(ob), 16, 0) + } elif (fi == 10) { + coopmatStore(acc22, cml_o, int(ob), 16, 0) + } elif (fi == 11) { + coopmatStore(acc23, cml_o, int(ob), 16, 0) + } elif (fi == 12) { + coopmatStore(acc30, cml_o, int(ob), 16, 0) + } elif (fi == 13) { + coopmatStore(acc31, cml_o, int(ob), 16, 0) + } elif (fi == 14) { + coopmatStore(acc32, cml_o, int(ob), 16, 0) + } else { + coopmatStore(acc33, cml_o, int(ob), 16, 0) } + barrier() + bounce_out(fr, fc, dr, dc, row0, cnt) + barrier() } - barrier() } } - } -} - -// the aligned L twin: staging guards + edge branch removed — its own pipeline for -// d + every cnt 128-multiples -[vk_dispatch(name = "q8_batch_mm_a_cls", grid = "wgs", params = "wgs : int64")] -class MmABatch : MoeCmBase { - @ssbo @binding = 0 wq : array // weight quant words - @ssbo @binding = 1 wsh : array // per-block weight scales - @ssbo @binding = 3 xqw : array // activation quant words - @ssbo @binding = 4 axs : array // per-block activation scales - @ssbo @binding = 5 y : array - @workgroup cma_a : uint[2560] // A tile: 128 weight rows x one block (32 K) as f16 pairs - @workgroup cma_b : uint[2560] // B tile: 128 token rows x one block - - [spirv_kernel(local_size_x = 128, name = "q8_batch_mm_a_cls_spv")] - def run { // nolint:STYLE038 — the verbatim tile pass, parity-locked to the shipped kernel - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let tix = reg.w - let wtiles = (pa.d + 127u) / 128u - let xt = tix / wtiles // token (B) tile - let wt = tix % wtiles // weight (A) tile - let gm0 = wt * 128u - let gn0 = xt * 128u - let nbb = pa.n / 32u - let tid = gl_LocalInvocationID.x - let warp_r = gl_SubgroupID % 2u // which 64-row half of the A tile - let warp_c = gl_SubgroupID / 2u // which 64-col half of the B tile - let lw = tid % 8u // which packed word (4 qs) of the staged row's block - let lr0 = tid / 8u // staging row base 0..15 - var acc00 : coopmatAcc_f16_16x16 - var acc01 : coopmatAcc_f16_16x16 - var acc02 : coopmatAcc_f16_16x16 - var acc03 : coopmatAcc_f16_16x16 - var acc10 : coopmatAcc_f16_16x16 - var acc11 : coopmatAcc_f16_16x16 - var acc12 : coopmatAcc_f16_16x16 - var acc13 : coopmatAcc_f16_16x16 - var acc20 : coopmatAcc_f16_16x16 - var acc21 : coopmatAcc_f16_16x16 - var acc22 : coopmatAcc_f16_16x16 - var acc23 : coopmatAcc_f16_16x16 - var acc30 : coopmatAcc_f16_16x16 - var acc31 : coopmatAcc_f16_16x16 - var acc32 : coopmatAcc_f16_16x16 - var acc33 : coopmatAcc_f16_16x16 - var kb = 0u - while (kb < nbb) { - for [unroll] (p in range(8)) { // stage A: 128 weight rows x 1 block, unguarded - let row = lr0 + uint(p) * 16u - let ib = wblk0 + (gm0 + row) * nbb + kb - let v = float4(int4(unpack8(int(wq[ib * 8u + lw])))) * float(wsh[ib]) - let o = row * CML_SST + lw * 2u - cma_a[o] = packHalf2x16(v.xy) - cma_a[o + 1u] = packHalf2x16(v.zw) - } - for [unroll] (p in range(8)) { // stage B: 128 token rows x 1 block, unguarded - let row = lr0 + uint(p) * 16u - let ib = (row0 + gn0 + row) * nbb + kb - let v = float4(int4(unpack8(int(xqw[ib * 8u + lw])))) * axs[ib] - let o = row * CML_SST + lw * 2u - cma_b[o] = packHalf2x16(v.xy) - cma_b[o + 1u] = packHalf2x16(v.zw) - } - barrier() - for [unroll] (ks in range(2)) { // 2 K-subtiles x (4 A rows x 4 B cols), their MMA order - let k8 = uint(ks) * 8u - var ca : coopmatA_f16_16x16 - var cb : coopmatB_f16_16x16 - coopmatLoad(ca, cma_a, int((warp_r * 64u) * CML_SST + k8), int(CML_SST), 0) - coopmatLoad(cb, cma_b, int((warp_c * 64u) * CML_SST + k8), int(CML_SST), 1) - acc00 = coopmatMulAdd(ca, cb, acc00) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 16u) * CML_SST + k8), int(CML_SST), 1) - acc01 = coopmatMulAdd(ca, cb, acc01) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 32u) * CML_SST + k8), int(CML_SST), 1) - acc02 = coopmatMulAdd(ca, cb, acc02) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 48u) * CML_SST + k8), int(CML_SST), 1) - acc03 = coopmatMulAdd(ca, cb, acc03) - coopmatLoad(ca, cma_a, int((warp_r * 64u + 16u) * CML_SST + k8), int(CML_SST), 0) - coopmatLoad(cb, cma_b, int((warp_c * 64u) * CML_SST + k8), int(CML_SST), 1) - acc10 = coopmatMulAdd(ca, cb, acc10) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 16u) * CML_SST + k8), int(CML_SST), 1) - acc11 = coopmatMulAdd(ca, cb, acc11) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 32u) * CML_SST + k8), int(CML_SST), 1) - acc12 = coopmatMulAdd(ca, cb, acc12) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 48u) * CML_SST + k8), int(CML_SST), 1) - acc13 = coopmatMulAdd(ca, cb, acc13) - coopmatLoad(ca, cma_a, int((warp_r * 64u + 32u) * CML_SST + k8), int(CML_SST), 0) - coopmatLoad(cb, cma_b, int((warp_c * 64u) * CML_SST + k8), int(CML_SST), 1) - acc20 = coopmatMulAdd(ca, cb, acc20) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 16u) * CML_SST + k8), int(CML_SST), 1) - acc21 = coopmatMulAdd(ca, cb, acc21) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 32u) * CML_SST + k8), int(CML_SST), 1) - acc22 = coopmatMulAdd(ca, cb, acc22) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 48u) * CML_SST + k8), int(CML_SST), 1) - acc23 = coopmatMulAdd(ca, cb, acc23) - coopmatLoad(ca, cma_a, int((warp_r * 64u + 48u) * CML_SST + k8), int(CML_SST), 0) - coopmatLoad(cb, cma_b, int((warp_c * 64u) * CML_SST + k8), int(CML_SST), 1) - acc30 = coopmatMulAdd(ca, cb, acc30) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 16u) * CML_SST + k8), int(CML_SST), 1) - acc31 = coopmatMulAdd(ca, cb, acc31) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 32u) * CML_SST + k8), int(CML_SST), 1) - acc32 = coopmatMulAdd(ca, cb, acc32) - coopmatLoad(cb, cma_b, int((warp_c * 64u + 48u) * CML_SST + k8), int(CML_SST), 1) - acc33 = coopmatMulAdd(ca, cb, acc33) - } - barrier() - kb++ } - // stores: y[(row0 + token) * d + wrow] — fragment weight-dim contiguous = column-major, stride d - let dr = gm0 + warp_r * 64u - let dc = gn0 + warp_c * 64u - var accw : coopmatAcc_f32_16x16 - let yb = (row0 + dc) * pa.d + dr - coopmatConvert(accw, acc00) - coopmatStore(accw, y, int(yb), int(pa.d), 1) - coopmatConvert(accw, acc01) - coopmatStore(accw, y, int(yb + 16u * pa.d), int(pa.d), 1) - coopmatConvert(accw, acc02) - coopmatStore(accw, y, int(yb + 32u * pa.d), int(pa.d), 1) - coopmatConvert(accw, acc03) - coopmatStore(accw, y, int(yb + 48u * pa.d), int(pa.d), 1) - coopmatConvert(accw, acc10) - coopmatStore(accw, y, int(yb + 16u), int(pa.d), 1) - coopmatConvert(accw, acc11) - coopmatStore(accw, y, int(yb + 16u * pa.d + 16u), int(pa.d), 1) - coopmatConvert(accw, acc12) - coopmatStore(accw, y, int(yb + 32u * pa.d + 16u), int(pa.d), 1) - coopmatConvert(accw, acc13) - coopmatStore(accw, y, int(yb + 48u * pa.d + 16u), int(pa.d), 1) - coopmatConvert(accw, acc20) - coopmatStore(accw, y, int(yb + 32u), int(pa.d), 1) - coopmatConvert(accw, acc21) - coopmatStore(accw, y, int(yb + 16u * pa.d + 32u), int(pa.d), 1) - coopmatConvert(accw, acc22) - coopmatStore(accw, y, int(yb + 32u * pa.d + 32u), int(pa.d), 1) - coopmatConvert(accw, acc23) - coopmatStore(accw, y, int(yb + 48u * pa.d + 32u), int(pa.d), 1) - coopmatConvert(accw, acc30) - coopmatStore(accw, y, int(yb + 48u), int(pa.d), 1) - coopmatConvert(accw, acc31) - coopmatStore(accw, y, int(yb + 16u * pa.d + 48u), int(pa.d), 1) - coopmatConvert(accw, acc32) - coopmatStore(accw, y, int(yb + 32u * pa.d + 48u), int(pa.d), 1) - coopmatConvert(accw, acc33) - coopmatStore(accw, y, int(yb + 48u * pa.d + 48u), int(pa.d), 1) - } -} - -// the mul_mm M-tile twin (64x64): the L scheme at half edge — 4 warps 2x2 over the tile, -// 2x2 fragments each. Routed in when cnt or d <= 64 -[vk_dispatch(name = "q8_batch_mm_m_cls", grid = "wgs", params = "wgs : int64")] -class MmMBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // weight quant words - @ssbo @binding = 1 wsh : array // per-block weight scales - @ssbo @binding = 3 xqw : array // activation quant words - @ssbo @binding = 4 axs : array // per-block activation scales - @ssbo @binding = 5 y : array - @workgroup cmm_a : uint[1280] // A tile: 64 weight rows x one block (32 K) as f16 pairs - @workgroup cmm_b : uint[1280] // B tile: 64 token rows x one block - @workgroup cml_o : float16[1024] // 4 warps x 16x16 f16-acc store bounce (edge tiles only) + } - [spirv_kernel(local_size_x = 128, name = "q8_batch_mm_m_cls_spv")] - def run { // nolint:STYLE038 — the verbatim tile pass, parity-locked to the shipped kernel - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let tix = reg.w - let wtiles = (pa.d + 63u) / 64u - let xt = tix / wtiles // token (B) tile - let wt = tix % wtiles // weight (A) tile - let gm0 = wt * 64u - let gn0 = xt * 64u - let nbb = pa.n / 32u - let tid = gl_LocalInvocationID.x - let warp_r = gl_SubgroupID % 2u // which 32-row half of the A tile - let warp_c = gl_SubgroupID / 2u // which 32-col half of the B tile - let lw = tid % 8u // which packed word (4 qs) of the staged row's block - let lr0 = tid / 8u // staging row base 0..15 + def m_tile(wblk0, row0, cnt, gm0, gn0, nbb, warp_r, warp_c, lw, lr0 : uint) { + static_if (BT == 64u) { var acc00 : coopmatAcc_f16_16x16 var acc01 : coopmatAcc_f16_16x16 var acc10 : coopmatAcc_f16_16x16 var acc11 : coopmatAcc_f16_16x16 var kb = 0u while (kb < nbb) { - for [unroll] (p in range(4)) { // stage A: 64 weight rows x 1 block; OOR rows -> 0 - let row = lr0 + uint(p) * 16u - let wrow = gm0 + row - var v = float4(0.0) - if (wrow < pa.d) { - let ib = wblk0 + wrow * nbb + kb - v = float4(int4(unpack8(int(wq[ib * 8u + lw])))) * float(wsh[ib]) - } - let o = row * CML_SST + lw * 2u - cmm_a[o] = packHalf2x16(v.xy) - cmm_a[o + 1u] = packHalf2x16(v.zw) - } - for [unroll] (p in range(4)) { // stage B: 64 token rows x 1 block; OOR rows -> 0 - let row = lr0 + uint(p) * 16u - var v = float4(0.0) - if (gn0 + row < cnt) { - let ib = (row0 + gn0 + row) * nbb + kb - v = float4(int4(unpack8(int(xqw[ib * 8u + lw])))) * axs[ib] - } - let o = row * CML_SST + lw * 2u - cmm_b[o] = packHalf2x16(v.xy) - cmm_b[o + 1u] = packHalf2x16(v.zw) - } + stage_a(wblk0, gm0, nbb, kb, lw, lr0) + stage_b(row0, gn0, cnt, nbb, kb, lw, lr0) barrier() for [unroll] (ks in range(2)) { // 2 K-subtiles x (2 A rows x 2 B cols), the L MMA order let k8 = uint(ks) * 8u var ca : coopmatA_f16_16x16 var cb : coopmatB_f16_16x16 - coopmatLoad(ca, cmm_a, int((warp_r * 32u) * CML_SST + k8), int(CML_SST), 0) - coopmatLoad(cb, cmm_b, int((warp_c * 32u) * CML_SST + k8), int(CML_SST), 1) + coopmatLoad(ca, cml_a, int((warp_r * 32u) * CML_SST + k8), int(CML_SST), 0) + coopmatLoad(cb, cml_b, int((warp_c * 32u) * CML_SST + k8), int(CML_SST), 1) acc00 = coopmatMulAdd(ca, cb, acc00) - coopmatLoad(cb, cmm_b, int((warp_c * 32u + 16u) * CML_SST + k8), int(CML_SST), 1) + coopmatLoad(cb, cml_b, int((warp_c * 32u + 16u) * CML_SST + k8), int(CML_SST), 1) acc01 = coopmatMulAdd(ca, cb, acc01) - coopmatLoad(ca, cmm_a, int((warp_r * 32u + 16u) * CML_SST + k8), int(CML_SST), 0) - coopmatLoad(cb, cmm_b, int((warp_c * 32u) * CML_SST + k8), int(CML_SST), 1) + coopmatLoad(ca, cml_a, int((warp_r * 32u + 16u) * CML_SST + k8), int(CML_SST), 0) + coopmatLoad(cb, cml_b, int((warp_c * 32u) * CML_SST + k8), int(CML_SST), 1) acc10 = coopmatMulAdd(ca, cb, acc10) - coopmatLoad(cb, cmm_b, int((warp_c * 32u + 16u) * CML_SST + k8), int(CML_SST), 1) + coopmatLoad(cb, cml_b, int((warp_c * 32u + 16u) * CML_SST + k8), int(CML_SST), 1) acc11 = coopmatMulAdd(ca, cb, acc11) } barrier() @@ -4847,7 +4491,8 @@ class MmMBatch : MoeCmBase { // stores: y[(row0 + token) * d + wrow] — same column-major layout as the L tile let dr = gm0 + warp_r * 32u let dc = gn0 + warp_c * 32u - if (gm0 + 64u <= pa.d && gn0 + 64u <= cnt) { + let whole = !EDGE || (gm0 + 64u <= pa.d && gn0 + 64u <= cnt) + if (whole) { var accw : coopmatAcc_f32_16x16 let yb = (row0 + dc) * pa.d + dr coopmatConvert(accw, acc00) @@ -4859,143 +4504,116 @@ class MmMBatch : MoeCmBase { coopmatConvert(accw, acc11) coopmatStore(accw, y, int(yb + 16u * pa.d + 16u), int(pa.d), 1) } else { - // edge tile: bounce each f16 acc through this warp's cml_o slab, bounds-checked write-out - let lane = gl_SubgroupInvocationID - let ob = gl_SubgroupID * 256u - for (fi in range(4)) { - let fr = uint(fi) / 2u // fragment row: weight offset fr*16 - let fc = uint(fi) % 2u // fragment col: token offset fc*16 - if (fi == 0) { - coopmatStore(acc00, cml_o, int(ob), 16, 0) - } elif (fi == 1) { - coopmatStore(acc01, cml_o, int(ob), 16, 0) - } elif (fi == 2) { - coopmatStore(acc10, cml_o, int(ob), 16, 0) - } else { - coopmatStore(acc11, cml_o, int(ob), 16, 0) - } - barrier() - for (e in range(8)) { - let idx = lane * 8u + uint(e) - let wr = dr + fr * 16u + idx / 16u - let tc = dc + fc * 16u + idx % 16u - if (wr < pa.d && tc < cnt) { - y[(row0 + tc) * pa.d + wr] = float(cml_o[ob + idx]) + static_if (EDGE) { + // edge tile: bounce each f16 acc through this warp's cml_o slab, bounds-checked write-out + let ob = gl_SubgroupID * 256u + for (fi in range(4)) { + let fr = uint(fi) / 2u // fragment row: weight offset fr*16 + let fc = uint(fi) % 2u // fragment col: token offset fc*16 + if (fi == 0) { + coopmatStore(acc00, cml_o, int(ob), 16, 0) + } elif (fi == 1) { + coopmatStore(acc01, cml_o, int(ob), 16, 0) + } elif (fi == 2) { + coopmatStore(acc10, cml_o, int(ob), 16, 0) + } else { + coopmatStore(acc11, cml_o, int(ob), 16, 0) } + barrier() + bounce_out(fr, fc, dr, dc, row0, cnt) + barrier() } - barrier() } } + } } } +// the L tile with edge tiles +[vk_dispatch(name = "q8_batch_mm_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class MmBatch : MmBatchT { + override BT = 128u + override EDGE = true +} + +// the aligned L twin: staging guards + edge branch removed — its own pipeline for d and every cnt 128-multiples +[vk_dispatch(name = "q8_batch_mm_a_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class MmABatch : MmBatchT { + override BT = 128u + override EDGE = false +} + +// the M twin (64x64), routed in when cnt or d <= 64 +[vk_dispatch(name = "q8_batch_mm_m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class MmMBatch : MmBatchT { + override BT = 64u + override EDGE = true +} + // Q4_0 coopmat f16 variant (fmt=4 stacks): the q40 math baked into f16 dequant — w = (nib-8)*d_b, // a = aq*xs (per-256 Q8_K) — then the same 4-subgroup MMA as CmF16Batch [vk_dispatch(name = "kq_batch_q40_cmf16_cls", grid = "wgs", params = "wgs : int64")] -class KqQ40CmF16 : MoeCmBase { +class KqQ40CmF16 : CmF16Base { @ssbo @binding = 0 wq : array // nibble-packed weight superblocks (32 words each) @ssbo @binding = 1 wsu : array // 5-word scale superblocks (d halves) - @ssbo @binding = 3 xqw : array // Q8_K activation words (64 per row-superblock) - @ssbo @binding = 4 axs : array // one Q8_K d per (row, superblock) - @ssbo @binding = 5 y : array - @workgroup cm_as : float16[8192] // 32 local rows x 256 k, row-major stride 256 (f16 dequant A) - @workgroup cm_bs : float16[8192] // 256 k x 32 cols, row-major stride 32 (f16 dequant B) - @workgroup cm_out : float[1024] // 4 subgroups x 16x16 f32 scratch (bounds-checked write-out) - [spirv_kernel(local_size_x = 128, name = "kq_batch_q40_cmf16_cls_spv")] - def run { // nolint:STYLE038 — the verbatim tile pass, parity-locked to the shipped kernel - let reg = region_rec() - let wsb0 = reg.x - let row0 = reg.y - let cnt = reg.z - let tix = reg.w - let wtiles = (pa.d + 31u) / 32u - let xt = tix / wtiles - let wt = tix % wtiles - let nsb = pa.n / 256u - let tid = gl_LocalInvocationID.x - let sg = gl_SubgroupID - let row_half = sg / 2u - let col_half = sg % 2u - var acc : coopmatAcc_f32_16x16 - var s = 0u - while (s < nsb) { - // stage A: 32 rows x 64 uints (one superblock of Q8_K acts), 16/thread — one thread stays - // in one row (16 divides 64), so xs (per-256) loads ONCE per thread - let arow0 = tid * 16u / 64u // this thread's row - let ac0 = tid * 16u % 64u // first chunk within the row - let axrow = xt * 32u + arow0 - var xs = 0.0 + def override stage(s, wblk0, row0, cnt, xt, wt, nblk, tid : uint) { + // stage A: 32 rows x 64 uints (one superblock of Q8_K acts), 16/thread — one thread stays + // in one row (16 divides 64), so xs (per-256) loads ONCE per thread + let arow0 = tid * 16u / 64u // this thread's row + let ac0 = tid * 16u % 64u // first chunk within the row + let axrow = xt * 32u + arow0 + var xs = 0.0 + if (axrow < cnt) { + xs = axs[(row0 + axrow) * nblk + s] + } + for (i in range(16)) { + let c = ac0 + uint(i) + let bu = c / 8u + let ww = c % 8u + var packed = 0u if (axrow < cnt) { - xs = axs[(row0 + axrow) * nsb + s] - } - for (i in range(16)) { - let c = ac0 + uint(i) - let bu = c / 8u - let ww = c % 8u - var packed = 0u - if (axrow < cnt) { - packed = xqw[((row0 + axrow) * nsb + s) * 64u + c] - } - let qv = int4(unpack8(int(packed))) - let o = arow0 * 256u + bu * 32u + ww * 4u - cm_as[o] = float16(float(qv.x) * xs) - cm_as[o + 1u] = float16(float(qv.y) * xs) - cm_as[o + 2u] = float16(float(qv.z) * xs) - cm_as[o + 3u] = float16(float(qv.w) * xs) - } - // stage B: 32 cols x 32 uints (one superblock of nibble-packed weights), 8/thread; each - // uint yields 4 lo-nibble weights (k = bu*32 + w*4 + j) and 4 hi (k + 16), dequant (nib-8)*d_b - for (i in range(8)) { - let u2 = tid * 8u + uint(i) - let col = u2 / 32u - let c = u2 % 32u - let bu = c / 4u - let w = c % 4u - let wcol = wt * 32u + col - var r = 0u - var db = 0.0 - if (wcol < pa.d) { - let wsb = wsb0 + wcol * nsb + s - r = wq[wsb * 32u + c] - let dp = unpackHalf2x16(wsu[wsb * 5u + bu / 2u]) - db = bu % 2u == 0u ? dp.x : dp.y - } - let k = bu * 32u + w * 4u - for (j in range(4)) { - let lo = int((r >> (uint(j) * 8u)) & 0x0Fu) - let hi = int((r >> (uint(j) * 8u + 4u)) & 0x0Fu) - cm_bs[(k + uint(j)) * 32u + col] = float16(float(lo - 8) * db) - cm_bs[(k + 16u + uint(j)) * 32u + col] = float16(float(hi - 8) * db) - } - } - barrier() - for (bb in range(8)) { - for (ks in range(2)) { - var a : coopmatA_f16_16x16 - var b : coopmatB_f16_16x16 - coopmatLoad(a, cm_as, int(row_half * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) - coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + col_half * 16u), 32, 0) - acc = coopmatMulAdd(a, b, acc) - } + packed = xqw[((row0 + axrow) * nblk + s) * 64u + c] + } + let qv = int4(unpack8(int(packed))) + let o = arow0 * 256u + bu * 32u + ww * 4u + cm_as[o] = float16(float(qv.x) * xs) + cm_as[o + 1u] = float16(float(qv.y) * xs) + cm_as[o + 2u] = float16(float(qv.z) * xs) + cm_as[o + 3u] = float16(float(qv.w) * xs) + } + // stage B: 32 cols x 32 uints (one superblock of nibble-packed weights), 8/thread; each + // uint yields 4 lo-nibble weights (k = bu*32 + w*4 + j) and 4 hi (k + 16), dequant (nib-8)*d_b + for (i in range(8)) { + let u2 = tid * 8u + uint(i) + let col = u2 / 32u + let c = u2 % 32u + let bu = c / 4u + let w = c % 4u + let wcol = wt * 32u + col + var r = 0u + var db = 0.0 + if (wcol < pa.d) { + let wsb = wblk0 + wcol * nblk + s + r = wq[wsb * 32u + c] + let dp = unpackHalf2x16(wsu[wsb * 5u + bu / 2u]) + db = bu % 2u == 0u ? dp.x : dp.y } - barrier() - s++ - } - coopmatStore(acc, cm_out, int(sg * 256u), 16, 0) - barrier() - let lane = gl_SubgroupInvocationID - for (e in range(8)) { - let idx = lane * 8u + uint(e) - let m = idx / 16u - let nn = idx % 16u - let orow = xt * 32u + row_half * 16u + m - let ocol = wt * 32u + col_half * 16u + nn - if (orow < cnt && ocol < pa.d) { - y[(row0 + orow) * pa.d + ocol] = cm_out[sg * 256u + idx] + let k = bu * 32u + w * 4u + for (j in range(4)) { + let lo = int((r >> (uint(j) * 8u)) & 0x0Fu) + let hi = int((r >> (uint(j) * 8u + 4u)) & 0x0Fu) + cm_bs[(k + uint(j)) * 32u + col] = float16(float(lo - 8) * db) + cm_bs[(k + 16u + uint(j)) * 32u + col] = float16(float(hi - 8) * db) } } } + + [spirv_kernel(local_size_x = 128, name = "kq_batch_q40_cmf16_cls_spv")] + def run { + let nsb = pa.n / 256u + body(nsb, nsb) + } } // ===== the cm2 (NV_cooperative_matrix2) prefill tiles: native planes, decode-in-load ===== @@ -5062,7 +4680,8 @@ struct VkIq2xxsBlk { class template KqCm2BatchT : MoeCmBase { @ssbo @binding = 0 @role = "weight" wq : array // the quant plane, read as 16-bit lanes @ssbo @binding = 1 @role = "weight" ws : array // the scale plane (q8: f16 per block; kq: 5 words per superblock) - @ssbo @binding = 3 xf16 : array // f16 activation plane + @ssbo @binding = 3 @role = "alias" xf16 : array // f16 activation plane (the tensor loads' view) + @ssbo @binding = 3 @role = "alias" xf16w4 : array // the same plane as 16-byte words: the KHR arm's activation stage @ssbo @binding = 5 y : array @workgroup wg_blk0 : uint // the region's block base, staged for the decode @template_constant BN : uint = 256u // the token column @@ -5082,9 +4701,12 @@ class template KqCm2BatchT : MoeCmBase { @workgroup @template_gate = IQ2XSGRID iq2xs_gridc : uint[1024] @template_constant IQ2XXSGRID : bool = false // stage the 2 KB iq2xxs u64 grid into workgroup memory @workgroup @template_gate = IQ2XXSGRID iq2xxs_gridc : uint[512] + @template_constant KHR : bool = false //!< the KHR cooperative-matrix arm: hand-staged f16 tiles over the format's own decode, no cm2 API (a card without NV_coopmat2) + @workgroup @template_gate = KHR khr_ao : uint[int(128u * KHR_STRIDE)] //! the A stage: 128 weight rows of f16 pairs; then the edge tile's f32 store bounce, 8 subgroups x 256 words + @workgroup @template_gate = KHR khr_b : uint[int(128u * KHR_STRIDE)] //! the B stage: BN (128) token rows of f16 pairs - [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE037,STYLE038 — the fast/edge path pair, barrier- and register-coupled + [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] + def run { let reg = region_rec() let wblk0 = reg.x let row0 = reg.y @@ -5110,8 +4732,6 @@ class template KqCm2BatchT : MoeCmBase { if (gl_LocalInvocationID.x == 0u) { wg_blk0 = wblk0 } - var tv : tensorView2Dt - tensorViewCreate(tv) static_if (IQLUT) { if (gl_LocalInvocationID.x < 16u) { let li = gl_LocalInvocationID.x @@ -5146,6 +4766,16 @@ class template KqCm2BatchT : MoeCmBase { } } barrier() // wg_blk0 (and the gated codebook) visible before the first decode load + static_if (KHR) { + khr_tile(wblk0, row0, cnt, k0, k1, ybase, xt, wt) + } else { + cm2_tile(wblk0, row0, cnt, k0, k1, ybase, xt, wt) + } + } + + def cm2_tile(wblk0, row0, cnt, k0, k1, ybase, xt, wt : uint) { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled + var tv : tensorView2Dt + tensorViewCreate(tv) var a : coopmatWgA_f16_128x64 var b : BT var acc : ACC @@ -5280,6 +4910,140 @@ class template KqCm2BatchT : MoeCmBase { coopmatConvert(accw, acc) coopmatStoreTensor(accw, y, ybase, tlo, t0, BN, m0, 128u, tv) } + + def khr_tile(wblk0, row0, cnt, k0, k1, ybase, xt, wt : uint) { // nolint:STYLE037,STYLE038 — the staged k loop and its fast/edge store are one register set + static_if (KHR) { // the gated staging arrays exist on the KHR instantiations alone + let t0 = row0 + xt * BN + let m0 = wt * 128u + let tid = gl_LocalInvocationID.x + let sg = gl_SubgroupID + let srow = tid >> 1u + let khalf = tid & 1u //! the low or the high 16 k of the step + let skh = khalf * 16u + let nbk = pa.n / BLKW + let wrow = m0 + srow + let trow = xt * BN + srow + let sbase = srow * KHR_STRIDE + khalf * KHR_KHALF_WORDS + let wrow_in_d = wrow < pa.d + let trow_in_cnt = trow < cnt + var acc0 : coopmatAcc_f32_16x16 + var acc1 : coopmatAcc_f32_16x16 + var acc2 : coopmatAcc_f32_16x16 + var acc3 : coopmatAcc_f32_16x16 + var acc4 : coopmatAcc_f32_16x16 + var acc5 : coopmatAcc_f32_16x16 + var acc6 : coopmatAcc_f32_16x16 + var acc7 : coopmatAcc_f32_16x16 + var k = k0 + while (k < k1) { + let kk = k + skh + if (wrow_in_d && kk < k1) { + let bcol = kk / BLKW + let e0 = kk - bcol * BLKW + static_if (DECV4) { + for (j in range(4)) { + let v4 = float4(decode_v4(wq[wblk0 + wrow * nbk + bcol], uint2(wrow, bcol), uint2(0u, e0 + uint(j) * 4u))) + khr_ao[sbase + uint(j) * 2u] = packHalf2x16(v4.xy) + khr_ao[sbase + uint(j) * 2u + 1u] = packHalf2x16(v4.zw) + } + } else { + for (j in range(int(KHR_KHALF_WORDS))) { + let lo = decode(wq[wblk0 + wrow * nbk + bcol], uint2(wrow, bcol), uint2(0u, e0 + uint(j) * 2u)) + let hi = decode(wq[wblk0 + wrow * nbk + bcol], uint2(wrow, bcol), uint2(0u, e0 + uint(j) * 2u + 1u)) + khr_ao[sbase + uint(j)] = packHalf2x16(float2(float(lo), float(hi))) + } + } + } else { + for (j in range(int(KHR_KHALF_WORDS))) { + khr_ao[sbase + uint(j)] = 0u + } + } + if (trow_in_cnt && kk < k1) { + let q = ((t0 + srow) * pa.n + kk) >> 3u // the 16 halves as two 16-byte words + for (h in range(2)) { + let w4 = xf16w4[q + uint(h)] + for (c in range(4)) { + khr_b[sbase + uint(h) * 4u + uint(c)] = w4[c] + } + } + } else { + for (j in range(int(KHR_KHALF_WORDS))) { + khr_b[sbase + uint(j)] = 0u + } + } + barrier() + for (ks in range(2)) { + let ko = uint(ks) * 8u + var a : coopmatA_f16_16x16 + coopmatLoad(a, khr_ao, int(sg * 16u * KHR_STRIDE + ko), int(KHR_STRIDE), 0) + var b : coopmatB_f16_16x16 + coopmatLoad(b, khr_b, int(ko), int(KHR_STRIDE), 1) + acc0 = coopmatMulAdd(a, b, acc0) + coopmatLoad(b, khr_b, int(16u * KHR_STRIDE + ko), int(KHR_STRIDE), 1) + acc1 = coopmatMulAdd(a, b, acc1) + coopmatLoad(b, khr_b, int(32u * KHR_STRIDE + ko), int(KHR_STRIDE), 1) + acc2 = coopmatMulAdd(a, b, acc2) + coopmatLoad(b, khr_b, int(48u * KHR_STRIDE + ko), int(KHR_STRIDE), 1) + acc3 = coopmatMulAdd(a, b, acc3) + coopmatLoad(b, khr_b, int(64u * KHR_STRIDE + ko), int(KHR_STRIDE), 1) + acc4 = coopmatMulAdd(a, b, acc4) + coopmatLoad(b, khr_b, int(80u * KHR_STRIDE + ko), int(KHR_STRIDE), 1) + acc5 = coopmatMulAdd(a, b, acc5) + coopmatLoad(b, khr_b, int(96u * KHR_STRIDE + ko), int(KHR_STRIDE), 1) + acc6 = coopmatMulAdd(a, b, acc6) + coopmatLoad(b, khr_b, int(112u * KHR_STRIDE + ko), int(KHR_STRIDE), 1) + acc7 = coopmatMulAdd(a, b, acc7) + } + barrier() + k += 32u + } + let mrow0 = m0 + sg * 16u + let tile_whole = m0 + 128u <= pa.d && xt * BN + BN <= cnt + if (tile_whole) { + let yb = ybase + t0 * pa.d + mrow0 + coopmatStore(acc0, y, int(yb), int(pa.d), 1) + coopmatStore(acc1, y, int(yb + 16u * pa.d), int(pa.d), 1) + coopmatStore(acc2, y, int(yb + 32u * pa.d), int(pa.d), 1) + coopmatStore(acc3, y, int(yb + 48u * pa.d), int(pa.d), 1) + coopmatStore(acc4, y, int(yb + 64u * pa.d), int(pa.d), 1) + coopmatStore(acc5, y, int(yb + 80u * pa.d), int(pa.d), 1) + coopmatStore(acc6, y, int(yb + 96u * pa.d), int(pa.d), 1) + coopmatStore(acc7, y, int(yb + 112u * pa.d), int(pa.d), 1) + } else { + let lane = gl_SubgroupInvocationID + let ob = sg * 256u + for (fi in range(8)) { + if (fi == 0) { + coopmatStore(acc0, khr_ao, int(ob), 16, 0) + } elif (fi == 1) { + coopmatStore(acc1, khr_ao, int(ob), 16, 0) + } elif (fi == 2) { + coopmatStore(acc2, khr_ao, int(ob), 16, 0) + } elif (fi == 3) { + coopmatStore(acc3, khr_ao, int(ob), 16, 0) + } elif (fi == 4) { + coopmatStore(acc4, khr_ao, int(ob), 16, 0) + } elif (fi == 5) { + coopmatStore(acc5, khr_ao, int(ob), 16, 0) + } elif (fi == 6) { + coopmatStore(acc6, khr_ao, int(ob), 16, 0) + } else { + coopmatStore(acc7, khr_ao, int(ob), 16, 0) + } + barrier() + for (e in range(8)) { + let idx = lane * 8u + uint(e) + let mr = mrow0 + idx / 16u + let tc = xt * BN + uint(fi) * 16u + idx % 16u + if (mr < pa.d && tc < cnt) { + y[ybase + (row0 + tc) * pa.d + mr] = uint_bits_to_float(khr_ao[ob + idx]) + } + } + barrier() + } + } + } + } } // the format templates: the block struct, the scale element, the block width and the decode @@ -5289,7 +5053,7 @@ class template Q8Cm2T : KqCm2BatchT { typedef ST = float16 override BLKW = 32u - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkQ8Blk; bc, cib : uint2) : float16 { let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] return ws[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) @@ -5302,7 +5066,7 @@ class template K4Cm2T : KqCm2BatchT { typedef ST = uint override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5316,7 +5080,7 @@ class template K4Cm2T : KqCm2BatchT { return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkK4Blk; bc, cib : uint2) : half4 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5339,7 +5103,7 @@ class template K6Cm2T : KqCm2BatchT { typedef ST = uint override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkK6Blk; bc, cib : uint2) : float16 { let e = cib.y let bu = e >> 5u @@ -5357,7 +5121,7 @@ class template K6Cm2T : KqCm2BatchT { return float16(d * float(sc) * float(q6)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkK6Blk; bc, cib : uint2) : half4 { let e = cib.y let bu = e >> 5u @@ -5389,7 +5153,7 @@ class template K5Cm2T : KqCm2BatchT { typedef ST = uint override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkK5Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5407,7 +5171,7 @@ class template K5Cm2T : KqCm2BatchT { return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkK5Blk; bc, cib : uint2) : half4 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5437,7 +5201,7 @@ class template Q40Cm2T : KqCm2BatchT { typedef ST = uint override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5450,7 +5214,7 @@ class template Q40Cm2T : KqCm2BatchT { return float16(d * (float(q) - 8.0)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkK4Blk; bc, cib : uint2) : half4 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5473,20 +5237,21 @@ class template Iq4xsCm2T : KqCm2BatchT { override IQLUT = true override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u let bidx = g * 16u + (e & 15u) let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu let q = (by >> ((e >> 4u) * 4u)) & 0xFu - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let d = unpackHalf2x16(ws[srow]).x - let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 2u + let w0 = ws[srow] + let d = unpackHalf2x16(w0).x + let sc = iq4xs_sc(w0, ws[srow + 1u], g) return float16(d * float(sc) * float(iq4lut[int(q)])) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkK4Blk; bc, cib : uint2) : half4 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5494,9 +5259,10 @@ class template Iq4xsCm2T : KqCm2BatchT { let l0 = uint(int(blk.qs[li])) & 0xFFFFu let l1 = uint(int(blk.qs[li + 1])) & 0xFFFFu let sh = (e >> 4u) * 4u - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let d = unpackHalf2x16(ws[srow]).x - let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 2u + let w0 = ws[srow] + let d = unpackHalf2x16(w0).x + let sc = iq4xs_sc(w0, ws[srow + 1u], g) let ds = d * float(sc) return half4(float4(ds * float(iq4lut[int((l0 >> sh) & 0xFu)]), ds * float(iq4lut[int((l0 >> (sh + 8u)) & 0xFu)]), ds * float(iq4lut[int((l1 >> sh) & 0xFu)]), ds * float(iq4lut[int((l1 >> (sh + 8u)) & 0xFu)]))) @@ -5509,7 +5275,7 @@ class template K3Cm2T : KqCm2BatchT { typedef ST = uint override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkK3Blk; bc, cib : uint2) : float16 { let e = cib.y let g = e >> 5u @@ -5525,7 +5291,7 @@ class template K3Cm2T : KqCm2BatchT { return float16(d * float(sc) * float(q3)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkK3Blk; bc, cib : uint2) : half4 { let e = cib.y let g = e >> 5u @@ -5555,7 +5321,7 @@ class template K2Cm2T : KqCm2BatchT { typedef ST = uint override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkK2Blk; bc, cib : uint2) : float16 { let e = cib.y let g = e >> 4u @@ -5569,7 +5335,7 @@ class template K2Cm2T : KqCm2BatchT { return float16(dm.x * float(pb & 15u) * float(q) - dm.y * float(pb >> 4u)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkK2Blk; bc, cib : uint2) : half4 { let e = cib.y let g = e >> 4u @@ -5595,7 +5361,7 @@ class template Iq3sCm2T : KqCm2BatchT { override IQ3GRID = true override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkIq3sBlk; bc, cib : uint2) : float16 { let e = cib.y let g = e >> 5u @@ -5608,13 +5374,13 @@ class template Iq3sCm2T : KqCm2BatchT { let sgb = uint(int(unpack8(blk.sg[int(si >> 1u)])[int(si & 1u)])) & 0xFFu let gw = iq3s_gridc[qb | ((qh << (8u - wj)) & 256u)] let gb = float((gw >> ((r & 3u) * 8u)) & 0xFFu) - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 2u let d = unpackHalf2x16(ws[srow]).x - let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + let sc = grid_sc(ws[srow + 1u], g) return float16(d * float(sc) * (((sgb >> (((r >> 2u) & 1u) * 4u + (r & 3u))) & 1u) != 0u ? -gb : gb)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkIq3sBlk; bc, cib : uint2) : half4 { let e = cib.y let g = e >> 5u @@ -5626,9 +5392,9 @@ class template Iq3sCm2T : KqCm2BatchT { let si = g * 4u + (r >> 3u) let sgb = uint(int(unpack8(blk.sg[int(si >> 1u)])[int(si & 1u)])) & 0xFFu let gw = iq3s_gridc[qb | ((qh << (8u - wj)) & 256u)] - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 2u let d = unpackHalf2x16(ws[srow]).x - let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + let sc = grid_sc(ws[srow + 1u], g) let ds = d * float(sc) let sgn = sgb >> (((r >> 2u) & 1u) * 4u) let g0 = float(gw & 0xFFu) @@ -5647,7 +5413,7 @@ class template Iq3xxsCm2T : KqCm2BatchT { override IQ3XGRID = true override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkIq3xxsBlk; bc, cib : uint2) : float16 { let e = cib.y let g = e >> 5u @@ -5667,13 +5433,13 @@ class template Iq3xxsCm2T : KqCm2BatchT { let sgb = sidx | ((tt & 1u) << 7u) let gw = iq3x_gridc[qb] let gb = float((gw >> ((r & 3u) * 8u)) & 0xFFu) - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 2u let d = unpackHalf2x16(ws[srow]).x - let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + let sc = grid_sc(ws[srow + 1u], g) return float16(d * float(sc) * (((sgb >> (r & 7u)) & 1u) != 0u ? -gb : gb)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkIq3xxsBlk; bc, cib : uint2) : half4 { let e = cib.y let g = e >> 5u @@ -5692,9 +5458,9 @@ class template Iq3xxsCm2T : KqCm2BatchT { tt = tt ^ (tt >> 1u) let sgb = sidx | ((tt & 1u) << 7u) let gw = iq3x_gridc[qb] - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 2u let d = unpackHalf2x16(ws[srow]).x - let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + let sc = grid_sc(ws[srow + 1u], g) let ds = d * float(sc) let sgn = sgb >> (r & 7u) let g0 = float(gw & 0xFFu) @@ -5713,7 +5479,7 @@ class template Iq4nlCm2T : KqCm2BatchT { override IQLUT = true override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5726,7 +5492,7 @@ class template Iq4nlCm2T : KqCm2BatchT { return float16(d * float(iq4lut[int(q)])) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkK4Blk; bc, cib : uint2) : half4 { let g = cib.y >> 5u let e = cib.y & 31u @@ -5749,7 +5515,7 @@ class template Iq2sCm2T : KqCm2BatchT { override IQ2SGRID = true override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkIq2sBlk; bc, cib : uint2) : float16 { let e = cib.y let g = e >> 5u @@ -5769,7 +5535,7 @@ class template Iq2sCm2T : KqCm2BatchT { return float16(d * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkIq2sBlk; bc, cib : uint2) : half4 { let e = cib.y let g = e >> 5u @@ -5802,7 +5568,7 @@ class template Iq2xsCm2T : KqCm2BatchT { override IQ2XSGRID = true override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkIq2xsBlk; bc, cib : uint2) : float16 { let e = cib.y let wi = e >> 3u @@ -5822,7 +5588,7 @@ class template Iq2xsCm2T : KqCm2BatchT { return float16(dv * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkIq2xsBlk; bc, cib : uint2) : half4 { let e = cib.y let wi = e >> 3u @@ -5855,7 +5621,7 @@ class template Iq2xxsCm2T : KqCm2BatchT { override IQ2XXSGRID = true override DECV4 = true - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode(blk : VkIq2xxsBlk; bc, cib : uint2) : float16 { let e = cib.y let g = e >> 5u @@ -5875,13 +5641,13 @@ class template Iq2xxsCm2T : KqCm2BatchT { let sgb = sidx | ((tt & 1u) << 7u) let gw = iq2xxs_gridc[qb * 2u + (j >> 2u)] let gb = float((gw >> ((j & 3u) * 8u)) & 0xFFu) - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 2u let dv = unpackHalf2x16(ws[srow]).x - let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + let sc = grid_sc(ws[srow + 1u], g) return float16(dv * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) } - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-decode-16bit-lanes")] def decode_v4(blk : VkIq2xxsBlk; bc, cib : uint2) : half4 { let e = cib.y let g = e >> 5u @@ -5900,9 +5666,9 @@ class template Iq2xxsCm2T : KqCm2BatchT { tt = tt ^ (tt >> 1u) let sgb = sidx | ((tt & 1u) << 7u) let gw = iq2xxs_gridc[qb * 2u + (j >> 2u)] - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 2u let dv = unpackHalf2x16(ws[srow]).x - let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + let sc = grid_sc(ws[srow + 1u], g) let ds = dv * float(sc) let sgn = sgb >> j let g0 = float(gw & 0xFFu) @@ -6293,13 +6059,145 @@ class Iq2xxsCm2SBatch : Iq2xxsCm2T { typedef FLO = tensorLayout2DPad } -// ===== the split-k reduce (sums the cm2 partial planes into y) ===== +// the KHR arm of every kq format (KHR, BN 128); the cm2 typedefs belong to the template's uncalled +// body and are never materialized +[vk_dispatch(name = "kq_batch_k4_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K4KhrBatch : K4Cm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k5_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K5KhrBatch : K5Cm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k6_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K6KhrBatch : K6Cm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_q40_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q40KhrBatch : Q40Cm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4xs_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4xsKhrBatch : Iq4xsCm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k3_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K3KhrBatch : K3Cm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3s_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3sKhrBatch : Iq3sCm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3xxs_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3xxsKhrBatch : Iq3xxsCm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4nl_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4nlKhrBatch : Iq4nlCm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k2_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K2KhrBatch : K2Cm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2s_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2sKhrBatch : Iq2sCm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xs_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xsKhrBatch : Iq2xsCm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xxs_khr_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xxsKhrBatch : Iq2xxsCm2T { + override KHR = true + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} struct SkRedArgs { nelem : uint // one partial plane's float count (cnt x d; always a 4-multiple, d is 32-aligned) k : uint // planes } +// ===== the split-k reduce (sums the cm2 partial planes into y) ===== + // 4 floats per thread, like the elementwise family [vk_dispatch(name = "splitk_reduce_cls", grid = "wgs", params = "wgs : int64")] class SplitKReduce { @@ -6393,6 +6291,8 @@ struct RopeKvArgs { kpair0 : uint // first k pair's global index (q pairs come first) npairs : uint layerbase : uint // this layer's K/V mirror base; the row rides tok[0].pos + boff : uint // this layer's row base in the bias buffer ([q | k | v] rows, one layer after another) + hasb : uint } // fused rope + KV-row store (decode, one token): one thread per rotation pair, each owning @@ -6406,6 +6306,7 @@ class template RopeKvStoreT { @ssbo @binding = 3 vmir : array // the V mirror plane @ssbo @binding = 4 cossin : array // this position's cos row then sin row @ssbo @binding = 5 tok : array // the shared per-token block (element 0) + @ssbo @binding = 6 @role = "weight" bias : array // per-layer [q | k | v] projection biases (hasb), else a placeholder @push_constant pa : RopeKvArgs @template_constant CLAMP16 : bool = true @@ -6424,25 +6325,27 @@ class template RopeKvStoreT { if (is_k) { let mrow = pa.layerbase + tok[0u].pos * pa.kvd let kb = pa.qd + h * pa.hs - let a0 = kvsrc[kb + e0] - let a1 = kvsrc[kb + e1] - // v is not roped — copy the same two elements raw into the V plane + let a0 = kvsrc[kb + e0] + (pa.hasb != 0u ? bias[pa.boff + kb + e0] : 0.0) + let a1 = kvsrc[kb + e1] + (pa.hasb != 0u ? bias[pa.boff + kb + e1] : 0.0) + // v is not roped — copy the same two elements raw (plus their bias) into the V plane let vb = pa.qd + pa.kvd + h * pa.hs + let v0 = kvsrc[vb + e0] + (pa.hasb != 0u ? bias[pa.boff + vb + e0] : 0.0) + let v1 = kvsrc[vb + e1] + (pa.hasb != 0u ? bias[pa.boff + vb + e1] : 0.0) static_if (CLAMP16) { kmir[mrow + h * pa.hs + e0] = float16(clamp(a0 * fcr - a1 * fci, -65504.0, 65504.0)) kmir[mrow + h * pa.hs + e1] = float16(clamp(a0 * fci + a1 * fcr, -65504.0, 65504.0)) - vmir[mrow + h * pa.hs + e0] = float16(clamp(kvsrc[vb + e0], -65504.0, 65504.0)) - vmir[mrow + h * pa.hs + e1] = float16(clamp(kvsrc[vb + e1], -65504.0, 65504.0)) + vmir[mrow + h * pa.hs + e0] = float16(clamp(v0, -65504.0, 65504.0)) + vmir[mrow + h * pa.hs + e1] = float16(clamp(v1, -65504.0, 65504.0)) } else { kmir[mrow + h * pa.hs + e0] = a0 * fcr - a1 * fci kmir[mrow + h * pa.hs + e1] = a0 * fci + a1 * fcr - vmir[mrow + h * pa.hs + e0] = kvsrc[vb + e0] - vmir[mrow + h * pa.hs + e1] = kvsrc[vb + e1] + vmir[mrow + h * pa.hs + e0] = v0 + vmir[mrow + h * pa.hs + e1] = v1 } } else { let qb = h * pa.hs - let a0 = qrows[qb + e0] - let a1 = qrows[qb + e1] + let a0 = qrows[qb + e0] + (pa.hasb != 0u ? bias[pa.boff + qb + e0] : 0.0) + let a1 = qrows[qb + e1] + (pa.hasb != 0u ? bias[pa.boff + qb + e1] : 0.0) qrows[qb + e0] = a0 * fcr - a1 * fci qrows[qb + e1] = a0 * fci + a1 * fcr } @@ -6476,6 +6379,8 @@ struct RopeKvBArgs { voff : uint kstride : uint qhs : uint // per-head q stride: hs, or 2 * hs on a gated model ([q | gate] per head, the gate half left in place) + boff : uint // this layer's row base in the bias buffer ([q | k | v] rows, one layer after another) + hasb : uint } // batched rope + KV-store (prefill, npos positions): q roped in place, k/v from the projection @@ -6487,6 +6392,7 @@ class template RopeKvStoreBT { @ssbo @binding = 2 kmir : array @ssbo @binding = 3 vmir : array @ssbo @binding = 4 cossin : array // per-position cos row then sin row (hs floats each) + @ssbo @binding = 5 @role = "weight" bias : array // per-layer [q | k | v] projection biases (hasb), else a placeholder @push_constant pa : RopeKvBArgs @template_constant CLAMP16 : bool = true @@ -6511,28 +6417,31 @@ class template RopeKvStoreBT { let e0 = rot ? (pa.neox != 0u ? j : j * 2u) : 2u * pa.half + 2u * (j - pa.half) let e1 = rot ? (pa.neox != 0u ? j + pa.half : j * 2u + 1u) : e0 + 1u let apos = pa.pos0 + p + let bhead = h * pa.hs if (is_k) { let kb = p * pa.kstride + h * pa.hs - let a0 = kvrows[kb + e0] - let a1 = kvrows[kb + e1] + let a0 = kvrows[kb + e0] + (pa.hasb != 0u ? bias[pa.boff + pa.qd + bhead + e0] : 0.0) + let a1 = kvrows[kb + e1] + (pa.hasb != 0u ? bias[pa.boff + pa.qd + bhead + e1] : 0.0) let mo = pa.layerbase + apos * pa.kvd + h * pa.hs let vb = pa.voff + p * pa.kstride + h * pa.hs + let v0 = kvrows[vb + e0] + (pa.hasb != 0u ? bias[pa.boff + pa.qd + pa.kvd + bhead + e0] : 0.0) + let v1 = kvrows[vb + e1] + (pa.hasb != 0u ? bias[pa.boff + pa.qd + pa.kvd + bhead + e1] : 0.0) static_if (CLAMP16) { kmir[mo + e0] = float16(clamp(a0 * fcr - a1 * fci, -65504.0, 65504.0)) kmir[mo + e1] = float16(clamp(a0 * fci + a1 * fcr, -65504.0, 65504.0)) - vmir[mo + e0] = float16(clamp(kvrows[vb + e0], -65504.0, 65504.0)) - vmir[mo + e1] = float16(clamp(kvrows[vb + e1], -65504.0, 65504.0)) + vmir[mo + e0] = float16(clamp(v0, -65504.0, 65504.0)) + vmir[mo + e1] = float16(clamp(v1, -65504.0, 65504.0)) } else { kmir[mo + e0] = a0 * fcr - a1 * fci kmir[mo + e1] = a0 * fci + a1 * fcr - vmir[mo + e0] = kvrows[vb + e0] - vmir[mo + e1] = kvrows[vb + e1] + vmir[mo + e0] = v0 + vmir[mo + e1] = v1 } - } elif (rot) { + } elif (rot || pa.hasb != 0u) { // a pass-through q pair (fcr 1, fci 0) is rewritten only when a bias lands on it let qhs = pa.qhs == 0u ? pa.hs : pa.qhs // 0 = compact (non-gated); a gated model passes 2 * hs let qb = p * (pa.qd / pa.hs) * qhs + h * qhs - let a0 = qrows[qb + e0] - let a1 = qrows[qb + e1] + let a0 = qrows[qb + e0] + (pa.hasb != 0u ? bias[pa.boff + bhead + e0] : 0.0) + let a1 = qrows[qb + e1] + (pa.hasb != 0u ? bias[pa.boff + bhead + e1] : 0.0) qrows[qb + e0] = a0 * fcr - a1 * fci qrows[qb + e1] = a0 * fci + a1 * fcr } @@ -6564,6 +6473,8 @@ struct QknRopeArgs { kwoff : uint eps : float qstride : uint // q head stride in qrows: 0 = hs; 2*hs on a gated-q model ([q | gate] per head, roped in place) + boff : uint // this layer's row base in the bias buffer ([q | k | v] rows, one layer after another) + hasb : uint } // qk_rms + rope_kv_store in ONE dispatch (qk_norm models), one wg per head-row; verbatim @@ -6578,6 +6489,7 @@ class template QknRopeKvT : RmsWgBase { @ssbo @binding = 5 wn : array // the q/k norm weight rows (its OWN member now — // the old kernel packed them into the cos buffer) @ssbo @binding = 6 tok : array // the shared per-token block (element 0) + @ssbo @binding = 7 @role = "weight" bias : array // per-layer [q | k | v] projection biases (hasb), else a placeholder @push_constant pa : QknRopeArgs @workgroup row : float[256] // this head's normed q or k row (hs <= 256) @template_constant CLAMP16 : bool = true @@ -6590,9 +6502,10 @@ class template QknRopeKvT : RmsWgBase { let tid = gl_LocalInvocationID.x let qstride = pa.qstride == 0u ? pa.hs : pa.qstride let sbase = is_k ? pa.qd + h * pa.hs : h * qstride + let bbase = is_k ? pa.qd + h * pa.hs : h * pa.hs var v = 0.0 if (tid < pa.hs) { - v = is_k ? kvsrc[sbase + tid] : qrows[sbase + tid] + v = (is_k ? kvsrc[sbase + tid] : qrows[sbase + tid]) + (pa.hasb != 0u ? bias[pa.boff + bbase + tid] : 0.0) } let ss = tid < pa.hs ? v * v : 0.0 let inv = wg_rms_inv(ss, pa.hs, pa.eps) @@ -6627,16 +6540,18 @@ class template QknRopeKvT : RmsWgBase { if (!is_k && tid >= 2u * pa.half && tid < pa.hs) { qrows[h * qstride + tid] = row[tid] } - // v is not roped or normed — the k-head wgs copy their own raw row into the V plane + // v is not roped or normed — the k-head wgs copy their own raw row (plus its bias) into the V plane if (is_k && tid < pa.hs) { let mrow = pa.layerbase + tok[0u].pos * pa.kvd + let vi = pa.qd + pa.kvd + h * pa.hs + tid + let vv = kvsrc[vi] + (pa.hasb != 0u ? bias[pa.boff + vi] : 0.0) static_if (CLAMP16) { - vmir[mrow + h * pa.hs + tid] = float16(clamp(kvsrc[pa.qd + pa.kvd + h * pa.hs + tid], -65504.0, 65504.0)) + vmir[mrow + h * pa.hs + tid] = float16(clamp(vv, -65504.0, 65504.0)) if (tid >= 2u * pa.half) { kmir[mrow + h * pa.hs + tid] = float16(clamp(row[tid], -65504.0, 65504.0)) } } else { - vmir[mrow + h * pa.hs + tid] = kvsrc[pa.qd + pa.kvd + h * pa.hs + tid] + vmir[mrow + h * pa.hs + tid] = vv if (tid >= 2u * pa.half) { kmir[mrow + h * pa.hs + tid] = row[tid] } @@ -6726,6 +6641,45 @@ class Q8Gemv { let private KQC_LOW = 0x0F0F0F0Fu // nibble mask let private KQC_ONES = 0x01010101u // sdot4 against this sums 4 signed bytes +let KQ_GRID_IQ2S_WORDS = 2048u +let KQ_GRID_IQ2XS_WORDS = 1024u +let KQ_GRID_IQ2XXS_WORDS = 512u +let KQ_GRID_IQ3S_WORDS = 512u +let KQ_GRID_IQ3XXS_WORDS = 256u +let KQ_GRID_IQ2S = 0u +let KQ_GRID_IQ2XS = KQ_GRID_IQ2S + KQ_GRID_IQ2S_WORDS +let KQ_GRID_IQ2XXS = KQ_GRID_IQ2XS + KQ_GRID_IQ2XS_WORDS +let KQ_GRID_IQ3S = KQ_GRID_IQ2XXS + KQ_GRID_IQ2XXS_WORDS +let KQ_GRID_IQ3XXS = KQ_GRID_IQ3S + KQ_GRID_IQ3S_WORDS +let KQ_GRID_WORDS = int(KQ_GRID_IQ3XXS + KQ_GRID_IQ3XXS_WORDS) +let KQ_GRID_BYTES = int64(KQ_GRID_WORDS) * 4l + +[arch(at="../ARCHITECTURE_GPU_VULKAN.md#kq-gemv-grid-buffer")] +def kq_grid_dev() : uint64 { + if (g_gpu.kq_grid_dev == 0ul) { + var words : array + words |> resize(KQ_GRID_WORDS) + for (i in range(int(KQ_GRID_IQ2S_WORDS))) { + words[int(KQ_GRID_IQ2S) + i] = iq2s_grid_word(i) + } + for (i in range(int(KQ_GRID_IQ2XS_WORDS))) { + words[int(KQ_GRID_IQ2XS) + i] = iq2xs_grid_word(i) + } + for (i in range(int(KQ_GRID_IQ2XXS_WORDS))) { + words[int(KQ_GRID_IQ2XXS) + i] = iq2xxs_grid_word(i) + words[int(KQ_GRID_IQ3S) + i] = iq3s_grid_word(i) + } + for (i in range(int(KQ_GRID_IQ3XXS_WORDS))) { + words[int(KQ_GRID_IQ3XXS) + i] = iq3xxs_grid_word(i) + } + g_gpu.kq_grid_dev = make_device_buf(KQ_GRID_BYTES) + unsafe { + upload_region_at(g_gpu.kq_grid_dev, 0l, addr(words[0]), KQ_GRID_BYTES) + } + } + return g_gpu.kq_grid_dev +} + class KqGemvBase { @ssbo @binding = 0 @role = "alias" wq : array // weight quant words (k5's qh rides this view) @ssbo @binding = 0 wq4 : array // vec4 view of the same words @@ -6734,6 +6688,7 @@ class KqGemvBase { @ssbo @binding = 3 xq4 : array // activation quant words, vec4 view @ssbo @binding = 4 axs : array // activation block scales @ssbo @binding = 5 y : array // output rows + @ssbo @binding = 6 @role = "weight" gridb : array // the grid codebooks (kq_grid_dev; the grid formats stage from it) @push_constant pa : GemvArgs // per-format: one 32-block's scaled contribution — every [spirv_kernel] variant overrides @@ -6796,11 +6751,11 @@ class KqGemvBase { // variant: 0 = sdot4 tile, 1 = cmf16, 2 = cmi8, 3 = mm M-tile, 4 = mm aligned-L, 5 = mm L def q8_batch_cls_variant(tile : int64; aligned : bool) : int { - if (g_gpu.coopmat_mode == 1) { + if (g_gpu.coopmat_mode == COOPMAT_F16) { return 1 - } elif (g_gpu.coopmat_mode == 2) { + } elif (g_gpu.coopmat_mode == COOPMAT_INT8) { return 2 - } elif (g_gpu.coopmat_mode >= 3) { + } elif (g_gpu.coopmat_mode >= COOPMAT_MM) { if (tile == BATCH_TILE) { // the small tier IS the sdot4 kernel — measured, not a fallback return 0 } @@ -6987,7 +6942,10 @@ def gemv_cls_set(fmt : int; bufs : uint64 const[6]; sizes : int64 const[6]; gbit if (fmt == int(KqFmt.q8)) { return set_q8_gemv_cls(bufs, sizes, gbits) } elif (kq_sb(fmt)) { - return set_kq_gemv_cls(bufs, sizes, gbits) // the family shares one layout — one set serves every kq format + // the family shares one layout — one set serves every kq format + return set_kq_gemv_cls(fixed_array(bufs[0], bufs[1], bufs[2], bufs[3], bufs[4], bufs[5], kq_grid_dev()), + fixed_array(sizes[0], sizes[1], sizes[2], sizes[3], sizes[4], sizes[5], KQ_GRID_BYTES), + fixed_array(gbits[0], gbits[1], gbits[2], gbits[3], gbits[4], gbits[5], 0u)) } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return VkDescriptorSet() @@ -7046,6 +7004,14 @@ class KqGemvK4 : KqGemvBase { } } +//! the codebook nibble dot: eight sdot4 over the LUT-decoded low and high nibbles of a 32-quant block +def iq4_dot(wv, xl, xh : uint4) : int { + var idot = sdot4(iq4_word(wv.x & KQC_LOW), xl.x) + sdot4(iq4_word(wv.y & KQC_LOW), xl.y) + idot += sdot4(iq4_word(wv.z & KQC_LOW), xl.z) + sdot4(iq4_word(wv.w & KQC_LOW), xl.w) + idot += sdot4(iq4_word((wv.x >> 4u) & KQC_LOW), xh.x) + sdot4(iq4_word((wv.y >> 4u) & KQC_LOW), xh.y) + return idot + sdot4(iq4_word((wv.z >> 4u) & KQC_LOW), xh.z) + sdot4(iq4_word((wv.w >> 4u) & KQC_LOW), xh.w) +} + // Q4_0: w = d*(q - 8) — the k4 nibble tiling with no min plane; -8 folds against the block sum [vk_dispatch(name = "kq_gemv_q40_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvQ40 : KqGemvBase { @@ -7072,18 +7038,12 @@ class KqGemvQ40 : KqGemvBase { // lanes (no block-sum term), the k4 scale-row shape with SIGNED (ls - 32) sub-scales and no min [vk_dispatch(name = "kq_gemv_iq4xs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvIq4xs : KqGemvBase { - def iq4_dot(wv, xl, xh : uint4) : int { - var idot = sdot4(iq4_word(wv.x & KQC_LOW), xl.x) + sdot4(iq4_word(wv.y & KQC_LOW), xl.y) - idot += sdot4(iq4_word(wv.z & KQC_LOW), xl.z) + sdot4(iq4_word(wv.w & KQC_LOW), xl.w) - idot += sdot4(iq4_word((wv.x >> 4u) & KQC_LOW), xh.x) + sdot4(iq4_word((wv.y >> 4u) & KQC_LOW), xh.y) - return idot + sdot4(iq4_word((wv.z >> 4u) & KQC_LOW), xh.z) + sdot4(iq4_word((wv.w >> 4u) & KQC_LOW), xh.w) - } - def override blk_contrib(wsb, blk, xsb : uint) : float { let xb4 = xsb * 16u + blk * 2u let idot = iq4_dot(wq4[wsb * 8u + blk], xq4[xb4], xq4[xb4 + 1u]) - let dd = unpackHalf2x16(wsu[wsb * 5u]).x - return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * idot) + let w0 = wsu[wsb * 2u] + let dd = unpackHalf2x16(w0).x + return axs[xsb] * dd * float(iq4xs_sc(w0, wsu[wsb * 2u + 1u], blk) * idot) } [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq4xs_cls_spv")] @@ -7099,12 +7059,6 @@ class KqGemvIq4xs : KqGemvBase { class KqGemvIq3s : KqGemvBase { @workgroup gridw : uint[512] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override blk_contrib(wsb, blk, xsb : uint) : float { let qb = wsb * 26u + blk * 2u let qh = (wq[wsb * 26u + 16u + blk / 4u] >> ((blk % 4u) * 8u)) & 255u @@ -7128,8 +7082,8 @@ class KqGemvIq3s : KqGemvBase { isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) } } - let dd = unpackHalf2x16(wsu[wsb * 5u]).x - return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + let dd = unpackHalf2x16(wsu[wsb * 2u]).x + return axs[xsb] * dd * float(grid_sc(wsu[wsb * 2u + 1u], blk) * isum) } [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq3s_cls_spv")] @@ -7137,7 +7091,7 @@ class KqGemvIq3s : KqGemvBase { let lid = gl_LocalInvocationID.x for (k in range(8)) { let idx = lid + uint(k) * 64u - gridw[idx] = iq3s_grid_word(int(idx)) + gridw[idx] = gridb[KQ_GRID_IQ3S + idx] } barrier() // the grid visible before the shell's block walk gemv_shell() @@ -7152,12 +7106,6 @@ class KqGemvIq3s : KqGemvBase { class KqGemvIq3xxs : KqGemvBase { @workgroup gridw : uint[256] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override blk_contrib(wsb, blk, xsb : uint) : float { let qb = wsb * 24u + blk * 2u let aux = wq[wsb * 24u + 16u + blk] @@ -7180,8 +7128,8 @@ class KqGemvIq3xxs : KqGemvBase { isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) } } - let dd = unpackHalf2x16(wsu[wsb * 5u]).x - return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + let dd = unpackHalf2x16(wsu[wsb * 2u]).x + return axs[xsb] * dd * float(grid_sc(wsu[wsb * 2u + 1u], blk) * isum) } [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq3xxs_cls_spv")] @@ -7189,7 +7137,7 @@ class KqGemvIq3xxs : KqGemvBase { let lid = gl_LocalInvocationID.x for (k in range(4)) { let idx = lid + uint(k) * 64u - gridw[idx] = iq3xxs_grid_word(int(idx)) + gridw[idx] = gridb[KQ_GRID_IQ3XXS + idx] } barrier() // the grid visible before the shell's block walk gemv_shell() @@ -7199,19 +7147,7 @@ class KqGemvIq3xxs : KqGemvBase { // IQ4_NL: iq4xs's codebook dot (iq4_word, pure ALU) folded by q40's per-block f16 d - no // sub-scales, no bsum term; the rows are q40's 32-word planes verbatim. [vk_dispatch(name = "kq_gemv_iq4nl_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] -class KqGemvIq4nl : KqGemvBase { - def iq4_dot(wv, xl, xh : uint4) : int { - var idot = sdot4(iq4_word(wv.x & KQC_LOW), xl.x) + sdot4(iq4_word(wv.y & KQC_LOW), xl.y) - idot += sdot4(iq4_word(wv.z & KQC_LOW), xl.z) + sdot4(iq4_word(wv.w & KQC_LOW), xl.w) - idot += sdot4(iq4_word((wv.x >> 4u) & KQC_LOW), xh.x) + sdot4(iq4_word((wv.y >> 4u) & KQC_LOW), xh.y) - return idot + sdot4(iq4_word((wv.z >> 4u) & KQC_LOW), xh.z) + sdot4(iq4_word((wv.w >> 4u) & KQC_LOW), xh.w) - } - - def q40_d(wsb, blk : uint) : float { - let dp = unpackHalf2x16(wsu[wsb * 5u + blk / 2u]) - return blk % 2u == 0u ? dp.x : dp.y - } - +class KqGemvIq4nl : KqGemvQ40 { def override blk_contrib(wsb, blk, xsb : uint) : float { let xb4 = xsb * 16u + blk * 2u let idot = iq4_dot(wq4[wsb * 8u + blk], xq4[xb4], xq4[xb4 + 1u]) @@ -7219,7 +7155,7 @@ class KqGemvIq4nl : KqGemvBase { } [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq4nl_cls_spv")] - def run { + def override run { gemv_shell() } } @@ -7381,12 +7317,6 @@ class KqGemvK6 : KqGemvBase { class KqGemvIq2s : KqGemvBase { @workgroup gridw : uint[2048] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override blk_contrib(wsb, blk, xsb : uint) : float { let qw = wq[wsb * 18u + blk] let sgw = wq[wsb * 18u + 8u + blk] @@ -7423,7 +7353,7 @@ class KqGemvIq2s : KqGemvBase { let lid = gl_LocalInvocationID.x for (k in range(32)) { let idx = lid + uint(k) * 64u - gridw[idx] = iq2s_grid_word(int(idx)) + gridw[idx] = gridb[KQ_GRID_IQ2S + idx] } barrier() // the grid visible before the shell's block walk gemv_shell() @@ -7437,12 +7367,6 @@ class KqGemvIq2s : KqGemvBase { class KqGemvIq2xs : KqGemvBase { @workgroup gridw : uint[1024] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override blk_contrib(wsb, blk, xsb : uint) : float { let xb4 = xsb * 16u + blk * 2u let xl = xq4[xb4] @@ -7478,7 +7402,7 @@ class KqGemvIq2xs : KqGemvBase { let lid = gl_LocalInvocationID.x for (k in range(16)) { let idx = lid + uint(k) * 64u - gridw[idx] = iq2xs_grid_word(int(idx)) + gridw[idx] = gridb[KQ_GRID_IQ2XS + idx] } barrier() // the iq2xs grid visible before the shell's block walk gemv_shell() @@ -7492,12 +7416,6 @@ class KqGemvIq2xs : KqGemvBase { class KqGemvIq2xxs : KqGemvBase { @workgroup gridw : uint[512] - def iq3s_signed(g, nib : uint) : uint { - let m1 = (nib * 0x00204081u) & 0x01010101u - let mask = m1 * 255u - return (g ^ mask) + m1 - } - def override blk_contrib(wsb, blk, xsb : uint) : float { let gw = wq[wsb * 16u + blk * 2u] let aux = wq[wsb * 16u + blk * 2u + 1u] @@ -7521,8 +7439,8 @@ class KqGemvIq2xxs : KqGemvBase { isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) } } - let dd = unpackHalf2x16(wsu[wsb * 5u]).x - return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + let dd = unpackHalf2x16(wsu[wsb * 2u]).x + return axs[xsb] * dd * float(grid_sc(wsu[wsb * 2u + 1u], blk) * isum) } [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq2xxs_cls_spv")] @@ -7530,7 +7448,7 @@ class KqGemvIq2xxs : KqGemvBase { let lid = gl_LocalInvocationID.x for (k in range(8)) { let idx = lid + uint(k) * 64u - gridw[idx] = iq2xxs_grid_word(int(idx)) + gridw[idx] = gridb[KQ_GRID_IQ2XXS + idx] } barrier() // the iq2xxs grid visible before the shell's block walk gemv_shell() diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index fdaec13854..3ce76d82d2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -63,9 +63,17 @@ let BATCH_Y_BYTES = 67_108_864l // one output plane (two exist: gate/single + let BATCH_META_BYTES = 262_144l // per-stack batch params + regions + workgroup map let BATCH_TILE = 32l // kernel tile edge (rows and cols) -// the Q8_0 batch kernel's tile edge — 128 for the mul_mm L-tile kernel (modes 3/4), 32 otherwise. +//! The prefill GEMM's cooperative-matrix mode (GpuState.coopmat_mode): the sdot4 batch tile, the f16 or +//! int8 coopmat tiles, the mul_mm L-tile (the KHR arm), or the cm2 decode-in-load tiles. +let COOPMAT_SDOT4 = 0 +let COOPMAT_F16 = 1 +let COOPMAT_INT8 = 2 +let COOPMAT_MM = 3 +let COOPMAT_CM2 = 4 + +// the Q8_0 batch kernel's tile edge — 128 for the mul_mm L-tile kernel (modes mm and cm2), 32 otherwise. // Meta builders MUST use the same edge the bound kernel derives (wtiles = ceil(d / edge)). -def q8_batch_tile : int64 => g_gpu.coopmat_mode >= 3 ? 128l : BATCH_TILE +def q8_batch_tile : int64 => g_gpu.coopmat_mode >= COOPMAT_MM ? 128l : BATCH_TILE let MM_SMALL_CNT = 192l // measured crossover (qwen3-4B pp: sdot4 +42% at 128, L +2.5% at 192) @@ -97,7 +105,7 @@ def mm_small_d : int64 { // small-cnt or small-d dispatch drops to the small tier — the L tile starves the grid and pads // its B panel there. Pure in (d, cnt, mode), so pipe pick and meta fill can never disagree. def q8_batch_tile_for(d, cnt : int64) : int64 { - if (g_gpu.coopmat_mode < 3) { + if (g_gpu.coopmat_mode < COOPMAT_MM) { return q8_batch_tile() } return d <= mm_small_d() || cnt < MM_SMALL_CNT ? mm_small_tile() : 128l @@ -127,8 +135,14 @@ def cm2_splitk_env : int64 { } -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] +def khr_kq_tile_on : bool => g_gpu.has_coopmat && g_gpu.subgroup_size == 32l + +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def cm2_tile_cols(d, cnt : int64) : int64 { + if (g_gpu.coopmat_mode == COOPMAT_MM) { + return 128l + } let forced = cm2_tile_env() if (forced == 32l || forced == 128l || forced == 256l) { return forced @@ -155,12 +169,15 @@ def cm2_tile_cols(d, cnt : int64) : int64 { def cm2_tiles_at(d, cnt, tc : int64) : int64 => ((d + 127l) / 128l) * ((cnt + tc - 1l) / tc) //! workgroups a (d, cnt) GEMM takes at its own tile pick -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def cm2_tiles(d, cnt : int64) : int64 => cm2_tiles_at(d, cnt, cm2_tile_cols(d, cnt)) //! Returns (chunks, 256-aligned chunk); (1, 0) = off. `sib_tiles` = the workgroups of the roles this one co-runs beside. -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def cm2_split_k(d, cnt, n, tc : int64; sib_tiles : int64 = 0l) : tuple { + if (g_gpu.coopmat_mode == COOPMAT_MM) { + return (nsplit = 1l, ksplit = 0l) + } let forced = cm2_splitk_env() let cores = int64(g_gpu.sm_count) var sk = 1l @@ -207,7 +224,7 @@ var g_coopmat_mode_force = -1 //! Test hook: pin the GEMM mode (0 sdot4 / 1 f16 / 2 int8 / 3 mm / 4 cm2) ahead of the resolved default and //! DASLLAMA_COOPMAT — must run before the tier's lazy device init. The CPU-reference tier tests //! pin 0 so their oracles stay reference-kernel-deterministic under any router default. -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def vk_force_coopmat_mode(m : int) { g_coopmat_mode_force = m } @@ -218,8 +235,8 @@ let STREAM_RESERVE = 1_100_000_000l // LEGACY fallback stream-slot carve — us // GLM Air exceed this constant by design) // deltanet chain capacities (npos chunks over DN_WINDOW windows, state carried device-side). -// Maxima match qwen35 (ds 128, 32 v-heads, conv dim 8192, dconv 4) — vk_moe_dn asserts them -// and the engine arm fail-closes on bigger models. +// Maxima cover the qwen35 family through Qwen3.8-27B (ds 128, 48 v-heads, conv dim 10240, dconv 4) +// with headroom — vk_moe_dn asserts them and the engine arm fail-closes on bigger models. let DN_MAX_DS = 128l //! The sequential scan's workgroup count for a head count and state width @@ -231,8 +248,9 @@ def dn_scan_wgs(nvh, ds : int64) : int64 { //! The prefill's beta/alpha tile GEMM's workgroups: 16-position tiles x 16-output groups over the 2 x nvh rows. def dn_ba_wgs(npos, nvh : int64) : int64 => ((npos + 15l) / 16l) * ((2l * nvh + 15l) / 16l) -let DN_MAX_HEADS = 32l -let DN_MAX_CD = 8_192l +let DN_MAX_HEADS = 64l +let DN_MAX_CD = 16_384l +let DN_MAX_QK = 8_192l //! the conv row's q + k region (2 kd): the part dn_conv stages in workgroup memory for the per-head norm let DN_MAX_DCONV = 4l let DN_WINDOW = 1_024l //! the scan workspace holds the raw o rows alone: position-major, di wide, rewritten in place by the out-norm @@ -273,10 +291,10 @@ let HEAT_UPLOAD_CAP_FILL = 8 // while the pool is underfull (warmup fill let AT_WINDOW = 512l let AT_CTX = 4_096l let AT_MAX_QD = 8_192l -let AT_MAX_KV = 1_024l +let AT_MAX_KV = 2_048l // an MHA 16 x 128 layer (Qwen1.5-MoE) - the k/v planes and their f16 shadows grow with it, nothing in a workgroup does let AT_MAX_HS = 256l let AT_META_BYTES = 128l -let AR_MAX_DIM = 4_096l // the add+rms row cap — == the ar_row workgroup slab +let AR_MAX_DIM = 8_192l // the add-rms kernels' row stash: 32 KB of the 48 KiB workgroup memory the tier requires of a device // smalls plane float offsets: [eps, scale] consts, the per-head qk-rms weight rows, then the // window's cos/sin rope rows packed contiguously (sin offset = AT_SM_COS + rows*half, meta [13]) let AT_SM_RMSQ = 16l @@ -377,6 +395,8 @@ struct DatLayer { fv : int fo : int norms : uint64 // the layer's qk-norm rows (q then k, hs floats each) + bias : uint64 // the layer's [q | k | v] projection bias row (has_bias), else unread + has_bias : bool s_q : VkDescriptorSet s_k : VkDescriptorSet s_v : VkDescriptorSet @@ -481,15 +501,19 @@ struct GpuState { staging : HostBuf fence : VkFence // the one reusable submit fence (reset after every wait) rows_per_wg : int64 // subgroups per workgroup = output rows per workgroup + subgroup_size : int64 // the device's subgroupSize (32 on NVIDIA; the KHR kq tile gates on it) has_coopmat : bool // device supports VK_KHR_cooperative_matrix (the f16/int8 tensor tiles) has_coopmat2 : bool // device supports NV_cooperative_matrix2 (tensor-addressed wg tiles + decode fns) has_coopmat2_fa : bool // ... plus the cm2 flash-attention trio (reductions, conversions, per-element ops) has_bda : bool // the device enabled bufferDeviceAddress (the coopmat2 creator alone does): storage buffers carry the device-address usage, allocation flag and query decvec_on : bool // DASLLAMA_VK_DECVEC asked and the device reports the four-wide decode callback (a dry bake creates no pipelines and leaves it off) full_sg_on : bool // DASLLAMA_VK_FULLSG asked and the device has the feature: every class pipeline pins REQUIRE_FULL_SUBGROUPS - coopmat_mode : int // Q8_0 prefill GEMM: 0 = sdot4, 1 = f16 coopmat, 2 = int8 coopmat, 3 = mul_mm L-tile, 4 = cm2 decode-in-load (the default where the device has it, else 3) + coopmat_mode : int // the prefill GEMM mode, a COOPMAT_* value (cm2 where the device has it, else mm) sm_count : int // shaderSMCount (VK_NV_shader_sm_builtins; 0 = unknown -> l-tile always, no split-k) weight_budget : int64 // resident-weight cap — queried heap budget minus reserve, or VRAM_BUDGET + os_budget_known : bool // the OS video memory manager answered at init (Windows): the resident plan sizes against the room it reports + os_shared_base : int64 // the adapter's shared usage at init - the desktop's own, before any buffer of ours + host_bytes : int64 // every live host-visible buffer of ours (make_host_buf's tally) - shared usage past base + this is demotion dry : bool // OFFLINE BAKE: accept/refuse arithmetic runs, every device call is gated off msr : int64 // maxStorageBufferRange — device-queried at init, config-supplied when dry @do_not_delete stacks : array @@ -566,6 +590,7 @@ struct GpuState { ffn_pend_bytes : int64 qkv_cmds : table // q plane offset -> the layer's decode q/k/v group dev_mem : table // device buffer -> its VkDeviceMemory (rollback destroy needs it) + kq_grid_dev : uint64 // the grid formats' codebook buffer (kq_grid_dev builds it; model-owned, 0 = not yet) @scratch dev_mapped : table // ReBAR: device buffer -> persistent CPU mapping (uploads write direct) rebar_type : int = -1 // DL|HV|HC memory type on the full-VRAM heap (-1 = staging path only) has_memprio : bool // device enabled VK_EXT_memory_priority — allocations chain priority 1.0 @@ -645,6 +670,7 @@ struct GpuState { dnd_bg : HostBuf // per-token raw β/γ + ring-parity staging (copied into the layer smalls) dnd_oq_dev : uint64 // the o row's Q8_0 image (fused-step out, the out GEMV's activations) dnd_os_dev : uint64 + dnd_of_dev : uint64 // the step's f32 o row binding (unused on this rail: its out planes are q8) dnd_steps : table // qkv plane offset -> the layer's step chain dnd_ready : bool // attention chain state (lazy — first attention dispatch creates it) @@ -656,8 +682,10 @@ struct GpuState { at_v_dev : uint64 // raw v rows at absolute positions at_vraw_dev : uint64 // the v GEMM's window-relative output (device-copied into at_v_dev) at_kv_host : HostBuf // host-cached k/v readback (k at 0, v at AT_WINDOW*AT_MAX_KV floats) + at_bias_dev : uint64 // the biased layer's [q | k | v] projection bias row (hasb), uploaded per layer at_prep_q_set : VkDescriptorSet at_prep_k_set : VkDescriptorSet + at_prep_v_set : VkDescriptorSet // the v bias-add pass (half 0, qk-norm off): a copy plus bias, in place on at_vraw at_attn_set : VkDescriptorSet at_rq_set : VkDescriptorSet // the o-requant family set (the encoder picks q8k/q8 per window) at_ready : bool @@ -783,6 +811,7 @@ def make_host_buf(bytes : int64; storage : bool; cached : bool = false; xfer_sha vk_check(vkMapMemory(dev_raw(), boost_value_to_vk(m), 0ul, uint64(bytes), mf, addr(mapped)), null) } g_gpu.host_mem |> insert(b._vk, m._vk) + g_gpu.host_bytes += bytes return HostBuf(buf = b._vk, mem = m._vk, mapped = mapped, bytes = bytes) } @@ -868,6 +897,7 @@ def destroy_host_buf(hb : HostBuf) { // vkFreeMemory implicitly unmaps vkFreeMemory(dev_raw(), reinterpret(hb.mem), null) } g_gpu.host_mem |> erase(hb.buf) + g_gpu.host_bytes -= hb.bytes } def nonowning_buf(h : uint64) : Buffer { @@ -876,7 +906,7 @@ def nonowning_buf(h : uint64) : Buffer { return b } -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def resolve_coopmat_mode(has_cm, has_cm2 : bool) : int { var cmode = has_cm2 ? 4 : (has_cm ? 3 : 0) if (has_cm) { @@ -906,6 +936,10 @@ struct VkHeapInfo { budget : int64 // largest device-local heap's current budget (0 = no VK_EXT_memory_budget) used : int64 size : int64 + os_budget : int64 // the OS video memory manager's ceiling for this process (0 = no OS answer: not Windows, no LUID, query failed) + os_usage : int64 // ... what this process holds now + os_system : int64 // ... what every process holds on the adapter (the desktop's footprint is system minus usage) + os_shared : int64 // ... and the system memory the adapter maps for every process (host-visible buffers, demoted planes) } def query_heap_info(phys : VkPhysicalDevice) : VkHeapInfo { @@ -940,12 +974,50 @@ def query_heap_info(phys : VkPhysicalDevice) : VkHeapInfo { r.used = int64(mb.heapUsage[best]) } } + let os = os_video_memory(phys) + r.os_budget = int64(os.budget) + r.os_usage = int64(os.usage) + r.os_system = int64(os.adapter_dedicated) + r.os_shared = int64(os.adapter_shared) return r } +//! The OS's video memory numbers right now, in bytes (all 0 = no OS answer): this process's dedicated +//! ceiling, what it holds, what every process holds on the adapter, the system memory the adapter maps - +//! the resident plan's auto arm sizes against them on Windows, and the demotion check reads them. +def vk_os_memory_now() : tuple { + if (g_gpu == null || g_gpu.dry || !g_gpu.os_budget_known) { + return (budget = 0l, usage = 0l, adapter_dedicated = 0l, adapter_shared = 0l) + } + let os = os_video_memory(g_gpu.phys) + return (budget = int64(os.budget), usage = int64(os.usage), adapter_dedicated = int64(os.adapter_dedicated), adapter_shared = int64(os.adapter_shared)) +} + +var private g_demotion_said = false +var private g_demotion_ticks = 0l // the last check's clock +let private DEMOTION_SLACK = 536_870_912l // shared growth the check ignores: transient staging, the desktop's own drift +let private DEMOTION_PERIOD_US = 1_000_000 // the counter query costs milliseconds - once a second, never per prefill of a small model + +//! Log once when the adapter's shared usage has grown past the desktop's baseline and our own +//! host-visible buffers - the manager has then demoted our planes to system memory, and every +//! weight read runs at PCIe speed with no error. +[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN_RESIDENCY.md#resident-plan")] +def vk_demotion_check(site : string) { + if (g_demotion_said || (g_demotion_ticks != 0l && get_time_usec(g_demotion_ticks) < DEMOTION_PERIOD_US)) { + return + } + g_demotion_ticks = ref_time_ticks() // clock: control - the once-a-second rate limit, never a reported figure + let os = vk_os_memory_now() + let demoted = os.adapter_shared - g_gpu.os_shared_base - g_gpu.host_bytes + if (os.budget > 0l && demoted > DEMOTION_SLACK) { + g_demotion_said = true + to_log(LOG_WARNING, "dasLLAMA vulkan tier: {site}: about {demoted / 1_000_000l} MB of this process's planes sit in system memory (the adapter maps {os.adapter_shared / 1_000_000l} MB of it, {os.adapter_dedicated / 1_000_000l} MB dedicated held by every process) - the driver demoted them and decode runs at PCIe speed; close other GPU applications, or pin DASLLAMA_GPU_VRAM_MB lower\n") + } +} + // The pre-carve weight cap, shared by vk_moe_init and the DlimConfiguration source. Env wins // outright (=0 pins WEIGHT_CAP — the misreporting-driver hatch); auto derives from heap SIZE, a -// hardware constant — same number every run. Transient pressure is the memprio shield's job. +// hardware constant — same number every run. Transient pressure is the resident plan's headroom. def resolve_weight_cap(heap : VkHeapInfo) : int64 { let vram_mb = gpu_want_vram_mb() if (vram_mb > 0l) { @@ -982,7 +1054,7 @@ def carved_budget : int64 { //! The streamed-prefill slot count: two (compute on one while the transfer queue fills the other) //! on the whole-group rail; ONE where the split can engage (the knob on, a cm2 device), whose on-demand //! partial copies never prefetch - a device the split cannot serve keeps both. -def stream_slot_count : int => gpu_want_moe_split() && g_gpu != null && g_gpu.has_coopmat2 && g_gpu.coopmat_mode == 4 ? 1 : 2 +def stream_slot_count : int => gpu_want_moe_split() && g_gpu != null && g_gpu.has_coopmat2 && g_gpu.coopmat_mode == COOPMAT_CM2 ? 1 : 2 var g_vk_dry_want = false var g_vk_dry_cfg = DlimVulkanConfig() @@ -1015,7 +1087,7 @@ def private query_sm_count(phys : VkPhysicalDevice) : int { return int(sp.shaderSMCount) } -[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-class-pipeline-build")] +[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#vk-class-pipeline-build")] def vk_moe_init : bool { if (g_init_failed) { // checked FIRST: a partially-built g_gpu must never report ready return false @@ -1034,8 +1106,9 @@ def vk_moe_init : bool { g_gpu.dry = true g_gpu.coopmat_mode = g_vk_dry_cfg.coopmat_mode g_gpu.has_coopmat = g_vk_dry_cfg.coopmat_mode > 0 - g_gpu.has_coopmat2 = g_vk_dry_cfg.coopmat_mode == 4 + g_gpu.has_coopmat2 = g_vk_dry_cfg.coopmat_mode == COOPMAT_CM2 g_gpu.rows_per_wg = int64((WG_X + uint(g_vk_dry_cfg.subgroup_size) - 1u) / uint(g_vk_dry_cfg.subgroup_size)) + g_gpu.subgroup_size = int64(g_vk_dry_cfg.subgroup_size) g_gpu.msr = g_vk_dry_cfg.max_storage_range g_gpu.weight_budget = g_vk_dry_cfg.vram_mb * 1_000_000l @@ -1070,7 +1143,7 @@ def vk_moe_init : bool { && timeline_semaphore_supported(g_gpu.phys)) { xfam = select_transfer_queue_family(g_gpu.phys) } - if (g_gpu.has_coopmat2 && g_gpu.coopmat_mode == 4) { + if (g_gpu.has_coopmat2 && g_gpu.coopmat_mode == COOPMAT_CM2) { // #77's creator: coopmat2 + BDA + vulkanMemoryModel + the query-then-enable extras g_gpu.device <- create_device_storage_8_16_int_dot_coopmat2(g_gpu.phys, g_gpu.fam, xfam, g_env_vulkan.vk_decvec) g_gpu.has_bda = true @@ -1108,11 +1181,10 @@ def vk_moe_init : bool { } to_log(LOG_INFO, "dasLLAMA vulkan tier: transfer queue armed (family {xfam})\n") } - // WDDM residency shield: allocations chain priority 1.0 (DASLLAMA_VK_MEMPRIO=0 = bisect hatch) g_gpu.has_memprio = (g_gpu.has_coopmat && memory_priority_supported(g_gpu.phys) && g_env_vulkan.vk_memprio) if (g_gpu.has_memprio) { - to_log(LOG_INFO, "dasLLAMA vulkan tier: memory-priority residency shield armed\n") + to_log(LOG_INFO, "dasLLAMA vulkan tier: memory-priority tags armed (DASLLAMA_VK_MEMPRIO=1 - the A/B arm)\n") } let sg = subgroup_properties(g_gpu.phys).subgroupSize if (sg < 32u) { @@ -1121,12 +1193,18 @@ def vk_moe_init : bool { return false } g_gpu.rows_per_wg = int64((WG_X + sg - 1u) / sg) // = gl_NumSubgroups for our 1D workgroup + g_gpu.subgroup_size = int64(sg) var pdp : VkPhysicalDeviceProperties vkGetPhysicalDeviceProperties(g_gpu.phys, pdp) g_gpu.msr = int64(pdp.limits.maxStorageBufferRange) // env-vs-auto resolution shared with the DlimConfiguration source (resolve_weight_cap) let heap = query_heap_info(g_gpu.phys) g_gpu.weight_budget = resolve_weight_cap(heap) + g_gpu.os_budget_known = heap.os_budget > 0l && heap.os_system > 0l + g_gpu.os_shared_base = heap.os_shared + if (g_gpu.os_budget_known) { + to_log(LOG_INFO, "dasLLAMA vulkan tier: OS video memory: ceiling {heap.os_budget / 1_000_000l} MB for this process, {heap.os_system / 1_000_000l} MB dedicated held on the adapter by every process ({heap.os_usage / 1_000_000l} MB ours), {heap.os_shared / 1_000_000l} MB shared - the resident plan sizes against the room left\n") + } if (heap.budget > 0l) { let vram_override = gpu_want_vram_mb() > 0l || is_some(g_env_gpu.gpu_vram_mb) // the honest cliff leaves 2GB of the raw heap for the desktop (6000MB healthy, 6803 demoted) @@ -1425,6 +1503,7 @@ def make_imported_buf(ptr : void?; bytes : int64) : HostBuf { return hb } g_gpu.host_mem |> insert(b._vk, m._vk) + g_gpu.host_bytes += bytes return HostBuf(buf = b._vk, mem = m._vk, mapped = ptr, bytes = bytes) } @@ -1641,6 +1720,7 @@ def vk_drop_model_state { } g_gpu.dev_mem |> clear() g_gpu.dev_mapped |> clear() + g_gpu.kq_grid_dev = 0ul g_vk_upload_us = 0l // per-model upload diagnostics — a reloaded model reports its own numbers g_vk_upload_bytes = 0l g_rdq_n = 0u // decode-role profiler likewise — averages must not span models @@ -1663,6 +1743,7 @@ def vk_drop_model_state { } } g_gpu.host_mem |> clear() + g_gpu.host_bytes = g_gpu.staging.bytes + g_pfq_buf.bytes // the process-lifetime pair survives the sweep // das-side shells + the lazy-family latches (the ensure_* paths rebuild on next use). // clear(), not delete, where values carry VK handles — nothing das-heap-owned lives inside // those structs, and delete would try to finalize the handle fields @@ -1817,12 +1898,12 @@ def vk_fmt_b32(fmt : int) : bool => fmt == 0 // lattice counts 256-weight superblocks def fmt_unit(fmt : int) : int64 => vk_fmt_b32(fmt) ? 32l : 256l +[arch(at="../ARCHITECTURE_GPU_QUANT_PLANES.md#metal-kq-split-scale-plane")] def arena_block_bytes(fmt : int) : tuple { if (fmt == 0) { return (wq = Q8_QPB, ws = Q8_SPB) // 32 int8 quants + one f16 scale } - // decoded scale row: [f16 d][f16 dmin][8 sc][8 mn] - return (wq = kq_qsb(vk_kq_schema_id(fmt)), ws = KQ_DEV_SSB) + return (wq = kq_qsb(vk_kq_schema_id(fmt)), ws = kq_dev_ssb_i(fmt)) } //! Row-length servability: the arena strides whole blocks, so a row that is not a block @@ -2019,6 +2100,7 @@ struct RLayer { fq, fk, fv, fo, f1, f3, f2 : int // the layer's descriptor sets (built once, referenced every token) s_quant_xb : VkDescriptorSet // requant class set for the attn-norm activation + s_quant_dno : VkDescriptorSet // the deltanet o row's Q8_K requant (a K-quant out plane) s_q, s_k, s_v : VkDescriptorSet // q/k/v class GEMV s_kv : VkDescriptorSet // merged k+v two-region class GEMV (fk == fv: same arena planes) kv_merged : bool @@ -2064,6 +2146,8 @@ struct RLayer { struct RDec { ready : bool + tokens_served : int64 + weight_bytes : int64 // the plan's weight bytes on the device (the warm-up self-check's bandwidth floor) n_layers, dim, qd, kv_dim, head_size, n_heads, kv_mul, hidden, vocab, seq_cap : int64 neox : bool kv16 : bool // the mirror codec: f16 rows (the default) vs f32; sessions must match @@ -2095,8 +2179,12 @@ struct RDec { dn_ba_dev : uint64 // f32 arm: every recurrent layer's [beta ; alpha] rows, [2 x nvh x dim] per layer, f16 dn_ba_bytes : int64 dn_oq_dev, dn_os_dev : uint64 // the step's Q8_0 o row - the out GEMV's activation image + dn_of_dev : uint64 // the step's f32 o row for a K-quant out plane - the Q8_K requant's source dn_parity : int // the shared conv-ring parity: every recurrent layer steps once per token norms_dev : uint64 // rms_att[l]/rms_ffn[l] interleaved + final, uploaded once + bias_dev : uint64 // the q/k/v projection biases, per layer [q | k | v] (qwen2; 0 = the model has none) + bias_bytes : int64 + has_bias : bool cos_dev : uint64 // this token's rope row (cos[half] + sin[half]) logits_dev : uint64 x_host, logits_host : HostBuf @@ -2310,6 +2398,7 @@ let VHB_VRAW = 0x4000u // at raw-v window let VHB_XF = 0x8000u // the cm2 expert chain's f32 activation rows (batch_xf_dev) let VHB_ATK16 = 0x10000u // at roped-k f16 shadow (the fa tile's K) let VHB_ATV16 = 0x20000u // at v f16 shadow (the fa tile's V) +let VHB_ATBIAS = 0x40000u // at projection bias row (the biased layer's [q | k | v]) // serial-seam region bits (decode GEMV preps, arena + standalone seams) — third namespace, same rail let VHG_META = 0x1u // ffn/dn params staged to device @@ -2531,7 +2620,7 @@ def private spv_override_load(kernel : string; var spv : array) { } } -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-class-pipeline-build")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#vk-class-pipeline-build")] def private spv_dump_save(kernel : string; spv : array) { let dir = g_env_vulkan.vk_spv_dump if (empty(dir)) { @@ -2548,7 +2637,7 @@ def private spv_dump_save(kernel : string; spv : array) { } } -[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-class-pipeline-build")] +[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#vk-class-pipeline-build")] def vkd_class_pipe(var c : VkdClass; ord : int; var spv : array; rmask, wmask : uint; kernel : string) { if (length(c.pipes) <= ord) { c.pipes |> resize(ord + 1) // empty slots hold null handles until their kernel ensures diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_decode.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_decode.das index 9cbc1dbb79..a14fd38973 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_decode.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_decode.das @@ -45,7 +45,7 @@ def vk_rdec_prepare(n_layers, n_attn, dim, qd, kv_dim, head_size, n_heads, hidde return false } let hybrid = dn.cd > 0l - if (hybrid && !(dn.ds <= DN_MAX_DS && dn.ds % 32l == 0l && (dn.ds & (dn.ds - 1l)) == 0l && dn.nvh <= DN_MAX_HEADS && dn.cd <= DN_MAX_CD + if (hybrid && !(dn.ds <= DN_MAX_DS && dn.ds % 32l == 0l && (dn.ds & (dn.ds - 1l)) == 0l && dn.nvh <= DN_MAX_HEADS && dn.cd <= DN_MAX_CD && 2l * dn.kd <= DN_MAX_QK && dn.dconv <= DN_MAX_DCONV && dn.di == dn.nvh * dn.ds && dn.kd == dn.nkh * dn.ds && dn.cd == 2l * dn.kd + dn.di && dn.di % 32l == 0l && dn.di <= DND_OQ_BYTES)) { to_log(LOG_WARNING, "dasLLAMA vulkan resident: deltanet geometry (cd {dn.cd}, kd {dn.kd}, di {dn.di}, heads {dn.nvh}/{dn.nkh}, state {dn.ds}, conv {dn.dconv}) is outside the step kernel's capacity\n") @@ -106,6 +106,7 @@ def vk_rdec_prepare(n_layers, n_attn, dim, qd, kv_dim, head_size, n_heads, hidde r.dn_proj_dev = make_device_buf((dn.cd + dn.di) * 4l) r.dn_oq_dev = make_device_buf(DND_OQ_BYTES) r.dn_os_dev = make_device_buf(DND_OS_BYTES) + r.dn_of_dev = make_device_buf(DND_OQ_BYTES * 4l) r.dn_parity = 0 ensure_dnd_state() // the staging buffers the state uploads and flushes ride } @@ -265,22 +266,23 @@ def vk_rdec_set_layer(l, mir_idx : int64; bq, bk, bv, bo, b1, b3, b2 : int64; fq fixed_array(0u, 0u, 0u, VHZ_XQ, VHZ_XQ, VHZ_KVK | VHZ_KVV)) } let mirbytes = g_rd.n_attn * g_rd.seq_cap * kvd * g_rd.kv_esz + let bias_bind = rd_bias_bind() if (g_rd.qk_norm) { // per-head q/k rmsnorm (class): q rows, k rows (kv_dev — kbase=qd rides the push), norms L.s_qkn = set_qk_rms_cls(fixed_array(g_rd.q_dev, g_rd.kv_dev, g_rd.norms_dev), fixed_array(qd * 4l, (qd + 2l * kvd) * 4l, g_rd.norms_bytes), fixed_array(VHZ_Q, VHZ_KVK, 0u)) - // the fused twin (class): norm rows bind the norms buffer, mirror row rides TokMeta - let qrbufs = fixed_array(g_rd.q_dev, g_rd.kv_dev, g_rd.k_mirror, g_rd.v_mirror, g_rd.cos_dev, g_rd.norms_dev, g_rd.tok_meta.buf) - let qrsizes = fixed_array(qbytes, (qd + 2l * kvd) * 4l, mirbytes, mirbytes, g_rd.head_size * 4l, g_rd.norms_bytes, 16l) - let qrbits = fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, VHZ_COS, 0u, 0u) + // the fused twin (class): norm rows bind the norms buffer, mirror row rides TokMeta, the bias rows their buffer + let qrbufs = fixed_array(g_rd.q_dev, g_rd.kv_dev, g_rd.k_mirror, g_rd.v_mirror, g_rd.cos_dev, g_rd.norms_dev, g_rd.tok_meta.buf, bias_bind.buf) + let qrsizes = fixed_array(qbytes, (qd + 2l * kvd) * 4l, mirbytes, mirbytes, g_rd.head_size * 4l, g_rd.norms_bytes, 16l, bias_bind.bytes) + let qrbits = fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, VHZ_COS, 0u, 0u, 0u) L.s_qkn_rope = (kv16 ? set_qkn_rope_f16_cls(qrbufs, qrsizes, qrbits) : set_qkn_rope_cls(qrbufs, qrsizes, qrbits)) } - // rope+store (class): per-token mirror row rides TokMeta - let rbufs = fixed_array(g_rd.q_dev, g_rd.kv_dev, g_rd.k_mirror, g_rd.v_mirror, g_rd.cos_dev, g_rd.tok_meta.buf) - let rsizes = fixed_array(qd * 4l, (qd + 2l * kvd) * 4l, mirbytes, mirbytes, g_rd.head_size * 4l, 16l) - let rbits = fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, VHZ_COS, 0u) + // rope+store (class): per-token mirror row rides TokMeta, the bias rows their buffer + let rbufs = fixed_array(g_rd.q_dev, g_rd.kv_dev, g_rd.k_mirror, g_rd.v_mirror, g_rd.cos_dev, g_rd.tok_meta.buf, bias_bind.buf) + let rsizes = fixed_array(qd * 4l, (qd + 2l * kvd) * 4l, mirbytes, mirbytes, g_rd.head_size * 4l, 16l, bias_bind.bytes) + let rbits = fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, VHZ_COS, 0u, 0u) L.s_rope = kv16 ? set_rope_kv_f16_cls(rbufs, rsizes, rbits) : set_rope_kv_cls(rbufs, rsizes, rbits) // attn (class): per-token cnt rides TokMeta, the scale rides the push let abufs = fixed_array(g_rd.q_dev, g_rd.k_mirror, g_rd.v_mirror, g_rd.attn_dev, g_rd.tok_meta.buf) @@ -373,6 +375,7 @@ def vk_rdec_set_dn_layer(l, bqkv, bz, bout, bbeta, balpha, b1, b3, b2, ba_off : let anykq = kq_sb(f1) || kq_sb(f2) verify(ensure_cls_dn_rq() && ensure_cls_ar() && ensure_cls_ar_rq() && ensure_q8_actrq_cls() && (!anykq || (ensure_cls_q8k_rq() && ensure_q8k_actrq_cls())) + && (!kq_sb(fout) || ensure_cls_q8k_rq()) && gemv_cls_ensure(fqkv) && gemv_cls_ensure(fz) && gemv_cls_ensure(fout) && (g_rd.dn_ba_f32 || gemv_cls_ensure(int(KqFmt.q8))) && gemv_cls_ensure(f1) && gemv_cls_ensure(f3) && gemv_cls_ensure(f2), @@ -408,11 +411,16 @@ def vk_rdec_set_dn_layer(l, bqkv, bz, bout, bbeta, balpha, b1, b3, b2, ba_off : fixed_array(pa.wqb, pa.wsb, META_BYTES, dim, (dim / 32l) * 4l, DND_SMALLS_BYTES), fixed_array(0u, 0u, 0u, VHZ_XQ, VHZ_XQ, VHZ_DNSM)) } - // the fused step: proj @0, state @1 (in place), smalls @2 (rw), o scales @3, o quants @4, tok @5 + // the fused step: proj @0, state @1 (in place), smalls @2 (rw), o scales @3, o quants @4, tok @5, o f32 @6 L.s_dnstep = set_dn_step_cls( - fixed_array(g_rd.dn_proj_dev, L.dn.state_dev, L.dn.smalls_dev, g_rd.dn_os_dev, g_rd.dn_oq_dev, g_rd.tok_meta.buf), - fixed_array((cd + di) * 4l, L.dn.state_bytes, DND_SMALLS_BYTES, DND_OS_BYTES, DND_OQ_BYTES, 16l), - fixed_array(VHZ_DNP, VHZ_DNST, VHZ_DNSM, VHZ_DNO, VHZ_DNO, 0u)) + fixed_array(g_rd.dn_proj_dev, L.dn.state_dev, L.dn.smalls_dev, g_rd.dn_os_dev, g_rd.dn_oq_dev, g_rd.tok_meta.buf, g_rd.dn_of_dev), + fixed_array((cd + di) * 4l, L.dn.state_bytes, DND_SMALLS_BYTES, DND_OS_BYTES, DND_OQ_BYTES, 16l, DND_OQ_BYTES * 4l), + fixed_array(VHZ_DNP, VHZ_DNST, VHZ_DNSM, VHZ_DNO, VHZ_DNO, 0u, VHZ_DNO)) + if (kq_sb(fout)) { + L.s_quant_dno = set_rq_cls(fixed_array(g_rd.dn_of_dev, g_rd.dn_oq_dev, g_rd.dn_os_dev), + fixed_array(DND_OQ_BYTES * 4l, DND_OQ_BYTES, DND_OS_BYTES), + fixed_array(VHZ_DNO, VHZ_DNO, VHZ_DNO)) + } L.s_dnout = gemv_cls_set(fout, fixed_array(pout.wq, pout.ws, L.m_dnout.buf, g_rd.dn_oq_dev, g_rd.dn_os_dev, g_rd.xb2_dev), fixed_array(pout.wqb, pout.wsb, META_BYTES, di, (di / 32l) * 4l, dim * 4l), fixed_array(0u, 0u, 0u, VHZ_DNO, VHZ_DNO, VHZ_XB2)) @@ -422,6 +430,25 @@ def vk_rdec_set_dn_layer(l, bqkv, bz, bout, bbeta, balpha, b1, b3, b2, ba_off : g_rd.tok_recorded = false } +[arch(at="../ARCHITECTURE_GPU_VULKAN_DECODE.md#hybrid-token-command")] +def vk_rdec_upload_bias(rows : array) { + if (g_gpu != null && g_gpu.dry) { + return + } + assert(g_rd != null, "vk_rdec_upload_bias before prepare") + g_rd.bias_bytes = long_length(rows) * 4l + g_rd.bias_dev = make_device_buf(g_rd.bias_bytes) + upload_region_at(g_rd.bias_dev, 0l, unsafe(addr(rows[0])), g_rd.bias_bytes) + g_rd.has_bias = true +} + +//! hasb 0 leaves the binding unread - the norms buffer only fills the slot +def private rd_bias_bind() : tuple { + return g_rd.has_bias ? (buf = g_rd.bias_dev, bytes = g_rd.bias_bytes) : (buf = g_rd.norms_dev, bytes = g_rd.norms_bytes) +} + +def private rd_bias_off(l : int64) : uint => uint(l * (g_rd.qd + 2l * g_rd.kv_dim)) + //! The f32 arm's beta/alpha rows for every recurrent layer, in layer order: per layer `[beta rows //! (nvh x dim) ; alpha rows (nvh x dim)]`; a layer's base is its `ba_off` at set_dn_layer. The //! device keeps them as f16 - half the bytes the prefill tile GEMM and the decode GEMV read per row. @@ -569,7 +596,7 @@ def private rd_encode_attn_head(raw : VkCommandBuffer; var h : VkHaz; var L : RL var pcf = QknRopeArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(g_rd.rot / 2l), neox = g_rd.neox ? 1u : 0u, nh = uint(g_rd.n_heads), layerbase = mirbase, qwoff = uint(nlfin + dim + l * 2l * hs), kwoff = uint(nlfin + dim + (l * 2l + 1l) * hs), - eps = g_rd.eps, qstride = qstride) + eps = g_rd.eps, qstride = qstride, boff = rd_bias_off(l), hasb = g_rd.has_bias ? 1u : 0u) if (g_rd.kv16) { enc_qkn_rope_f16_cls(raw, h, L.s_qkn_rope, pcf, g_rd.n_heads + kvd / hs) } else { @@ -587,7 +614,7 @@ def private rd_encode_attn_head(raw : VkCommandBuffer; var h : VkHaz; var L : RL } var pcr = RopeKvArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(hs / 2l), neox = g_rd.neox ? 1u : 0u, kpair0 = uint(qd / 2l), npairs = uint(qd / 2l + kvd / 2l), - layerbase = mirbase) + layerbase = mirbase, boff = rd_bias_off(l), hasb = g_rd.has_bias ? 1u : 0u) if (g_rd.kv16) { enc_rope_kv_f16_cls(raw, h, L.s_rope, pcr, (qd / 2l + kvd / 2l + int64(WG_X) - 1l) / int64(WG_X)) } else { @@ -646,9 +673,13 @@ def private rd_encode_dn_head(raw : VkCommandBuffer; var h : VkHaz; var L : RLay dconv = uint(g_rd.dn_dconv), beta_off = uint(DN_SM_BETA), g_off = uint(DN_SM_G), taps_off = uint(DN_SM_TAPS), wnorm_off = uint(DN_SM_WNORM), a_off = uint(DN_SM_A), dt_off = uint(DN_SM_DT), hist_off = uint(DN_SM_HIST), - eps = g_rd.eps, qscale = 1.0 / sqrt(float(g_rd.dn_ds))) + eps = g_rd.eps, qscale = 1.0 / sqrt(float(g_rd.dn_ds)), out_f32 = kq_sb(L.fout) ? 1u : 0u) enc_dn_step_cls(raw, h, L.s_dnstep, pcd, nvh) pfq_ts(raw) + if (kq_sb(L.fout)) { // no pfq_ts here - the requant bills to the out role, so the stamp count stands + var pcr = RqArgs(inbase = 0u, nblk = uint(di / 256l)) + enc_cls_q8k_rq(raw, h, L.s_quant_dno, pcr, (di / 8l + 255l) / 256l) + } var pco = GemvArgs(n = uint(di), d = uint(dim), nreg = 1u, ybase = 0u) gemv_cls_enc(L.fout, raw, h, L.s_dnout, pco, (dim + rpw - 1l) / rpw) pfq_ts(raw) @@ -872,6 +903,31 @@ def private rdq_sample { } } +let private RD_PAGING_BYTES_PER_US = 40_000l // 40 GB/s, the PCIe class + +//! The resident driver's device weight bytes (rdec_note_weight_bytes's seat; the plan hands them over after prepare) +def vk_rdec_note_weight_bytes(bytes : int64) { + if (g_rd != null) { + g_rd.weight_bytes = bytes + } +} + +[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN_RESIDENCY.md#resident-plan")] // the first four tokens and every 256th +def private rd_logits_guard(logits : array) { + for (v in logits) { + return if (v != 0.0) + } + panic("dasLLAMA vulkan tier: the device returned an all-zero logits row - the card is over-committed (planes demoted past what the driver can page back) or lost; close other GPU applications, lower DASLLAMA_GPU_VRAM_MB, or restart") +} + +[cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN_RESIDENCY.md#resident-plan")] // once per armed driver +def private rd_warmup_check(wall_us : int64) { + vk_demotion_check("decode warm-up") + if (g_rd.weight_bytes > 0l && wall_us > g_rd.weight_bytes / RD_PAGING_BYTES_PER_US) { + to_log(LOG_WARNING, "dasLLAMA vulkan tier: a decode token ran under {RD_PAGING_BYTES_PER_US / 1_000l} GB/s over {g_rd.weight_bytes / 1_000_000l} MB of resident weights - the paging class; close other GPU applications or lower DASLLAMA_GPU_VRAM_MB\n") + } +} + //! Resident-driver token: run the whole stack on device from the embedded residual `x` and this //! position's rope row (cos[half]+sin[half]) -> logits. `cnt` = cached positions incl. this one; K/V //! for `pos` store into the mirror. Recorded once — a step is two memcpys + position words + one submit. @@ -894,6 +950,13 @@ def vk_rdec_token(x : array; cossin : array; pos, cnt : int64; var submit_wait(g_rd.cmd) memcpy(addr(logits[0]), g_rd.logits_host.mapped, g_rd.vocab * 4l) } + g_rd.tokens_served++ + if (g_rd.tokens_served <= 4l || (g_rd.tokens_served & 255l) == 0l) { + rd_logits_guard(logits) + } + if (g_rd.tokens_served == 8l) { + rd_warmup_check(int64(get_time_usec(ts0))) + } if (vk_prof()) { g_rdq_wall_us += double(get_time_usec(ts0)) if (g_rdq_count % 32l == 31l) { @@ -1005,6 +1068,7 @@ def private ensure_dnd_state { g_gpu.dnd_bg = make_host_buf(DND_BG_BYTES, false) g_gpu.dnd_oq_dev = make_device_buf(DND_OQ_BYTES) g_gpu.dnd_os_dev = make_device_buf(DND_OS_BYTES) + g_gpu.dnd_of_dev = make_device_buf(DND_OQ_BYTES * 4l) g_gpu.dnd_ready = true to_log(LOG_INFO, "dasLLAMA vulkan tier: deltanet decode step engaged\n") } @@ -1044,9 +1108,9 @@ def private dnd_step_make(s_qkv, s_z, s_o : int; woq, woz, woo : int64; fixed_array(0u, 0u, 0u, 0u, 0u, VHG_Y2)) // fused step (class): proj = the shared y (qkv rows + z at cd); state advances in place — residency IS that binding; parity rides TokMeta st.fused_set = set_dn_step_cls( - fixed_array(g_gpu.stacks[s_qkv].y_dev, st.state_dev, st.smalls_dev, g_gpu.dnd_os_dev, g_gpu.dnd_oq_dev, st.tok.buf), - fixed_array(Y_BYTES, st.state_bytes, DND_SMALLS_BYTES, DND_OS_BYTES, DND_OQ_BYTES, 16l), - fixed_array(VHG_Y1 | VHG_Y2, VHG_ST, VHG_SM, VHG_WS, VHG_WS, 0u)) + fixed_array(g_gpu.stacks[s_qkv].y_dev, st.state_dev, st.smalls_dev, g_gpu.dnd_os_dev, g_gpu.dnd_oq_dev, st.tok.buf, g_gpu.dnd_of_dev), + fixed_array(Y_BYTES, st.state_bytes, DND_SMALLS_BYTES, DND_OS_BYTES, DND_OQ_BYTES, 16l, DND_OQ_BYTES * 4l), + fixed_array(VHG_Y1 | VHG_Y2, VHG_ST, VHG_SM, VHG_WS, VHG_WS, 0u, VHG_WS)) // out GEMV: out planes@0/1, its once-filled regions@2, the o image@3/4, its y_dev@5 st.out_set = gemv_cls_set(fo, fixed_array(g_gpu.stacks[s_o].wqbuf, g_gpu.stacks[s_o].wsbuf, g_gpu.stacks[s_o].meta.buf, g_gpu.dnd_oq_dev, g_gpu.dnd_os_dev, g_gpu.stacks[s_o].y_dev), @@ -1119,7 +1183,7 @@ def vk_moe_dn_step(var yp : float?; s_qkv, s_z, s_o : int; woq, woz, woo : int64 dim, cd, kd, di, nvh, nkh, ds, dconv : int64; eps : float) { let taps = dconv - 1l assert(ds <= DN_MAX_DS && ds % 32l == 0l && (ds & (ds - 1l)) == 0l && nvh <= DN_MAX_HEADS && cd <= DN_MAX_CD - && dconv <= DN_MAX_DCONV && di == nvh * ds && kd == nkh * ds && cd == 2l * kd + di + && 2l * kd <= DN_MAX_QK && dconv <= DN_MAX_DCONV && di == nvh * ds && kd == nkh * ds && cd == 2l * kd + di && dim % 32l == 0l && di % 32l == 0l && di <= DND_OQ_BYTES, "dasLLAMA vulkan tier: deltanet geometry outside step capacity") ensure_dnd_state() @@ -1674,7 +1738,8 @@ def vk_moe_span_dec(var a : MoeSpanDec) { ensure_ffn_state() for (li in range64(nl)) { dat_layer_ensure(a.woq[li], a.kvd) - dat_layer_sets(a.woq[li], a.wok[li], a.wov[li], a.woo[li], a.fq[li], a.fk[li], a.fv[li], a.fo[li], a.rmsq[li], a.rmsk[li], a.qk_norm) + dat_layer_sets(a.woq[li], a.wok[li], a.wov[li], a.woo[li], a.fq[li], a.fk[li], a.fv[li], a.fo[li], a.rmsq[li], a.rmsk[li], + a.bq[li], a.bk[li], a.bv[li], a.qk_norm) } ensure_span_state(a) assert(g_gpu.tail_pending_woq < 0l, "dasLLAMA vulkan tier: decode span with a decode ffn tail pending") @@ -2038,7 +2103,8 @@ def private ensure_dat_state(dim, qd, kvd, hs : int64) { } [cold_path, arch(at="../ARCHITECTURE_GPU_VULKAN_DECODE.md#decode-attention-block")] // once per layer: the chain's sets -def private dat_layer_sets(woq, wok, wov, woo : int64; fq, fk, fv, fo : int; rmsqp : float const?; rmskp : float const?; qk_norm : bool) { +def private dat_layer_sets(woq, wok, wov, woo : int64; fq, fk, fv, fo : int; rmsqp : float const?; rmskp : float const?; + bqp : float const?; bkp : float const?; bvp : float const?; qk_norm : bool) { var L & = unsafe(g_gpu.dat_layers[woq]) if (L.sets_made) { return @@ -2080,6 +2146,18 @@ def private dat_layer_sets(woq, wok, wov, woo : int64; fq, fk, fv, fo : int; rms g_gpu.stacks[sq].xq.buf, g_gpu.stacks[sq].xs.buf, g_gpu.dat_kv_dev), fixed_array(g_gpu.stacks[sv].wqbytes, ws_range(sv), META_BYTES, XQ_BYTES, XS_BYTES, (qd + 2l * kvd) * 4l), fixed_array(0u, 0u, 0u, 0u, 0u, VHZ_KVV)) + L.has_bias = bqp != null + if (L.has_bias) { + L.bias = make_device_buf((qd + 2l * kvd) * 4l) + unsafe { + upload_region_at(L.bias, 0l, reinterpret(bqp), qd * 4l) + upload_region_at(L.bias, qd * 4l, reinterpret(bkp), kvd * 4l) + upload_region_at(L.bias, (qd + kvd) * 4l, reinterpret(bvp), kvd * 4l) + } + } + // hasb 0 leaves this binding unread - the cos buffer only fills the slot + let bbuf = L.has_bias ? L.bias : g_gpu.dat_cos.buf + let bbytes = L.has_bias ? (qd + 2l * kvd) * 4l : hs * 4l if (qk_norm) { L.norms = make_device_buf(2l * hs * 4l) unsafe { @@ -2087,14 +2165,14 @@ def private dat_layer_sets(woq, wok, wov, woo : int64; fq, fk, fv, fo : int; rms upload_region_at(L.norms, hs * 4l, reinterpret(rmskp), hs * 4l) } L.s_qkr = set_qkn_rope_f16_cls( - fixed_array(g_gpu.dat_q_dev, g_gpu.dat_kv_dev, L.k_mir, L.v_mir, g_gpu.dat_cos.buf, L.norms, g_gpu.dat_tok.buf), - fixed_array(qd * 4l, (qd + 2l * kvd) * 4l, mirbytes, mirbytes, hs * 4l, 2l * hs * 4l, 16l), - fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, 0u, 0u, 0u)) + fixed_array(g_gpu.dat_q_dev, g_gpu.dat_kv_dev, L.k_mir, L.v_mir, g_gpu.dat_cos.buf, L.norms, g_gpu.dat_tok.buf, bbuf), + fixed_array(qd * 4l, (qd + 2l * kvd) * 4l, mirbytes, mirbytes, hs * 4l, 2l * hs * 4l, 16l, bbytes), + fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, 0u, 0u, 0u, 0u)) } else { L.s_qkr = set_rope_kv_f16_cls( - fixed_array(g_gpu.dat_q_dev, g_gpu.dat_kv_dev, L.k_mir, L.v_mir, g_gpu.dat_cos.buf, g_gpu.dat_tok.buf), - fixed_array(qd * 4l, (qd + 2l * kvd) * 4l, mirbytes, mirbytes, hs * 4l, 16l), - fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, 0u, 0u)) + fixed_array(g_gpu.dat_q_dev, g_gpu.dat_kv_dev, L.k_mir, L.v_mir, g_gpu.dat_cos.buf, g_gpu.dat_tok.buf, bbuf), + fixed_array(qd * 4l, (qd + 2l * kvd) * 4l, mirbytes, mirbytes, hs * 4l, 16l, bbytes), + fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, 0u, 0u, 0u)) } L.s_attn = set_da_attn_f16_cls( fixed_array(g_gpu.dat_q_dev, L.k_mir, L.v_mir, g_gpu.dat_attn_dev, g_gpu.dat_tok.buf), @@ -2199,13 +2277,14 @@ def private at_chain_enc(raw : VkCommandBuffer; var h : VkHaz; var L : DatLayer; gemv_cls_enc(fk, raw, h, L.s_k, pck, (kvd + rpw - 1l) / rpw) var pcv = GemvArgs(n = uint(dim), d = uint(kvd), nreg = 1u, ybase = uint(qd + kvd)) gemv_cls_enc(fv, raw, h, L.s_v, pcv, (kvd + rpw - 1l) / rpw) + let hb = L.has_bias ? 1u : 0u // the layer's own row is bound, so boff stays 0 if (qk_norm) { var pcf = QknRopeArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(hs / 2l), - neox = neox ? 1u : 0u, nh = uint(n_heads), layerbase = 0u, qwoff = 0u, kwoff = uint(hs), eps = eps) + neox = neox ? 1u : 0u, nh = uint(n_heads), layerbase = 0u, qwoff = 0u, kwoff = uint(hs), eps = eps, boff = 0u, hasb = hb) enc_qkn_rope_f16_cls(raw, h, L.s_qkr, pcf, n_heads + kvd / hs) } else { var pcr = RopeKvArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(hs / 2l), - neox = neox ? 1u : 0u, kpair0 = uint(qd / 2l), npairs = uint(qd / 2l + kvd / 2l), layerbase = 0u) + neox = neox ? 1u : 0u, kpair0 = uint(qd / 2l), npairs = uint(qd / 2l + kvd / 2l), layerbase = 0u, boff = 0u, hasb = hb) enc_rope_kv_f16_cls(raw, h, L.s_qkr, pcr, (qd / 2l + kvd / 2l + int64(WG_X) - 1l) / int64(WG_X)) } var pca = DaAttnArgs(kvd = uint(kvd), hs = uint(hs), kv_mul = uint(kv_mul), kbase = 0u, vbase = 0u, scale = scale) @@ -2243,11 +2322,12 @@ if (!L.recorded || vk_rdec_rerecord_on()) { [hot_path, arch(at="../ARCHITECTURE_GPU_VULKAN_DECODE.md#decode-attention-block")] // per token def vk_moe_attn_dec(var xb2p : float?; var kp : float?; var vp : float?; woq, wok, wov, woo : int64; fq, fk, fv, fo : int; xqp : int8 const?; xsp : float const?; cosp : float const?; rmsqp : float const?; rmskp : float const?; + bqp : float const?; bkp : float const?; bvp : float const?; pos, dim, qd, kvd, hs, n_heads, kv_mul : int64; scale, eps : float; neox, qk_norm : bool) { let ts0 = ref_time_ticks() ensure_dat_state(dim, qd, kvd, hs) dat_layer_ensure(woq, kvd) - dat_layer_sets(woq, wok, wov, woo, fq, fk, fv, fo, rmsqp, rmskp, qk_norm) + dat_layer_sets(woq, wok, wov, woo, fq, fk, fv, fo, rmsqp, rmskp, bqp, bkp, bvp, qk_norm) var L : DatLayer? unsafe { L = g_gpu.dat_layers?[woq] // the ensure above made the key: a read, never an insert diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index bcdfb93c35..01027011ef 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -40,7 +40,10 @@ var private @scratch g_pf_ids_stage : array [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] def private pf_f16_feed(f : int) : bool { - return (f == int(KqFmt.q8) || kq_sb(f)) && g_gpu.coopmat_mode == 4 + if (g_gpu.coopmat_mode == COOPMAT_CM2) { + return f == int(KqFmt.q8) || kq_sb(f) + } + return g_gpu.coopmat_mode == COOPMAT_MM && kq_sb(f) && khr_kq_tile_on() } [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] @@ -57,10 +60,81 @@ def private pf_qkv_feed_fused(l : int64) : bool => !g_rd.layers[l].recurrent && let private PF_CHUNK_MAX = 8l // the overlap ramp doubles 1,2,4 then holds here; the cmd ring is sized from it -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] +def khr_cls_ensure(fmt : int) : bool { + if (fmt == int(KqFmt.k4)) return ensure_kq_batch_k4_khr_cls() + if (fmt == int(KqFmt.k5)) return ensure_kq_batch_k5_khr_cls() + if (fmt == int(KqFmt.k6)) return ensure_kq_batch_k6_khr_cls() + if (fmt == int(KqFmt.q40)) return ensure_kq_batch_q40_khr_cls() + if (fmt == int(KqFmt.iq4xs)) return ensure_kq_batch_iq4xs_khr_cls() + if (fmt == int(KqFmt.k3)) return ensure_kq_batch_k3_khr_cls() + if (fmt == int(KqFmt.iq3s)) return ensure_kq_batch_iq3s_khr_cls() + if (fmt == int(KqFmt.iq3xxs)) return ensure_kq_batch_iq3xxs_khr_cls() + if (fmt == int(KqFmt.iq4nl)) return ensure_kq_batch_iq4nl_khr_cls() + if (fmt == int(KqFmt.k2)) return ensure_kq_batch_k2_khr_cls() + if (fmt == int(KqFmt.iq2s)) return ensure_kq_batch_iq2s_khr_cls() + if (fmt == int(KqFmt.iq2xs)) return ensure_kq_batch_iq2xs_khr_cls() + verify(fmt == int(KqFmt.iq2xxs), "vk prefill: the KHR ensure fell through for a format the f16 feed admits") + return ensure_kq_batch_iq2xxs_khr_cls() +} + +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] +def khr_cls_set(fmt : int; bufs : uint64 const[5]; sizes : int64 const[5]; gbits : uint const[5]) : VkDescriptorSet { + if (fmt == int(KqFmt.k4)) return set_kq_batch_k4_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.k5)) return set_kq_batch_k5_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.k6)) return set_kq_batch_k6_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.q40)) return set_kq_batch_q40_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.iq4xs)) return set_kq_batch_iq4xs_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.k3)) return set_kq_batch_k3_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.iq3s)) return set_kq_batch_iq3s_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.iq3xxs)) return set_kq_batch_iq3xxs_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.iq4nl)) return set_kq_batch_iq4nl_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.k2)) return set_kq_batch_k2_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.iq2s)) return set_kq_batch_iq2s_khr_cls(bufs, sizes, gbits) + if (fmt == int(KqFmt.iq2xs)) return set_kq_batch_iq2xs_khr_cls(bufs, sizes, gbits) + verify(fmt == int(KqFmt.iq2xxs), "vk prefill: the KHR set fell through for a format the f16 feed admits") + return set_kq_batch_iq2xxs_khr_cls(bufs, sizes, gbits) +} + +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] +def khr_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var s : VkDescriptorSet; var pc : BatchArgs; groups : int64) { + if (fmt == int(KqFmt.k4)) { + enc_kq_batch_k4_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.k5)) { + enc_kq_batch_k5_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.k6)) { + enc_kq_batch_k6_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.q40)) { + enc_kq_batch_q40_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_batch_iq4xs_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_batch_k3_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_batch_iq3s_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_batch_iq3xxs_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_batch_iq4nl_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_batch_k2_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_batch_iq2s_khr_cls(raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_batch_iq2xs_khr_cls(raw, h, s, pc, groups) + } else { + verify(fmt == int(KqFmt.iq2xxs), "vk prefill: the KHR enc fell through for a format the f16 feed admits") + enc_kq_batch_iq2xxs_khr_cls(raw, h, s, pc, groups) + } +} + +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def cm2_cls_ensure(fmt : int; tc : int64) : bool { verify(fmt == int(KqFmt.q8) || kq_sb(fmt), "vk prefill: a cm2 tile for a format the f16 feed never admits") verify(tc == 256l || tc == 128l || tc == 32l, "vk prefill: a cm2 tile column the pick never returns") + if (g_gpu.coopmat_mode == COOPMAT_MM) { + return khr_cls_ensure(fmt) + } if (fmt == int(KqFmt.k4)) { return tc == 256l ? ensure_kq_batch_k4_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k4_cm2m_cls() : ensure_kq_batch_k4_cm2s_cls()) } @@ -104,8 +178,11 @@ def cm2_cls_ensure(fmt : int; tc : int64) : bool { return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : int64 const[5]; gbits : uint const[5]) : VkDescriptorSet { + if (g_gpu.coopmat_mode == COOPMAT_MM) { + return khr_cls_set(fmt, bufs, sizes, gbits) + } if (fmt == int(KqFmt.k4)) { if (tc == 256l) { return set_kq_batch_k4_cm2l_cls(bufs, sizes, gbits) @@ -191,9 +268,11 @@ def cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : int64 con return tc == 128l ? set_q8_batch_cm2m_cls(bufs, sizes, gbits) : set_q8_batch_cm2s_cls(bufs, sizes, gbits) } -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : VkHaz; var s : VkDescriptorSet; var pc : BatchArgs; groups : int64) { - if (fmt == int(KqFmt.k4)) { + if (g_gpu.coopmat_mode == COOPMAT_MM) { + khr_cls_enc(fmt, raw, h, s, pc, groups) + } elif (fmt == int(KqFmt.k4)) { if (tc == 256l) { enc_kq_batch_k4_cm2l_cls(raw, h, s, pc, groups) } elif (tc == 128l) { @@ -389,7 +468,7 @@ def private pf_gemm_enc(raw : VkCommandBuffer; var h : VkHaz; idx : int; fmt : i } } elif (kq_sb(fmt)) { // the role's format is static, so the slot never collides with the q8 router's variants - let q40cm = fmt == int(KqFmt.q40) && g_gpu.coopmat_mode == 1 + let q40cm = fmt == int(KqFmt.q40) && g_gpu.coopmat_mode == COOPMAT_F16 let key = idx * 16 + 6 if (!key_exists(g_rd.pf_cls_gemm, key)) { verify(kq_batch_cls_ensure(fmt, q40cm), "vk prefill: the kq batch class rail must engage") @@ -545,6 +624,9 @@ def private rd_pf_recurrent(raw : VkCommandBuffer; var h : VkHaz; l : int64; b : if (dn6) { var pco = ActArgs(nelem = uint(wlen * di), gelu = 0u, nblk = 0u) enc_f16cvt_cls(raw, h, g_rd.pf_dnof_set, pco, (wlen * di / 4l + 255l) / 256l) + } elif (kq_sb(L.fout)) { + var pcr = RqArgs(inbase = uint(DN_O_OFF), nblk = uint(wlen * di / 256l)) + enc_cls_q8k_rq(raw, h, g_rd.pf_dnrq_set, pcr, (wlen * di / 8l + 255l) / 256l) } else { var pcr = RqArgs(inbase = uint(DN_O_OFF), nblk = uint(wlen * di / 32l)) enc_cls_dn_rq(raw, h, g_rd.pf_dnrq_set, pcr, (wlen * di / 4l + 255l) / 256l) @@ -619,20 +701,21 @@ def private pf_setup { } ensure_batch_state() // hq_dev/hs_dev for the FFN act // f16-fed groups (pf_f16_feed) take f16 convert roles; kq consumers the Q8_K quant/act forms - var any6 = false + var any_f16_feed = false var anykq = kq_sb(g_rd.cls_fmt_marker) for (L in g_rd.layers) { - any6 = any6 || pf_f16_feed(L.fq) || pf_f16_feed(L.fo) || pf_f16_feed(L.f1) || pf_f16_feed(L.f2) - anykq = anykq || kq_sb(L.fq) || kq_sb(L.fo) || kq_sb(L.f1) || kq_sb(L.f2) || (L.recurrent && (kq_sb(L.fqkv) || kq_sb(L.fz))) + any_f16_feed = (any_f16_feed || pf_f16_feed(L.fq) || pf_f16_feed(L.fo) || pf_f16_feed(L.f1) || pf_f16_feed(L.f2) + || (L.recurrent && (pf_f16_feed(L.fqkv) || pf_f16_feed(L.fz) || pf_f16_feed(L.fout)))) + anykq = anykq || kq_sb(L.fq) || kq_sb(L.fo) || kq_sb(L.f1) || kq_sb(L.f2) || (L.recurrent && (kq_sb(L.fqkv) || kq_sb(L.fz) || kq_sb(L.fout))) } - if (any6) { + if (any_f16_feed) { assert(np % 32l == 0l, "dasLLAMA vulkan tier: the s tile's unclamped column needs a 32-row-multiple window") // the resident planes carry no read slack g_rd.pf_xf = make_device_buf(np * dim * 2l) g_rd.pf_af = make_device_buf(np * qd * 2l) g_rd.pf_hf = make_device_buf(np * hid * 2l) } // cm2 split-k scratch (one SSBO range): cap at the device range HERE where the size is negotiated - if (any6 && g_gpu.coopmat_mode == 4) { + if (any_f16_feed && g_gpu.coopmat_mode == COOPMAT_CM2) { var skb = 262_144l * int64(g_gpu.sm_count) let fsk = cm2_splitk_env() if (fsk > 1l) { @@ -666,11 +749,11 @@ def private pf_setup { && (!g_rd.pf_facm2 || (g_rd.head_size == 64l ? ensure_fa_cm2_h64_cls() : (g_rd.head_size == 128l ? ensure_fa_cm2_h128_cls() : (g_rd.q_gated ? ensure_fa_cm2_h256_g_cls() : ensure_fa_cm2_h256_cls())))) - && (!g_rd.pf_facm2 || !any6 + && (!g_rd.pf_facm2 || !any_f16_feed || (g_rd.head_size == 64l ? ensure_fa_cm2_h64_f16_cls() : (g_rd.head_size == 128l ? ensure_fa_cm2_h128_f16_cls() : (g_rd.q_gated ? ensure_fa_cm2_h256_g_f16_cls() : ensure_fa_cm2_h256_f16_cls())))) && ensure_q8_actrq_cls() && (!g_rd.qk_norm || ensure_qk_rms_cls()) - && (!any6 || (ensure_f16cvt_cls() && ensure_actf16_cls() && ensure_cls_ar_f16_b())) + && (!any_f16_feed || (ensure_f16cvt_cls() && ensure_actf16_cls() && ensure_cls_ar_f16_b())) && (!anykq || (ensure_cls_q8k_rq() && ensure_q8k_actrq_cls())), "vk prefill: the class rails must engage on a live device") if (g_rd.pf_facm2) { @@ -699,7 +782,7 @@ def private pf_setup { g_rd.pf_dnrq_set = set_rq_cls(fixed_array(g_gpu.dn_ws_dev, g_gpu.hq_dev, g_gpu.hs_dev), fixed_array(DN_WS_BYTES, BATCH_HQ_BYTES, BATCH_HS_BYTES), fixed_array(VHZ_DNWS, VHZ_HQ, VHZ_HQ)) - if (g_gpu.coopmat_mode == 4) { + if (any_f16_feed) { g_rd.pf_dnof = make_device_buf(np * di * 2l) g_rd.pf_dnof_set = set_f16cvt_cls(fixed_array(g_gpu.dn_ws_dev, g_rd.pf_dnof), fixed_array(DN_WS_BYTES, np * di * 2l), fixed_array(VHZ_DNWS, VHZ_HQ)) @@ -724,10 +807,10 @@ def private pf_setup { fixed_array(VHZ_XB, VHZ_XQ, VHZ_XQ))) // (GEMM roles 1/2/3/7/10/11/13 build lazily per router variant at record time) if (!L.recurrent) { - // 4 rope+store (class: q, kv, Kmir, Vmir, cos) - let rbufs = fixed_array(g_rd.pf_q, g_rd.pf_kv, g_rd.k_mirror, g_rd.v_mirror, g_rd.pf_cos) - let rsizes = fixed_array(np * qrow * 4l, np * 2l * kvd * 4l, mirbytes, mirbytes, np * g_rd.head_size * 4l) - let rgbits = fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, VHZ_COS) + // 4 rope+store (class: q, kv, Kmir, Vmir, cos, bias) + let rbufs = fixed_array(g_rd.pf_q, g_rd.pf_kv, g_rd.k_mirror, g_rd.v_mirror, g_rd.pf_cos, g_rd.has_bias ? g_rd.bias_dev : g_rd.norms_dev) + let rsizes = fixed_array(np * qrow * 4l, np * 2l * kvd * 4l, mirbytes, mirbytes, np * g_rd.head_size * 4l, g_rd.has_bias ? g_rd.bias_bytes : g_rd.norms_bytes) + let rgbits = fixed_array(VHZ_Q, VHZ_KVK | VHZ_KVV, VHZ_MIR, VHZ_MIR, VHZ_COS, 0u) g_rd.pf_sets[b + 4] = (kv16 ? set_rope_kv_b_f16_cls(rbufs, rsizes, rgbits) : set_rope_kv_b_cls(rbufs, rsizes, rgbits)) // 5 attn (class: q, Kmir, Vmir, out) — the cm2 fa tile / h128 coopmat twin when they serve @@ -819,6 +902,7 @@ def private pf_setup { [hot_path] // the per-window resident prefill — encode + one submit per window def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int64; var logits : array) { let noids : array + vk_demotion_check("prefill") pf_run(x_batch, noids, false, 1.0, cos_batch, npos, logits) } @@ -826,6 +910,7 @@ def vk_rdec_prefill(x_batch : array; cos_batch : array; npos : int def vk_rdec_prefill_ids(tokens : array; emb_scale : float; cos_batch : array; npos : int64; var logits : array) { assert(g_rd != null && (g_rd.emb_blk >= 0l || g_rd.emb_f32 != 0ul), "vk_rdec_prefill_ids without a placed embd plane") let nox : array + vk_demotion_check("prefill") pf_run(nox, tokens, true, emb_scale, cos_batch, npos, logits) } @@ -1038,7 +1123,8 @@ def private pf_run(x_batch : array; ids : array; use_ids : bool; e var pcr = RopeKvBArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(g_rd.rot / 2l), neox = g_rd.neox ? 1u : 0u, kpair0 = uint(qd / 2l), ppp = uint(pairs), layerbase = uint(mirb), npos = uint(wlen), pos0 = uint(w0), - voff = uint(kv_merged ? kvd : wlen * kvd), kstride = uint(kv_merged ? 2l * kvd : kvd), qhs = uint(qhs)) + voff = uint(kv_merged ? kvd : wlen * kvd), kstride = uint(kv_merged ? 2l * kvd : kvd), qhs = uint(qhs), + boff = uint(l * (qd + 2l * kvd)), hasb = g_rd.has_bias ? 1u : 0u) if (g_rd.kv16) { enc_rope_kv_b_f16_cls(raw, h, g_rd.pf_sets[b + 4], pcr, ropewg) } else { @@ -1567,7 +1653,7 @@ def private cmd_copy_batch_acts(raw : VkCommandBuffer; xq_bytes, xs_bytes : int6 // build the chunk's schedule: regions clipped to the row window [r0, r1) (row0 re-based to the // window), then the per-workgroup region map. Weight offsets follow the stack's format units. // Returns the workgroup count; `n` rides the push block (param removal = the deletion sweep's) -[unused_argument(n), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[unused_argument(n), arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def private fill_batch_meta(si : int; offs : int64 const?; nregions : int64; n, d : int64; r0, r1 : int64; cm2 : bool = false) : uint { let fmt = g_gpu.stacks[si].fmt let unit = fmt_unit(fmt) @@ -1624,7 +1710,7 @@ let private BATCH_CLS_CM2S = 7 // batch_cls_sets slots of the cm2 s / m / l ti let private BATCH_CLS_CM2M = 8 let private BATCH_CLS_CM2L = 9 -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def private stack_batch_enc_cm2(raw : VkCommandBuffer; var h : VkHaz; si : int; nwg : uint; n, d : int64) { unsafe { var st & = g_gpu.stacks[si] @@ -1651,7 +1737,7 @@ def private stack_batch_enc(raw : VkCommandBuffer; var h : VkHaz; si : int; nwg var st & = g_gpu.stacks[si] let fmt = st.fmt // Q4_0 rides the f16 coopmat arm too (no s8-native for nibbles — mode 2 falls back to sdot4) - let q40cm = fmt == int(KqFmt.q40) && g_gpu.coopmat_mode == 1 + let q40cm = fmt == int(KqFmt.q40) && g_gpu.coopmat_mode == COOPMAT_F16 var v = 6 // the stack's fixed class (kq); 5 = the q40 f16 MMA twin if (fmt == int(KqFmt.q8)) { v = q8_batch_cls_variant(st.batch_tile, st.batch_aligned) @@ -1988,7 +2074,7 @@ def vk_moe_dn(var yp : float?; woq, woz, woo : int64; return } assert(ds <= DN_MAX_DS && ds % 16l == 0l && (ds & (ds - 1l)) == 0l && nvh <= DN_MAX_HEADS && cd <= DN_MAX_CD - && dconv <= DN_MAX_DCONV && di == nvh * ds && kd == nkh * ds && cd == 2l * kd + di + && 2l * kd <= DN_MAX_QK && dconv <= DN_MAX_DCONV && di == nvh * ds && kd == nkh * ds && cd == 2l * kd + di && npos > taps && dim % 32l == 0l && di % 32l == 0l, "dasLLAMA vulkan tier: deltanet geometry outside chain capacity") ensure_batch_state() @@ -2069,15 +2155,21 @@ def private ensure_at_state { g_gpu.at_v_dev = make_device_buf(AT_CTX * AT_MAX_KV * 4l) g_gpu.at_vraw_dev = make_device_buf(AT_WINDOW * AT_MAX_KV * 4l) g_gpu.at_kv_host = make_host_buf(2l * AT_WINDOW * AT_MAX_KV * 4l, true, [cached = true]) - // prep (class: GEMM rows, prepped panel, gate stash, smalls) — the k form's gate view never fires + g_gpu.at_bias_dev = make_device_buf((AT_MAX_QD + 2l * AT_MAX_KV) * 4l) + // prep (class: GEMM rows, prepped panel, gate stash, smalls, projection bias) — the k/v forms' gate view never fires g_gpu.at_prep_q_set = set_at_prep_cls( - fixed_array(g_gpu.batch_y1_dev, g_gpu.at_q_dev, g_gpu.at_g_dev, g_gpu.at_smalls_dev), - fixed_array(BATCH_Y_BYTES, AT_WINDOW * AT_MAX_QD * 4l, AT_WINDOW * AT_MAX_QD * 4l, AT_SMALLS_BYTES), - fixed_array(VHB_Y1, VHB_ATQ, VHB_ATG, VHB_SM)) + fixed_array(g_gpu.batch_y1_dev, g_gpu.at_q_dev, g_gpu.at_g_dev, g_gpu.at_smalls_dev, g_gpu.at_bias_dev), + fixed_array(BATCH_Y_BYTES, AT_WINDOW * AT_MAX_QD * 4l, AT_WINDOW * AT_MAX_QD * 4l, AT_SMALLS_BYTES, (AT_MAX_QD + 2l * AT_MAX_KV) * 4l), + fixed_array(VHB_Y1, VHB_ATQ, VHB_ATG, VHB_SM, VHB_ATBIAS)) g_gpu.at_prep_k_set = set_at_prep_cls( - fixed_array(g_gpu.batch_y2_dev, g_gpu.at_k_dev, g_gpu.at_g_dev, g_gpu.at_smalls_dev), - fixed_array(BATCH_Y_BYTES, AT_CTX * AT_MAX_KV * 4l, AT_WINDOW * AT_MAX_QD * 4l, AT_SMALLS_BYTES), - fixed_array(VHB_Y2, VHB_ATK, 0u, VHB_SM)) + fixed_array(g_gpu.batch_y2_dev, g_gpu.at_k_dev, g_gpu.at_g_dev, g_gpu.at_smalls_dev, g_gpu.at_bias_dev), + fixed_array(BATCH_Y_BYTES, AT_CTX * AT_MAX_KV * 4l, AT_WINDOW * AT_MAX_QD * 4l, AT_SMALLS_BYTES, (AT_MAX_QD + 2l * AT_MAX_KV) * 4l), + fixed_array(VHB_Y2, VHB_ATK, 0u, VHB_SM, VHB_ATBIAS)) + // IN PLACE on at_vraw: src and outp are the same buffer + g_gpu.at_prep_v_set = set_at_prep_cls( + fixed_array(g_gpu.at_vraw_dev, g_gpu.at_vraw_dev, g_gpu.at_g_dev, g_gpu.at_smalls_dev, g_gpu.at_bias_dev), + fixed_array(AT_WINDOW * AT_MAX_KV * 4l, AT_WINDOW * AT_MAX_KV * 4l, AT_WINDOW * AT_MAX_QD * 4l, AT_SMALLS_BYTES, (AT_MAX_QD + 2l * AT_MAX_KV) * 4l), + fixed_array(VHB_VRAW, VHB_VRAW, VHB_ATG, VHB_SM, VHB_ATBIAS)) // attn (class: q panel, k panel, v rows, gate/out plane) g_gpu.at_attn_set = set_at_attn_cls( fixed_array(g_gpu.at_q_dev, g_gpu.at_k_dev, g_gpu.at_v_dev, g_gpu.at_g_dev), @@ -2093,12 +2185,12 @@ def private ensure_at_state { //! Does the cm2 flash-attention tile serve this window? A coopmat2-fa device, the fa knob, head 64 //! or 128, and no gate (this chain wires neither the h256 stamps nor their gated epilogue - gated models keep at_attn). -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-expert-chain")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-expert-chain")] def private at_fa_serves(hs : int64; gated : bool) : bool { return !gated && g_gpu.has_coopmat2_fa && vk_fa_on() && (hs == 64l || hs == 128l) } -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-expert-chain")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-expert-chain")] def private ensure_at_fa_state(hs : int64) { if (g_gpu.at_fa_ready) { return @@ -2127,7 +2219,7 @@ def private ensure_at_fa_state(hs : int64) { [arch(at="../ARCHITECTURE_GPU_VULKAN.md#vk-prefill-window-chain"), arch(at="../ARCHITECTURE_GPU_VULKAN_DECODE.md#decode-attention-block")] def private record_at_cmd(s_q, s_k, s_v, s_o : int; nwg_q, nwg_k, nwg_v, nwg_o : uint; rows, w0, qd, kv_dim, dim, n_heads, hs, half, n_kv, kv_mul, o_nblk : int64; - axq_bytes, axs_bytes, sm_bytes : int64; o_kq, gated, qknorm, fa : bool; + axq_bytes, axs_bytes, sm_bytes : int64; o_kq, gated, qknorm, fa, hasb : bool; eps, scale : float; woq : int64) { let raw = g_gpu.batch_cmd let rf : VkCommandBufferResetFlags @@ -2152,14 +2244,21 @@ def private record_at_cmd(s_q, s_k, s_v, s_o : int; nwg_q, nwg_k, nwg_v, nwg_o : stack_batch_enc(raw, h, s_v, nwg_v, dim, kv_dim) pfq_ts(raw) let pflags = uint((gated ? 1 : 0) | (qknorm ? 2 : 0)) + let hb = hasb ? 1u : 0u var pcq = AtPrepArgs(d = uint(qd), nh = uint(n_heads), hs = uint(hs), half = uint(half), flags = pflags, rms_off = uint(AT_SM_RMSQ), cos_off = uint(AT_SM_COS), - sin_off = uint(AT_SM_COS + rows * half), obase = 0u, qsrc = 1u, eps = eps) + sin_off = uint(AT_SM_COS + rows * half), obase = 0u, qsrc = 1u, boff = 0u, hasb = hb, eps = eps) enc_at_prep_cls(raw, h, g_gpu.at_prep_q_set, pcq, rows) var pck = AtPrepArgs(d = uint(kv_dim), nh = uint(n_kv), hs = uint(hs), half = uint(half), flags = pflags, rms_off = uint(AT_SM_RMSK), cos_off = uint(AT_SM_COS), - sin_off = uint(AT_SM_COS + rows * half), obase = uint(w0), qsrc = 0u, eps = eps) + sin_off = uint(AT_SM_COS + rows * half), obase = uint(w0), qsrc = 0u, boff = uint(qd), hasb = hb, eps = eps) enc_at_prep_cls(raw, h, g_gpu.at_prep_k_set, pck, rows) + if (hasb) { + var pcv = AtPrepArgs(d = uint(kv_dim), nh = uint(n_kv), hs = uint(hs), half = 0u, + flags = 0u, rms_off = 0u, cos_off = 0u, sin_off = 0u, obase = 0u, qsrc = 0u, + boff = uint(qd + kv_dim), hasb = 1u, eps = eps) + enc_at_prep_cls(raw, h, g_gpu.at_prep_v_set, pcv, rows) + } pfq_ts(raw) // append raw v at absolute positions vhz_dep(raw, h, VHB_VRAW, VHB_ATV, true) @@ -2225,8 +2324,9 @@ def vk_moe_attn(var yp : float?; var kp : float?; var vp : float?; woq, wok, wov, woo : int64; fq, fk, fv, fo : int; xqp : int8 const?; xsp : float const?; rmsqp : float const?; rmskp : float const?; + bqp : float const?; bkp : float const?; bvp : float const?; cosp : float const?; sinp : float const?; - npos, dim, qd, kv_dim, hs, rot, n_heads, kv_mul : int64; + npos, dim, qd, kv_dim, hs, rot, n_heads : int64; scale : float; eps : float; gated : bool; qknorm : bool) { let s_q = find_stack(woq, fq, "prefill attn q") let s_k = find_stack(wok, fk, "prefill attn k") @@ -2239,6 +2339,7 @@ def vk_moe_attn(var yp : float?; var kp : float?; var vp : float?; "dasLLAMA vulkan tier: q/k/v stack format class mismatch in one attention chain") let half = rot / 2l let n_kv = kv_dim / hs + let kv_mul = n_kv > 0l ? n_heads / n_kv : 0l assert(npos <= AT_CTX && qd <= AT_MAX_QD && kv_dim <= AT_MAX_KV && (hs == 128l || hs == 256l) && half >= 1l && 2l * half <= hs && n_heads * hs == qd && n_kv * hs == kv_dim && n_heads == n_kv * kv_mul @@ -2268,6 +2369,14 @@ def vk_moe_attn(var yp : float?; var kp : float?; var vp : float?; memcpy(reinterpret(sp + AT_SM_RMSK), reinterpret(rmskp), hs * 4l) } } + let hasb = bqp != null + if (hasb) { + unsafe { + upload_region_at(g_gpu.at_bias_dev, 0l, reinterpret(bqp), qd * 4l) + upload_region_at(g_gpu.at_bias_dev, qd * 4l, reinterpret(bkp), kv_dim * 4l) + upload_region_at(g_gpu.at_bias_dev, (qd + kv_dim) * 4l, reinterpret(bvp), kv_dim * 4l) + } + } var q_region = fixed_array(woq, 0l, 0l) var k_region = fixed_array(wok, 0l, 0l) var v_region = fixed_array(wov, 0l, 0l) @@ -2305,7 +2414,7 @@ def vk_moe_attn(var yp : float?; var kp : float?; var vp : float?; rows, r0, qd, kv_dim, dim, n_heads, hs, half, n_kv, kv_mul, rows * qd / (fo != 0 ? 256l : 32l), rows * dim, rows * (dim / xunit) * 4l, (AT_SM_COS + 2l * rows * half) * 4l, - fo != 0, gated, qknorm, fa, eps, scale, woq) + fo != 0, gated, qknorm, fa, hasb, eps, scale, woq) submit_wait(g_gpu.batch_cmd) let ts2 = ref_time_ticks() unsafe { @@ -2527,7 +2636,7 @@ def vk_moe_ffn_streamed_xf(g : int; var goutp : float?; offs1 : int64 const?; of //! MoeGpuFfnXfOkFn: does the cm2 chain serve a layer whose triple is (f1, f3, f2) at this window? //! Mode 4 on a coopmat2 device, every stack a format the f16 feed admits, and the window //! inside the f32 x plane's cap. The combined (npos > 0) form only. -[arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-expert-chain")] +[arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice"), arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-expert-chain")] def vk_moe_ffn_xf_ok(npos, n : int64; f1, f3, f2 : int) : bool { if (g_gpu == null || g_gpu.dry || !g_gpu.has_coopmat2) { return false @@ -2556,11 +2665,11 @@ def private ensure_xf_state { to_log(LOG_INFO, "dasLLAMA vulkan tier: cm2 expert chain engaged\n") } -let private TILE_READ_SLACK = 32l // the s tile's unclamped column reads up to 31 rows past a region's last row (ARCHITECTURE_GPU_VULKAN.md sec.2.2l) +let private TILE_READ_SLACK = 32l // rows of read slack past a region's last row //! chunk row cap of the cm2 chain: the gathered f16 image (batch_xq window), the f16 hidden plane //! (hq_dev) and the f32 output planes must all fit; 128-aligned like the tiles' columns -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] def private ffn_cm2_chunk_rows(n, nfe : int64) : int64 { var r = min(BATCH_XQ_BYTES / (n * 2l), BATCH_HQ_BYTES / (nfe * 2l)) - TILE_READ_SLACK r = min(r, BATCH_Y_BYTES / (max(nfe, n) * 4l)) @@ -2569,7 +2678,7 @@ def private ffn_cm2_chunk_rows(n, nfe : int64) : int64 { return rows } -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-expert-chain")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-expert-chain")] def private record_ffn_cm2_cmd(s1, s3, s2 : int; nwg_gu, nwg_dn : uint; rows, n, nfe, nk, npos : int64; r0, r1 : int64; is_gelu : bool) { let raw = g_gpu.batch_cmd @@ -2643,7 +2752,7 @@ def private xf_record_chunk(s1, s3, s2 : int; offs1 : int64 const?; offs3 : int6 //! MoeGpuFfnXfFn: the cm2 route of the batch arm. `xp` = the window's f32 rows (npos x n, one //! per position); `invp`/`wp` = the combine's slot map and weights over nrows = npos x k bucket //! rows. Chunked over bucket-row windows like the quant arm; combined rows read back once at the end. -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-expert-chain")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-expert-chain")] def vk_moe_ffn_batch_xf(var goutp : float?; offs1 : int64 const?; offs3 : int64 const?; offs2 : int64 const?; nregions : int64; xp : float const?; wp : float const?; invp : uint const?; n, nfe, nrows, npos : int64; f1, f3, f2 : int; is_gelu : bool) { let st = xf_prepare(offs1, offs3, offs2, xp, wp, invp, n, nrows, npos, f1, f3, f2) let chunk = ffn_cm2_chunk_rows(n, nfe) @@ -2678,7 +2787,7 @@ def vk_moe_ffn_xf_async_rows(n, nfe : int64) : int64 => ffn_cm2_chunk_rows(n, nf //! MoeGpuFfnXfBeginFn: the cm2 chain over the given regions, submitted and left in flight - the //! streamed-layer split runs its CPU regions meanwhile; vk_moe_ffn_batch_xf_join collects. -[arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-expert-chain")] +[arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-expert-chain")] def vk_moe_ffn_batch_xf_begin(offs1 : int64 const?; offs3 : int64 const?; offs2 : int64 const?; nregions : int64; xp : float const?; wp : float const?; invp : uint const?; n, nfe, nrows, npos : int64; f1, f3, f2 : int; is_gelu : bool) { assert(!g_xf_pending, "dasLLAMA vulkan tier: async cm2 chain begin while one is already in flight") let cap = ffn_cm2_chunk_rows(n, nfe) diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_seams.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_seams.das index 4af8b8734a..1ee7dfebfb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_seams.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_seams.das @@ -81,10 +81,10 @@ def private ensure_rks_state { g_gpu.rks_q_host = make_host_buf(RKS_MAX_QD * 4l, false, [cached = true]) g_gpu.rks_kv_host = make_host_buf(2l * RKS_MAX_KVD * 4l, false, [cached = true]) g_gpu.rks_set = set_rope_kv_cls( - fixed_array(g_gpu.rks_q_dev, g_gpu.rks_qkv_dev, g_gpu.rks_k_dev, g_gpu.rks_v_dev, g_gpu.rks_scal.buf, g_gpu.rks_tok.buf), + fixed_array(g_gpu.rks_q_dev, g_gpu.rks_qkv_dev, g_gpu.rks_k_dev, g_gpu.rks_v_dev, g_gpu.rks_scal.buf, g_gpu.rks_tok.buf, g_gpu.rks_scal.buf), fixed_array(RKS_MAX_QD * 4l, (RKS_MAX_QD + 2l * RKS_MAX_KVD) * 4l, - RKS_MAX_CTX * RKS_MAX_KVD * 4l, RKS_MAX_CTX * RKS_MAX_KVD * 4l, XS_BYTES, 16l), - fixed_array(VHG_Y1, 0u, VHG_Y2, VHG_Y3, 0u, 0u)) // roped q / K mirror / V mirror + RKS_MAX_CTX * RKS_MAX_KVD * 4l, RKS_MAX_CTX * RKS_MAX_KVD * 4l, XS_BYTES, 16l, XS_BYTES), + fixed_array(VHG_Y1, 0u, VHG_Y2, VHG_Y3, 0u, 0u, 0u)) // roped q / K mirror / V mirror; the bias slot is a never-read placeholder (the seam has no bias arm) g_gpu.rks_ready = true } diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 47078c8629..a77738163c 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -1370,3 +1370,14 @@ same way would be a few MB. Same nature: `dasModuleVulkan` (11-17 MB) ships in every bundle, the macOS one included, where it only backs the tier probe. Neither is a priority - the bundles are peanuts beside a gguf - but both are where the archive shrinks if it ever has to. +120. **The image mint holds two whole models.** Loading a 48 GB Qwen3-Coder-Next Q4_K_M + through `load_model_cached` on a 256 GB box died "out of heap memory, requested 91 GB, used + 232 GB": the mint path keeps the converted planar model (48 GB served, the 46 GB source + prefetch, the pinned mirrors of 37 streamed layers' experts) while it writes the `.dlim` and + then loads AGAIN to map it; when the save aborted ("plane 'k4q' failed writing 32 GB - disk + full") it fell into a second full conversion beside the first. The aborted-save half is fixed: + a declined save now serves the streamed build from memory. The other half stands: on a good + save release the planar model before the map (or map the just-written planes without a second + read). The disk was full because the comparison bench mints an image per model and per flavor + by default - 37 debug-jit images, 293 GB, in one day's walk; the bench under + `--for-debug-purposes` should run `DASLLAMA_IMAGE=0` itself, since its flavor serves no rig. diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md index b67ebd9275..2fdd02cd78 100644 --- a/modules/dasLLAMA/followup_metal.md +++ b/modules/dasLLAMA/followup_metal.md @@ -91,6 +91,44 @@ Measured ceiling of ALL remaining converts (`DASLLAMA_METAL_PREFILL_SKIP=act_cvt re-opens the numerics bars across the parity and prefill suites. ITS OWN ARC, planned; the acceptance bar is the existing parity suites plus a pp/tg board A/B per class. +## 6. The Metal kernel twins - the census for the Mac session (Boris, 2026-09-07) + +The full census is `plans/kernel_twins_census.md` section 2 (every `[metal_dispatch]` class read, +suspected twins compared line by line); this entry is the map so the research is not done twice. +Boris: the Mac kernels are a follow-up done on the Mac, together with the Metal side of the Qwen +zoo. Facts that decide the order: + +- 296 dispatch classes in `dasllama_metal_kernels.das` (175) + `dasllama_metal_prefill.das` + (121): 173 are template stamps, 14 ride a `def abstract` base, ~44 are hand-written twins + (~3000 lines addressable), ~65 singletons; 15 near-miss pairs must stay apart (listed there). +- Two thirds of the debt is one family: the per-format GEMV, MvB2/B4 and MvB8 copies - 36 + classes over 12 formats (`MetalKqGemv*` 7985-9319, `MetalKqMv*T` 8368-10970, `MetalKqMvB8*` + 8477-11102) where iq4xs and iq4nl differ in ONE line and iq2s and iq2xs in eight; the shells + are byte-identical. One format-abstract shell per family with one `def abstract` decode per + format (`stage_w16` / `dot_block`) closes ~1930 lines; `MetalKqGemvIq2xxsFam` and + `MetalKqMulMmSplitTensorBase.stage16` already prove the shape in-tree. Keep the per-box crowns + (`K5C`, `Iq3sF4`, `Iq3xxsF4`) as separate stamps; the F4 forks bundled a second change + (`first_row * 2u` -> `* 4u`, `sumf[2]` -> `[4]`, `ib += 4u` -> `2u`) = two constants, F4 + ROWS; + the `TILED` arm duplicates the b-loop for a measured +2% (k4) - prove the generated MSL + byte-identical per stamp before and after. +- Then: the SqAttn `BATCHED` axis (10 templates, ~250 lines; `MetalSqAttnCombT` ships the exact + pattern); the `MetalKqMulMmK45T` 12-bool `static_if` ladder into the `stage16` scaffold plus + the tensor K45/K6 x Db pair (~285; removes the coupled-bool trap where `MetalKqMulMmIq4nl` + must set `IQ4XS` and `IQ4NL`; the Db forms sit on the sanctioned float-A list); the four dense + mul_mm shells onto a `MetalMoeMulMmBase` twin (~145); the MoE GEMV `GATHERED` axis (~230, the + `float4` x view stays its own axis - a measured 2.25x); the zero-risk singles (CrossVx f16/f32, + Q8MvB2/B4 onto `MetalGemvB24T`, argmax rows, rope-store batched, Q8MulMmDb, DequantK6H, + G4aMag/Q3aPow, the bias pair; ~325). +- Rules for every conversion: a stamp's `tgmem=` string is `__msl_tgmem`, so a + hand class becoming a stamp changes it and drops its `[metal_kernel(name=..)]`; a + `@template_gate`d field may be named only inside a `static_if` arm on its own axis (a ternary + infers both arms); the kernel-unit gates (`tests/test_metal_gemv_kernels.das`, + `test_metal_gemm_kernels.das`) are the parity lock per format - green before and after, on the + M1 first, the M5 pass after. +- Detect-dupe (`utils/detect-dupe`) over the two files finds the exact-clone shells and the + copied helpers; it does not see a twin whose bodies differ by a guard or an accumulator + count, which is why the census reads the bodies. + ## 5. The dense-KQ tensor twins' missing stamps The nine iquant/split-scale tensor mul_mm twins stamp `T` and `TH` only. k4/k5/k6 additionally diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index ef5877728b..313e8d8cd8 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -371,12 +371,17 @@ Ordered roughly by user-visible value; re-rank against zen2 measurements before vulkan-tagged tests) or make it refuse loudly when a test declares a vulkan requirement it cannot satisfy. -17. **VK_EXT_pageable_device_local_memory - the missing half of the residency shield.** The - tier chains VK_EXT_memory_priority (priority 1.0 on every device allocation, the armed - "residency shield"); the companion extension - runtime `vkSetDeviceMemoryPriorityEXT` + - the pageable-aware device-local signal WDDM wants - has zero references in the tree. - Small addition: enable when present, and consider demoting cold stacks' priority instead - of only boosting everything. +17. **The residency plan off Windows still guesses.** On Windows the plan sizes against the OS + video memory budget (`ARCHITECTURE_GPU_VULKAN_RESIDENCY.md` sec.2.2n, the `os_video_memory` + boost helper); elsewhere the 27% headroom share stands, measured on the tagged arm and never + re-measured. Two tails: (a) Linux NVIDIA fails an allocation past the card with + `OUT_OF_DEVICE_MEMORY`, which `vk_check` turns into a process death - the arena reserve and + the mirror allocation should catch it and retry the plan at a shorter context, the remedy + the plan already carries; (b) AMD's kernel driver evicts buffers the way WDDM does, so the + Mesa `heapBudget` (kernel accounting, system-wide) should feed the plan the way the Windows + budget does, and the demotion check needs an AMD reading. The 8 GB-card ladder that set the + fixed 2 GiB term is also tagged-arm data. The boost creators enable both memory extensions + whenever the card reports them; `vkSetDeviceMemoryPriorityEXT` stays unreferenced. MEASURED INCIDENT (2026-08-27, zen2): the ReBAR weight arena (mapped host-visible|device-local heap, "uploads write direct to VRAM") LOST WDDM residency mid-session - every weight-reading role fell to PCIe speed (decode 254 ms/token = 13.4 @@ -631,16 +636,16 @@ module) is independent and can land any time - it is pure structure. those is the memo's delta 1 (scale hoist), delta 2 (scale interleave) and delta 8 (codebook and prologue) rows, each behind its own probe A/B. -35. **The grid-format GEMV workgroup re-stage is a fixed per-workgroup cost - amplified on - small models.** Every u64-grid gemv (iq2s 8 KB, iq2xs 4 KB) stages the codebook into - workgroup memory per 2-row workgroup, so tg pays a fixed latency the row length must - amortize. On the 3B i1 vehicle iq2s tg landed 0.81x llama.cpp (~350 GB/s effective); - the 1B IQ2_XS vehicle lands 0.54x (188.7 vs 349.9 t/s = ~84 GB/s effective - latency- - bound, while its cm2 pp512 sits at a healthy 0.77x). Levers, in likely order: persist - the staged grid across the row loop (one stage per SM residency, not per workgroup), - widen rows-per-workgroup for grid formats, or fold the grid into a device-buffer read - the L2 serves. Done = 1B-class grid-format tg within the k-format band on the same - vehicle. +35. **The grid-format tiles still stage their codebooks from the constant composite.** The + GEMV half is done: the five grid GEMVs stage from the family's grid buffer + (`ARCHITECTURE_GPU_VULKAN.md` sec.2.2ab) and sit in the k-format band (iq2s 84 -> 388 + GB/s at the 27B gate shape; the serial constant read, not the stage's bytes, was the + cost). `KqBatch*` and the cm2 `IQ*GRID` axes still stage through the `*_grid_word` + accessors - amortized over a tile, so no measured loss yet; the buffer form is there + when a probe row says otherwise. The other lever stands: rows-per-subgroup > 1 for the + small-d roles (a 27B k/v projection dispatches 1024 two-row workgroups), the shell's + `rg` loop striding the grid, so a stage covers more rows. Done = a small-d grid GEMV + row within the k-format band in `harness/vk_gemv_probe.das`. 36. **Hand-laid four-wide decode twins for iq3s, iq2s and iq2xxs.** The emitter's synthesized twin (four calls of the scalar decode) loses on three formats (`harness/vk_gemm_probe.das -- @@ -731,7 +736,11 @@ module) is independent and can land any time - it is pure structure. run exits clean. (e) The tier's kernels assume a 32-lane subgroup - the shuffle reductions' xor ladders, the scan's lane-pair arithmetic (`lanes_per_pair`), the GEMV's rows per workgroup - and the init refuses a smaller `subgroupSize`; a wider one (a wave64 device) runs those - arms unmeasured. Done = a validation message naming `qk_rms_cls` by name, and a wave64 run + arms unmeasured. The KHR kq tile (`KqCm2BatchT`'s `KHR` arm, 2026-09-07) is the one site + that GATES on it instead: eight subgroups own its 128 weight rows, so `pf_f16_feed` admits + a kq format in mode 3 only at subgroup 32 (`khr_kq_tile_on`), and a wave64 card keeps its kq planes on + the sdot4 batch tile - the wave64 twin (four subgroups, 32 rows each) is owed with the + wave64 run. Done = a validation message naming `qk_rms_cls` by name, and a wave64 run of the kernel-unit suite. The emitter's operand laziness itself - eager `select` / bool `&` `|` as language surface, the purity heuristic dropped, the 12 kernel sites converted - is `plans/shader_emitter_followups.md` item 1. @@ -740,3 +749,68 @@ module) is independent and can land any time - it is pure structure. with nothing to assert registers `t |> skip`, so on a box without a Vulkan device every cell of the model-free suite reports PASS having done nothing - the hybrid ladder's fa/dn cells follow the file's idiom. Done = a file-wide conversion to `t |> skip` on the no-device and no-cm2 paths. +42. **End of the 27B arc: the cm2-disabled sweep.** Every 27B row on the board is a cm2 row + (NV_cooperative_matrix2 on the 5060 Ti). Boris (2026-09-07): measure each file with the + tensor tiles off, both engines - ours `DASLLAMA_COOPMAT=mm` (KHR coopmat mul_mm) and + `DASLLAMA_COOPMAT=sdot4` + `DASLLAMA_VK_FA=0` (no cooperative matrix at all), llama.cpp + `GGML_VK_DISABLE_COOPMAT2=1` and `GGML_VK_DISABLE_COOPMAT=1` - to learn what cm2 buys, + where we stand against llama.cpp on the same arm, and whether every format's non-cm2 tile + (the `KqBatch*` sdot4 tier serves every kq format) decodes and prefills correctly - the + bench's sanity argmax and logit must match the cm2 run's. The real proof of a lower target + is a run on one (a Turing or Ampere card, an RDNA card): the knob arms exercise the kernels, + not the device-creation path a card without the extension takes. Done = the four arms on + UD-IQ4_XS, UD-Q3_K_XL and i1-IQ3_S in the PR body. The row from a non-5060 card is the + real-hardware pass, a follow-up arc after every family works here (Boris 2026-09-07): rented + boxes - AWS g4dn (T4, the KHR arm on NVIDIA's driver) and g6e/p4d (48-80 GB, the fully + resident 27B Q8 and 35B MoE), an RDNA3/4 card from a GPU marketplace (AWS's AMD parts predate + cooperative matrix) - each with a written plan of what to run where. + FIRST PAIR (UD-IQ4_XS, 5060 Ti, pin 14000, 2026-09-07): decode is flat on every arm (ours + 23.4 on all three, llama.cpp 24.1) and the sanity argmax holds (pp 13, tg 5709). Prefill: + cm2 860.8 vs 813.7; KHR coopmat 221.2 vs 677.5 (0.33x); no coopmat 215.7 vs 312.6 (0.69x). + The cause of the 0.33x: the kq formats have no KHR-coopmat tile - `DASLLAMA_COOPMAT=mm` + serves only the q8 planes on the mul_mm L-tile and every kq plane on the `KqBatch*` sdot4 + tile, so mm and sdot4 read the same on a kq-only file. A card without NV_coopmat2 (every + AMD and Intel part, the GTX line) prefilled a 27B at a third of the reference exe's rate. + The fix was the format decode on the mul_mm L-tile, the way it moved onto the cm2 template. + THE KHR kq TILE LANDED (2026-09-07 evening, `ARCHITECTURE_GPU_VULKAN_GEMM.md` sec.2.2l): the + cm2 template's KHR arm, every kq format, 0 of 89600 off on all thirteen; the 4B Q4_K_M mm row + 1564 -> 2378 (0.56x llama.cpp's KHR 4221), the 27B UD-IQ4_XS 221 -> 395 (0.59x of 675), decode + unchanged. Its three measured steps: the decode call on the plane element (the block copy ran + the tile at a third), 16-byte activation loads, a 32-deep k step (the 64-deep one blew the + 49152 B workgroup cap on the iq2 grids). THE SLAB (the same evening): the probe's copy of the + tile body read 33 TFLOP/s where the shipped class read 25.6, and the difference was the 8 KB + edge-store slab the shipped class carried beside its two 10 KB staging arrays - the probe's + `slab` arm, the copy plus that one array touched on a path no dispatch takes, reads 28.5: + 8 KB more shared memory per workgroup is one workgroup fewer per SM. The staging arrays are + now `uint` f16 pairs (the activation words stored as they arrive, no unpack) and the edge + tile's f32 fragments bounce through the weight array once the k loop is done, the row guards + hoisted out of the loop: the shipped class 25.6 -> 32.1 / 23.9 -> 31.4 / 25.6 -> 32.2 on the + gate / down / q shapes, the kernel suite 108 of 108 with its edge-tile cells, the hybrid file + 10 of 10 on the KHR arm, the 4B Q4_K_M mm row 2378 -> 3051 (0.72x of llama.cpp's 4221; the + probe's khr row 32.6 / 31.8 / 32.6 on a box holding 1.3 GB for other processes). The 27B + UD-IQ4_XS row's re-measure is owed: at 1.2 GB held by other processes the resident plan's + KV room fell to 147 MB, under the 2048-position minimum, and the driver declined - the row + needs the box the board's 395.2 was taken on (about 650 MB held). + WHERE THE REST IS: llama.cpp's non-cm2 path does not + run K-quants on tensor cores at all - `quantize_y` (ggml-vulkan.cpp, needs integer dot and no + coopmat2) routes them to the integer MMQ tile (`matmul_q4_k_q8_1`: 128 threads, 128 x 128 x 32, + Q8_1 activations with a per-32 (d, sum), a 4 x 32 f32 register block per thread over + dotPacked4x8, the sub-block (d*sc, dmin*m) folded per 32-k block), measured 43-46 TFLOP/s-eq + on the 9728/4096 x 2560 shapes and 27-31 on the 80-workgroup ones (its Vulkan build under + `GGML_VK_PERF_LOGGER=1`, the 4B prefill on the 5060 Ti) against our f16 KHR tile's 32 on the + large shapes (`harness/vk_gemm_probe.das -- khrx`, same card, after the slab fix above; 25 before it). The road to parity on the KHR arm is that tile on our side - a new body with + per-format int8 word decodes (the sdot4 `KqBatch` tier's `stage_w` is the same decode at a + 32 x 32 superblock shape) - and it lifts every integer-dot card with or without coopmat. + THE FIRST PROTOTYPES (2026-09-07, `harness/vk_gemm_probe.das -- mmqx`): three register-block + shapes over the sdot4 k4 staging (64 x 32 at 4 x 8 per thread staged per superblock; the + reference exe's one-block stage at 64 x 128, 4 x 16; that stage at 4 x 4 as named scalars) + all read 8.8-9.2 TFLOP/s against the shipped tile's 12.2, and the named-scalar twin with + CONSTANTS in place of the plane reads reads 11.0 - the inner loop caps the shape, not the + memory: per block a thread issues 24 shared loads and 160 sdot4 for 16 outputs, the reference + exe's 4 x 32 block 40 loads for 1024. Two prerequisites before the next prototype: (a) the + SPIR-V emitter unrolls `for [unroll] (i in range(N))` at emission (today it emits a hinted + loop, and a fixed-array local stays a Function-storage variable indexed by the loop counter - + `plans/shader_emitter_followups.md`), so a 128-accumulator block can be written as an array + with constant indices the driver promotes to registers; (b) a hardware profile of the ceiling + twin (Nsight, not another blind bisect) to see whether the sdot4 issue rate, the shared-load + rate or the barrier stalls at two workgroups per SM hold it at 11. diff --git a/modules/dasLLAMA/harness/REVIEW.md b/modules/dasLLAMA/harness/REVIEW.md index fe00d1405a..9471d879be 100644 --- a/modules/dasLLAMA/harness/REVIEW.md +++ b/modules/dasLLAMA/harness/REVIEW.md @@ -17,3 +17,9 @@ tune-sidecar service: sidecar lookup, download, or submit - is a defect; it call through `dasllama/dasllama_exchange` (the module `../dasllama/dasllama_exchange.das` declares) instead.** A `dashv` call at a reference engine's server (a llama-server on localhost) is not the exchange. + +**A `[vk_dispatch]` class in this folder never silences the lens's never-accessed check with +`@role = "alias"` or `"weight"` on a binding its kernel reads through a `dasllama/` base's +body - it reads that binding in a body this folder's file declares.** The lens collects +accesses from this file's bodies only, and `@role` keeps the access out of the hazard masks, so +a base-read binding silenced here escapes the hazard rail. diff --git a/modules/dasLLAMA/harness/_vk_probe_fixture.das b/modules/dasLLAMA/harness/_vk_probe_fixture.das new file mode 100644 index 0000000000..953dde4310 --- /dev/null +++ b/modules/dasLLAMA/harness/_vk_probe_fixture.das @@ -0,0 +1,22 @@ +options gen2 +options indenting = 4 + +// The synthetic plane fixture the Vulkan probes share: a word hash for quant bytes and a packed +// f16-pair scale word, the same shapes the kernel suite's fixtures use. + +module _vk_probe_fixture shared public + +//! one hashed word of synthetic quant bytes for index i +def hash_word(i : uint) : uint { + var x = i * 2654435761u + x ^= x >> 16u + x *= 2246822519u + x ^= x >> 13u + return x +} + +//! two packed f16 scales per word, small positive values (the kernel-suite fixture shape) +def cool_f16_pair(i : uint) : uint { + let h = 0x2c00u + (i % 64u) + return h | (h << 16u) +} diff --git a/modules/dasLLAMA/harness/vk_gemm_probe.das b/modules/dasLLAMA/harness/vk_gemm_probe.das index 4470344f14..6ec8ab6a2b 100644 --- a/modules/dasLLAMA/harness/vk_gemm_probe.das +++ b/modules/dasLLAMA/harness/vk_gemm_probe.das @@ -8,7 +8,8 @@ options _dasllama_internal = true // is tests/test_vulkan_kernels.das. // Args: gate|down|q|kv|kvm|qkv|tl pin a shape (the long-window ngfx GPU Trace vehicle); // k4|k6 the K-quant tiles; cm2: any kq format's l/m tiles; cm2x|k6x|ref the bisect and -// reference arms. +// reference arms; khrx the KHR kq tile's weight-stage and structural bisect; mmqx the integer tile +// prototypes. // Shapes default to Llama-3.2-3B Q8 geometry (dim 3072, hidden 8192, kv_dim 1024). require math @@ -22,24 +23,13 @@ require dasllama/dasllama_vulkan_dispatch require dasllama/dasllama_vulkan_classes require dasllama/dasllama_vulkan_prefill require dasllama/dasllama_kqformat +require daslib/typemacro_boost +require _vk_probe_fixture // hash_word, cool_f16_pair: the synthetic planes both probes build let private MM_TILE = 128 let private DISPATCHES = 16 let private SUBMITS = 100 - -def private hash_word(i : uint) : uint { - var x = i * 2654435761u - x ^= x >> 16u - x *= 2246822519u - x ^= x >> 13u - return x -} - -def private cool_f16_pair(i : uint) : uint { - // two packed f16 scales per word, small positive values (the kernel-suite fixture shape) - let h = 0x2c00u + (i % 64u) - return h | (h << 16u) -} +let private ROUNDS = 4 //! the interleaved sweeps: the arms alternate per round, SUBMITS / ROUNDS timed submits each, best-of across rounds struct private ShapeBufs { wq : array @@ -749,6 +739,1197 @@ class K6PxPair16 : K6PxBase { } } +let private KHRPX_STRIDE = 40u + +class KhrPxBase : MoeCmBase { + @ssbo @binding = 1 @role = "alias" @readonly ws : array + @ssbo @binding = 3 @role = "alias" @readonly xf16w4 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + @workgroup khr_a : float16[5120] + @workgroup khr_b : float16[5120] + + [spirv_decode] + def decode_v4(blk : VkK4Blk; bc, cib : uint2) : half4 { + let g = cib.y >> 5u + let e = cib.y & 31u + let li = int((g * 16u + (e & 15u)) >> 1u) + let l0 = uint(int(blk.qs[li])) & 0xFFFFu + let l1 = uint(int(blk.qs[li + 1])) & 0xFFFFu + let sh = (e >> 4u) * 4u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dm = unpackHalf2x16(ws[srow]) + let ds = dm.x * float((ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu) + let dmn = dm.y * float((ws[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu) + return half4(float4(ds * float((l0 >> sh) & 0xFu) - dmn, ds * float((l0 >> (sh + 8u)) & 0xFu) - dmn, + ds * float((l1 >> sh) & 0xFu) - dmn, ds * float((l1 >> (sh + 8u)) & 0xFu) - dmn)) + } + + //! an override fills khr_a[sbase .. sbase + 16) with weight row wrow's elements e0 .. e0 + 16 of block bcol + def stage_a(wblk0, wrow, bcol, e0, sbase : uint) { + for (j in range(16)) { + khr_a[sbase + uint(j)] = float16(0.0625) + } + } + + //! an override fills khr_b[sbase .. sbase + 16) from the activation plane at xb; this is the shipped stage, two 16-byte words + def stage_b(xb, sbase : uint) { + let q = xb >> 3u //! xb is a multiple of 16 halves + for (h in range(2)) { + let w4 = xf16w4[q + uint(h)] + for (c in range(4)) { + let p = unpackHalf2x16(w4[c]) + let o = sbase + uint(h) * 8u + uint(c) * 2u + khr_b[o] = float16(p.x) + khr_b[o + 1u] = float16(p.y) + } + } + } + + def body { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wblk0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 127u) / 128u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + barrier() + let t0 = row0 + xt * 128u + let m0 = wt * 128u + let tid = gl_LocalInvocationID.x + let sg = gl_SubgroupID + let srow = tid >> 1u + let skh = (tid & 1u) * 16u + let sbase = srow * KHRPX_STRIDE + skh + let wrow = m0 + srow + var acc0 : coopmatAcc_f32_16x16 + var acc1 : coopmatAcc_f32_16x16 + var acc2 : coopmatAcc_f32_16x16 + var acc3 : coopmatAcc_f32_16x16 + var acc4 : coopmatAcc_f32_16x16 + var acc5 : coopmatAcc_f32_16x16 + var acc6 : coopmatAcc_f32_16x16 + var acc7 : coopmatAcc_f32_16x16 + var k = 0u + while (k < pa.n) { + let kk = k + skh + let bcol = kk >> 8u + stage_a(wblk0, wrow, bcol, kk - bcol * 256u, sbase) + stage_b((t0 + srow) * pa.n + kk, sbase) + barrier() + for (ks in range(2)) { + let ko = uint(ks) * 16u + var a : coopmatA_f16_16x16 + coopmatLoad(a, khr_a, int(sg * 16u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 0) + var b : coopmatB_f16_16x16 + coopmatLoad(b, khr_b, int(ko), int(KHRPX_STRIDE), 1) + acc0 = coopmatMulAdd(a, b, acc0) + coopmatLoad(b, khr_b, int(16u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 1) + acc1 = coopmatMulAdd(a, b, acc1) + coopmatLoad(b, khr_b, int(32u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 1) + acc2 = coopmatMulAdd(a, b, acc2) + coopmatLoad(b, khr_b, int(48u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 1) + acc3 = coopmatMulAdd(a, b, acc3) + coopmatLoad(b, khr_b, int(64u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 1) + acc4 = coopmatMulAdd(a, b, acc4) + coopmatLoad(b, khr_b, int(80u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 1) + acc5 = coopmatMulAdd(a, b, acc5) + coopmatLoad(b, khr_b, int(96u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 1) + acc6 = coopmatMulAdd(a, b, acc6) + coopmatLoad(b, khr_b, int(112u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 1) + acc7 = coopmatMulAdd(a, b, acc7) + } + barrier() + k += 32u + } + let yb = t0 * pa.d + m0 + sg * 16u + coopmatStore(acc0, y, int(yb), int(pa.d), 1) + coopmatStore(acc1, y, int(yb + 16u * pa.d), int(pa.d), 1) + coopmatStore(acc2, y, int(yb + 32u * pa.d), int(pa.d), 1) + coopmatStore(acc3, y, int(yb + 48u * pa.d), int(pa.d), 1) + coopmatStore(acc4, y, int(yb + 64u * pa.d), int(pa.d), 1) + coopmatStore(acc5, y, int(yb + 80u * pa.d), int(pa.d), 1) + coopmatStore(acc6, y, int(yb + 96u * pa.d), int(pa.d), 1) + coopmatStore(acc7, y, int(yb + 112u * pa.d), int(pa.d), 1) + } +} + +[vk_dispatch(name = "khrpx_nil", grid = "wgs", params = "wgs : int64")] +class KhrPxNil : KhrPxBase { + @ssbo @binding = 0 @role = "alias" @readonly wq : array //! bound, never read: the ceiling stages a constant + + [spirv_kernel(local_size_x = 256, name = "khrpx_nil_spv")] + def run { + body() + } +} + +class KhrPxShipBase : KhrPxBase { + @ssbo @binding = 0 @readonly wq : array + + //! the shipped tile's weight stage: four decodes on the plane element into the row's 16 halves + def stage_decoded(wblk0, wrow, bcol, e0, sbase : uint) { + for (j in range(4)) { + let v4 = decode_v4(wq[wblk0 + wrow * (pa.n >> 8u) + bcol], uint2(wrow, bcol), uint2(0u, e0 + uint(j) * 4u)) + let o = sbase + uint(j) * 4u + khr_a[o] = v4.x + khr_a[o + 1u] = v4.y + khr_a[o + 2u] = v4.z + khr_a[o + 3u] = v4.w + } + } + + def override stage_a(wblk0, wrow, bcol, e0, sbase : uint) { + stage_decoded(wblk0, wrow, bcol, e0, sbase) + } +} + +[vk_dispatch(name = "khrpx_ship", grid = "wgs", params = "wgs : int64")] +class KhrPxShip : KhrPxShipBase { + [spirv_kernel(local_size_x = 256, name = "khrpx_ship_spv")] + def run { + body() + } +} + +//! ship plus the shipped tile's 8 KB edge-store slab, touched on a path no dispatch takes: the occupancy cost of the third workgroup array alone +[vk_dispatch(name = "khrpx_slab", grid = "wgs", params = "wgs : int64")] +class KhrPxSlab : KhrPxShipBase { + @workgroup khr_o : float[2048] + + def override stage_a(wblk0, wrow, bcol, e0, sbase : uint) { + stage_decoded(wblk0, wrow, bcol, e0, sbase) + if (pa.map_off == 0u) { + khr_o[sbase] = float(khr_a[sbase]) + khr_a[sbase] = float16(khr_o[sbase + 1u]) + } + } + + [spirv_kernel(local_size_x = 256, name = "khrpx_slab_spv")] + def run { + body() + } +} + +[vk_dispatch(name = "khrpx_nob", grid = "wgs", params = "wgs : int64")] +class KhrPxNoB : KhrPxShipBase { + @ssbo @binding = 3 @role = "alias" @readonly xf16 : array //! the B fragments load straight from the plane's half view + + def override body { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wblk0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 127u) / 128u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + barrier() + let t0 = row0 + xt * 128u + let m0 = wt * 128u + let tid = gl_LocalInvocationID.x + let sg = gl_SubgroupID + let srow = tid >> 1u + let skh = (tid & 1u) * 16u + let sbase = srow * KHRPX_STRIDE + skh + let wrow = m0 + srow + var acc0 : coopmatAcc_f32_16x16 + var acc1 : coopmatAcc_f32_16x16 + var acc2 : coopmatAcc_f32_16x16 + var acc3 : coopmatAcc_f32_16x16 + var acc4 : coopmatAcc_f32_16x16 + var acc5 : coopmatAcc_f32_16x16 + var acc6 : coopmatAcc_f32_16x16 + var acc7 : coopmatAcc_f32_16x16 + var k = 0u + while (k < pa.n) { + let kk = k + skh + let bcol = kk >> 8u + stage_a(wblk0, wrow, bcol, kk - bcol * 256u, sbase) + barrier() + for (ks in range(2)) { + let ko = uint(ks) * 16u + var a : coopmatA_f16_16x16 + coopmatLoad(a, khr_a, int(sg * 16u * KHRPX_STRIDE + ko), int(KHRPX_STRIDE), 0) + let bb = t0 * pa.n + k + ko + var b : coopmatB_f16_16x16 + coopmatLoad(b, xf16, int(bb), int(pa.n), 1) + acc0 = coopmatMulAdd(a, b, acc0) + coopmatLoad(b, xf16, int(bb + 16u * pa.n), int(pa.n), 1) + acc1 = coopmatMulAdd(a, b, acc1) + coopmatLoad(b, xf16, int(bb + 32u * pa.n), int(pa.n), 1) + acc2 = coopmatMulAdd(a, b, acc2) + coopmatLoad(b, xf16, int(bb + 48u * pa.n), int(pa.n), 1) + acc3 = coopmatMulAdd(a, b, acc3) + coopmatLoad(b, xf16, int(bb + 64u * pa.n), int(pa.n), 1) + acc4 = coopmatMulAdd(a, b, acc4) + coopmatLoad(b, xf16, int(bb + 80u * pa.n), int(pa.n), 1) + acc5 = coopmatMulAdd(a, b, acc5) + coopmatLoad(b, xf16, int(bb + 96u * pa.n), int(pa.n), 1) + acc6 = coopmatMulAdd(a, b, acc6) + coopmatLoad(b, xf16, int(bb + 112u * pa.n), int(pa.n), 1) + acc7 = coopmatMulAdd(a, b, acc7) + } + barrier() + k += 32u + } + let yb = t0 * pa.d + m0 + sg * 16u + coopmatStore(acc0, y, int(yb), int(pa.d), 1) + coopmatStore(acc1, y, int(yb + 16u * pa.d), int(pa.d), 1) + coopmatStore(acc2, y, int(yb + 32u * pa.d), int(pa.d), 1) + coopmatStore(acc3, y, int(yb + 48u * pa.d), int(pa.d), 1) + coopmatStore(acc4, y, int(yb + 64u * pa.d), int(pa.d), 1) + coopmatStore(acc5, y, int(yb + 80u * pa.d), int(pa.d), 1) + coopmatStore(acc6, y, int(yb + 96u * pa.d), int(pa.d), 1) + coopmatStore(acc7, y, int(yb + 112u * pa.d), int(pa.d), 1) + } + + [spirv_kernel(local_size_x = 256, name = "khrpx_nob_spv")] + def run { + body() + } +} + +[vk_dispatch(name = "khrpx_k64", grid = "wgs", params = "wgs : int64")] +class KhrPxK64 : MoeCmBase { //! its own base: the 32-step staging arrays would count against its footprint + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @readonly ws : array + @ssbo @binding = 3 @readonly xf16 : array + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint + @workgroup khr_a64 : float16[9216] + @workgroup khr_b64 : float16[9216] + + [spirv_decode] + def decode_v4(blk : VkK4Blk; bc, cib : uint2) : half4 { + let g = cib.y >> 5u + let e = cib.y & 31u + let li = int((g * 16u + (e & 15u)) >> 1u) + let l0 = uint(int(blk.qs[li])) & 0xFFFFu + let l1 = uint(int(blk.qs[li + 1])) & 0xFFFFu + let sh = (e >> 4u) * 4u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dm = unpackHalf2x16(ws[srow]) + let ds = dm.x * float((ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu) + let dmn = dm.y * float((ws[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu) + return half4(float4(ds * float((l0 >> sh) & 0xFu) - dmn, ds * float((l0 >> (sh + 8u)) & 0xFu) - dmn, + ds * float((l1 >> sh) & 0xFu) - dmn, ds * float((l1 >> (sh + 8u)) & 0xFu) - dmn)) + } + + def body { // nolint:STYLE038 — the whole-tile path at the 64-deep step, one register set + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wblk0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let ttiles = (cnt + 127u) / 128u + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let xt = tix % ttiles + let wt = tix / ttiles + if (gl_LocalInvocationID.x == 0u) { + wg_blk0 = wblk0 + } + barrier() + let t0 = row0 + xt * 128u + let m0 = wt * 128u + let tid = gl_LocalInvocationID.x + let sg = gl_SubgroupID + let srow = tid >> 1u + let skh = (tid & 1u) * 32u + let sbase = srow * 72u + skh + let wrow = m0 + srow + let nbk = pa.n >> 8u + var acc0 : coopmatAcc_f32_16x16 + var acc1 : coopmatAcc_f32_16x16 + var acc2 : coopmatAcc_f32_16x16 + var acc3 : coopmatAcc_f32_16x16 + var acc4 : coopmatAcc_f32_16x16 + var acc5 : coopmatAcc_f32_16x16 + var acc6 : coopmatAcc_f32_16x16 + var acc7 : coopmatAcc_f32_16x16 + var k = 0u + while (k < pa.n) { + let kk = k + skh + let bcol = kk >> 8u + let e0 = kk - bcol * 256u + for (j in range(8)) { + let v4 = decode_v4(wq[wblk0 + wrow * nbk + bcol], uint2(wrow, bcol), uint2(0u, e0 + uint(j) * 4u)) + let o = sbase + uint(j) * 4u + khr_a64[o] = v4.x + khr_a64[o + 1u] = v4.y + khr_a64[o + 2u] = v4.z + khr_a64[o + 3u] = v4.w + } + let xb = (t0 + srow) * pa.n + kk + for (j in range(32)) { + khr_b64[sbase + uint(j)] = xf16[xb + uint(j)] + } + barrier() + for (ks in range(4)) { + let ko = uint(ks) * 16u + var a : coopmatA_f16_16x16 + coopmatLoad(a, khr_a64, int(sg * 16u * 72u + ko), 72, 0) + var b : coopmatB_f16_16x16 + coopmatLoad(b, khr_b64, int(ko), 72, 1) + acc0 = coopmatMulAdd(a, b, acc0) + coopmatLoad(b, khr_b64, int(16u * 72u + ko), 72, 1) + acc1 = coopmatMulAdd(a, b, acc1) + coopmatLoad(b, khr_b64, int(32u * 72u + ko), 72, 1) + acc2 = coopmatMulAdd(a, b, acc2) + coopmatLoad(b, khr_b64, int(48u * 72u + ko), 72, 1) + acc3 = coopmatMulAdd(a, b, acc3) + coopmatLoad(b, khr_b64, int(64u * 72u + ko), 72, 1) + acc4 = coopmatMulAdd(a, b, acc4) + coopmatLoad(b, khr_b64, int(80u * 72u + ko), 72, 1) + acc5 = coopmatMulAdd(a, b, acc5) + coopmatLoad(b, khr_b64, int(96u * 72u + ko), 72, 1) + acc6 = coopmatMulAdd(a, b, acc6) + coopmatLoad(b, khr_b64, int(112u * 72u + ko), 72, 1) + acc7 = coopmatMulAdd(a, b, acc7) + } + barrier() + k += 64u + } + let yb = t0 * pa.d + m0 + sg * 16u + coopmatStore(acc0, y, int(yb), int(pa.d), 1) + coopmatStore(acc1, y, int(yb + 16u * pa.d), int(pa.d), 1) + coopmatStore(acc2, y, int(yb + 32u * pa.d), int(pa.d), 1) + coopmatStore(acc3, y, int(yb + 48u * pa.d), int(pa.d), 1) + coopmatStore(acc4, y, int(yb + 64u * pa.d), int(pa.d), 1) + coopmatStore(acc5, y, int(yb + 80u * pa.d), int(pa.d), 1) + coopmatStore(acc6, y, int(yb + 96u * pa.d), int(pa.d), 1) + coopmatStore(acc7, y, int(yb + 112u * pa.d), int(pa.d), 1) + } + + [spirv_kernel(local_size_x = 256, name = "khrpx_k64_spv")] + def run { + body() + } +} + +[vk_dispatch(name = "khrpx_inline", grid = "wgs", params = "wgs : int64")] +class KhrPxInline : KhrPxBase { + @ssbo @binding = 0 @readonly wq32 : array //! the k4 plane as words: one block = 64 lanes = 32 words + + def override stage_a(wblk0, wrow, bcol, e0, sbase : uint) { + let bw = (wblk0 + wrow * (pa.n >> 8u) + bcol) * 32u + let srow = (wg_blk0 + wrow * (pa.n >> 8u) + bcol) * 5u + let dm = unpackHalf2x16(ws[srow]) + for (j in range(4)) { + let e = e0 + uint(j) * 4u + let g = e >> 5u + let ee = e & 31u + let w = wq32[bw + ((g * 16u + (ee & 15u)) >> 2u)] //! lanes li, li + 1 share one word (li even) + let l0 = w & 0xFFFFu + let l1 = w >> 16u + let sh = (ee >> 4u) * 4u + let ds = dm.x * float((ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu) + let dmn = dm.y * float((ws[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu) + let o = sbase + uint(j) * 4u + khr_a[o] = float16(ds * float((l0 >> sh) & 0xFu) - dmn) + khr_a[o + 1u] = float16(ds * float((l0 >> (sh + 8u)) & 0xFu) - dmn) + khr_a[o + 2u] = float16(ds * float((l1 >> sh) & 0xFu) - dmn) + khr_a[o + 3u] = float16(ds * float((l1 >> (sh + 8u)) & 0xFu) - dmn) + } + } + + [spirv_kernel(local_size_x = 256, name = "khrpx_inline_spv")] + def run { + body() + } +} + +let private KHR_ARMS = 8 +let private MMQ_ARMS = 5 + +//! the interleaved timing of one sweep's arms: after a two-submit warm-up each, every round times +//! SUBMITS / ROUNDS submits of every arm in turn, and an arm's figure is its best round +def private sweep_best_us(var raws : array) : array { + var best : array + best |> resize(length(raws)) + for (raw, bv in raws, best) { + bv = 1.0e30lf + submit_wait(raw) + submit_wait(raw) + } + for (_r in range(ROUNDS)) { + for (raw, bv in raws, best) { + let t0 = ref_time_ticks() + for (_s in range(SUBMITS / ROUNDS)) { + submit_wait(raw) + } + let us = double(get_time_usec(t0)) / double((SUBMITS / ROUNDS) * DISPATCHES) + if (us < bv) { + bv = us + } + } + } + return <- best +} + +//! one more submit of an arm's command buffer, then the output buffer its last dispatch wrote read back into `out` +def private read_back_y(var raw : VkCommandBuffer; dev : uint64; bytes : int64; var out : array) { + submit_wait(raw) + out |> resize(bytes / 4l) + var host = make_host_buf(bytes, true, [cached = true]) + var rc = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(rc, begin), null) + cmd_copy_whole(rc, dev, host.buf, bytes) + vk_check(vkEndCommandBuffer(rc), null) + submit_wait(rc) + unsafe { + memcpy(addr(out[0]), host.mapped, bytes) + } +} + +def private max_rel_diff(a, b : array) : double { + var mx = 0.0 + var mref = 0.0 + for (x, y in a, b) { + mx = max(mx, abs(x - y)) + mref = max(mref, abs(y)) + } + return mref > 0.0 ? double(mx / mref) : 0.0lf +} + +def private khr_arm_set(v, fmt : int; wqd, wsd, scd, xfd, xqd, axsd, yd : uint64; wq_bytes, ws_bytes, sc_bytes, xf_bytes, xq_bytes, axs_bytes, y_bytes : int64) : VkDescriptorSet { + if (v == 4) { + return kq_batch_cls_set_for(fmt, false, fixed_array(wqd, wsd, scd, xqd, axsd, yd), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xq_bytes, axs_bytes, y_bytes), fixed_array(1u, 2u, 0u, 4u, 8u, 16u)) + } + let bufs = fixed_array(wqd, wsd, scd, xfd, yd) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + if (v == 0) { + return set_khrpx_nil(bufs, sizes, gbits) + } elif (v == 1) { + return set_khrpx_ship(bufs, sizes, gbits) + } elif (v == 2) { + return set_khrpx_inline(bufs, sizes, gbits) + } elif (v == 3) { + return set_kq_batch_k4_khr_cls(bufs, sizes, gbits) + } elif (v == 5) { + return set_khrpx_nob(bufs, sizes, gbits) + } elif (v == 6) { + return set_khrpx_k64(bufs, sizes, gbits) + } + verify(v == 7, "khrx: the arm table has eight arms") + return set_khrpx_slab(bufs, sizes, gbits) +} + +def private khr_arm_enc(v, fmt : int; raw : VkCommandBuffer; var hz : VkHaz; var s : VkDescriptorSet; var pc : BatchArgs; wgs : int64) { + if (v == 0) { + enc_khrpx_nil(raw, hz, s, pc, wgs) + } elif (v == 1) { + enc_khrpx_ship(raw, hz, s, pc, wgs) + } elif (v == 2) { + enc_khrpx_inline(raw, hz, s, pc, wgs) + } elif (v == 3) { + enc_kq_batch_k4_khr_cls(raw, hz, s, pc, wgs) + } elif (v == 4) { + kq_batch_cls_enc_for(fmt, false, raw, hz, s, pc, wgs) + } elif (v == 5) { + enc_khrpx_nob(raw, hz, s, pc, wgs) + } elif (v == 6) { + enc_khrpx_k64(raw, hz, s, pc, wgs) + } else { + verify(v == 7, "khrx: the arm table has eight arms") + enc_khrpx_slab(raw, hz, s, pc, wgs) + } +} + +[arch(at="../ARCHITECTURE_MEASUREMENT.md#one-benchmark-rig")] +def private run_khr_shape(name : string; d, n, cnt : int) { // nolint:STYLE038 — one interleaved measurement sweep over eight arms + let fmt = int(KqFmt.k4) + let nsb = n / 256 + let totsb = d * nsb + let bb = arena_block_bytes(fmt) + let wq_words = int(bb.wq / 4l) + let ws_words = int(bb.ws / 4l) + var wqh : array + var wsuh : array + var xfh : array + var xqh : array + var axsh : array + wqh |> resize(totsb * wq_words) + wsuh |> resize(totsb * ws_words) + xfh |> resize(cnt * n / 2) + xqh |> resize(cnt * nsb * 64) + axsh |> resize(cnt * nsb) + for (i in range(totsb * wq_words)) { + wqh[i] = hash_word(uint(i) + 29u) + } + for (sb in range(totsb)) { + wsuh[sb * ws_words] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.00005 * float(1 + sb % 5))) + for (wi in range(1, ws_words)) { + wsuh[sb * ws_words + wi] = hash_word(uint(sb * ws_words + wi) + 613u) + } + } + for (i in range(cnt * n / 2)) { + xfh[i] = cool_f16_pair(uint(i) * 3u + 5u) + } + for (i in range(cnt * nsb * 64)) { + xqh[i] = hash_word(uint(i) * 13u + 1u) + } + for (i in range(cnt * nsb)) { + axsh[i] = 0.0625 + } + let wq_bytes = int64(totsb) * bb.wq + let ws_bytes = long_length(wsuh) * 4l + let xf_bytes = int64(cnt * n) * 2l + let xq_bytes = long_length(xqh) * 4l + let axs_bytes = int64(cnt * nsb) * 4l + let y_bytes = int64(cnt * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let xqd = make_device_buf(xq_bytes) + let axsd = make_device_buf(axs_bytes) + let yd = make_device_buf(y_bytes) + let flop = 2.0lf * double(cnt) * double(d) * double(n) + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + upload_region_at(xqd, 0l, addr(xqh[0]), xq_bytes) + upload_region_at(axsd, 0l, addr(axsh[0]), axs_bytes) + let yd2 = make_device_buf(y_bytes) //! consecutive dispatches alternate outputs, as the served graph overlaps them + let vnames = fixed_array("nil ", "ship ", "inline", "khr ", "kq ", "nob ", "k64 ", "slab ") + let compared = fixed_array(false, true, true, false, false, false, true, true) //! khr is the reference; nil stages constants, nob no B, and kq reads its own Q8 activation fixture: timing-only + var raws : array + raws |> reserve(KHR_ARMS) + for (v in range(KHR_ARMS)) { + let tile = v == 4 ? 32 : 128 + let wgs = ((cnt + tile - 1) / tile) * ((d + tile - 1) / tile) + var sched : array + sched |> resize(4 + wgs) + sched[2] = uint(cnt) + let sc_bytes = int64(4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + delete sched + var sa = khr_arm_set(v, fmt, wqd, wsd, scd, xfd, xqd, axsd, yd, wq_bytes, ws_bytes, sc_bytes, xf_bytes, xq_bytes, axs_bytes, y_bytes) + var sb = khr_arm_set(v, fmt, wqd, wsd, scd, xfd, xqd, axsd, yd2, wq_bytes, ws_bytes, sc_bytes, xf_bytes, xq_bytes, axs_bytes, y_bytes) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = 4u) + for (k in range(DISPATCHES)) { + var hz : VkHaz //! fresh per dispatch: no barrier between them, the served graph's shape + if (k % 2 == 0) { + khr_arm_enc(v, fmt, raw, hz, sa, pc, int64(wgs)) + } else { + khr_arm_enc(v, fmt, raw, hz, sb, pc, int64(wgs)) + } + } + vk_check(vkEndCommandBuffer(raw), null) + raws |> push(raw) + } + var best <- sweep_best_us(raws) + let last_out = DISPATCHES % 2 == 0 ? yd2 : yd + var y_ref : array + read_back_y(raws[3], last_out, y_bytes, y_ref) + var y_arm : array + for (v in range(KHR_ARMS)) { + let tf = flop / (best[v] * 1000000.0lf) + var tail = "timing-only" + if (compared[v]) { + read_back_y(raws[v], last_out, y_bytes, y_arm) + tail = "max rel diff {max_rel_diff(y_arm, y_ref):.5f} vs khr" + } + print("{name} {vnames[v]}: {best[v] / 1000.0lf} ms/dispatch {tf} TFLOP/s {tail}\n") + } + delete y_ref + delete y_arm + delete best + delete raws + } + delete wqh + delete wsuh + delete xfh + delete xqh + delete axsh +} + +[vk_dispatch(name = "mmqpx_k4", grid = "wgs", params = "wgs : int64")] +class MmqPxK4 : MoeCmBase { + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @readonly wsu : array + @ssbo @binding = 3 @readonly xqw : array + @ssbo @binding = 4 @readonly axs : array + @ssbo @binding = 5 y : array + @workgroup mx4 : uint4[1088] //! 64 token rows x 16 uint4 chunks, stride 17 + @workgroup mw4 : uint4[544] //! 32 columns x 16 chunks, stride 17 (the sdot4 tile's staging) + @workgroup mxs : float[64] + @workgroup w_dsc : float[256] + @workgroup w_dmin : float[256] + + //! KqBatchK4.stage_w verbatim: chunk k of column wcol's superblock s, the nibble split at k's half + def stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let rb4 = wsb * 32u + bu * 4u + let r0 = wq[rb4] + let r1 = wq[rb4 + 1u] + let r2 = wq[rb4 + 2u] + let r3 = wq[rb4 + 3u] + if (hh == 0u) { + wv = uint4(r0 & 0x0F0F0F0F, r1 & 0x0F0F0F0F, r2 & 0x0F0F0F0F, r3 & 0x0F0F0F0F) + } else { + wv = uint4((r0 >> 4u) & 0x0F0F0F0F, (r1 >> 4u) & 0x0F0F0F0F, + (r2 >> 4u) & 0x0F0F0F0F, (r3 >> 4u) & 0x0F0F0F0F) + } + } + mw4[grp * 17u + k] = wv + } + } + + //! KqBatchBase.load_dm verbatim: the d/min pair's planes for (block sblk, column slane) + def stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + var sbv = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let dm = unpackHalf2x16(wsu[wsb * 5u]) + let sc = int((wsu[wsb * 5u + 1u + sblk / 4u] >> ((sblk % 4u) * 8u)) & 0xFF) + let mn = int((wsu[wsb * 5u + 3u + sblk / 4u] >> ((sblk % 4u) * 8u)) & 0xFF) + sa = dm.x * float(sc) + sbv = dm.y * float(mn) + } + w_dsc[sblk * 32u + slane] = sa + w_dmin[sblk * 32u + slane] = sbv + } + + [spirv_kernel(local_size_x = 64, name = "mmqpx_k4_spv")] + def run { + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wsb0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let wtiles = (pa.d + 31u) / 32u + let xt = tix / wtiles + let wt = tix % wtiles + let nsb = pa.n / 256u + let tid = gl_LocalInvocationID.x + let tg = tid / 4u + let cg = tid % 4u + var acc : float[32] + for [unroll] (i in range(32)) { + acc[i] = 0.0 + } + var s = 0u + while (s < nsb) { + let xrow = xt * 64u + tid + for (ki in range(16)) { + let k = uint(ki) + var xv = uint4(0u) + if (xrow < cnt) { + let xb4 = ((row0 + xrow) * nsb + s) * 64u + k * 4u + xv = uint4(xqw[xb4], xqw[xb4 + 1u], xqw[xb4 + 2u], xqw[xb4 + 3u]) + } + mx4[tid * 17u + k] = xv + } + mxs[tid] = xrow < cnt ? axs[(row0 + xrow) * nsb + s] : 0.0 + for (i in range(4)) { + let idx = tid + 64u * uint(i) + stage_w(wsb0, wt * 32u + idx / 8u, nsb, s, idx % 8u, idx / 8u) + stage_ws(wsb0, wt, nsb, s, idx) + } + barrier() + for (bu in range(8)) { + let k0 = uint(bu) * 2u + var xw : uint4[8] + var bs : int[4] + for [unroll] (t in range(4)) { + let xr = (tg * 4u + uint(t)) * 17u + k0 + xw[t * 2] = mx4[xr] + xw[t * 2 + 1] = mx4[xr + 1u] + bs[t] = (sdot4(xw[t * 2].x, 0x01010101) + sdot4(xw[t * 2].y, 0x01010101) + sdot4(xw[t * 2].z, 0x01010101) + sdot4(xw[t * 2].w, 0x01010101) + + sdot4(xw[t * 2 + 1].x, 0x01010101) + sdot4(xw[t * 2 + 1].y, 0x01010101) + sdot4(xw[t * 2 + 1].z, 0x01010101) + sdot4(xw[t * 2 + 1].w, 0x01010101)) + } + for [unroll] (c in range(8)) { + let col = cg * 8u + uint(c) + let w0 = mw4[col * 17u + k0] + let w1 = mw4[col * 17u + k0 + 1u] + let dsc = w_dsc[uint(bu) * 32u + col] + let dmin = w_dmin[uint(bu) * 32u + col] + for [unroll] (t in range(4)) { + let idot = (sdot4(w0.x, xw[t * 2].x) + sdot4(w0.y, xw[t * 2].y) + sdot4(w0.z, xw[t * 2].z) + sdot4(w0.w, xw[t * 2].w) + + sdot4(w1.x, xw[t * 2 + 1].x) + sdot4(w1.y, xw[t * 2 + 1].y) + sdot4(w1.z, xw[t * 2 + 1].z) + sdot4(w1.w, xw[t * 2 + 1].w)) + acc[t * 8 + c] += mxs[tg * 4u + uint(t)] * (dsc * float(idot) - dmin * float(bs[t])) + } + } + } + barrier() + s++ + } + for [unroll] (t in range(4)) { + let xrow = xt * 64u + tg * 4u + uint(t) + for [unroll] (c in range(8)) { + let col = wt * 32u + cg * 8u + uint(c) + if (xrow < cnt && col < pa.d) { + y[(row0 + xrow) * pa.d + col] = acc[t * 8 + c] + } + } + } + } +} + +[vk_dispatch(name = "mmqpx_k4b", grid = "wgs", params = "wgs : int64")] +class MmqPxK4Blk : MoeCmBase { + @ssbo @binding = 0 @readonly wq : array + @ssbo @binding = 1 @readonly wsu : array + @ssbo @binding = 3 @readonly xqw : array + @ssbo @binding = 4 @readonly axs : array + @ssbo @binding = 5 y : array + @workgroup bx4 : uint4[128] //! 64 token rows x one block (2 uint4 chunks) + @workgroup bw4 : uint4[256] //! 128 columns x one block + @workgroup bxs : float[64] + @workgroup w_dsc : float[128] + @workgroup w_dmin : float[128] + + [spirv_kernel(local_size_x = 128, name = "mmqpx_k4b_spv")] + def run { // nolint:STYLE038 — one measurement body + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let wsb0 = sched[rb] + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let wtiles = (pa.d + 127u) / 128u + let xt = tix / wtiles + let wt = tix % wtiles + let nsb = pa.n / 256u + let tid = gl_LocalInvocationID.x + let tg = tid / 8u + let cg = tid % 8u + var acc : float[64] + for [unroll] (i in range(64)) { + acc[i] = 0.0 + } + var s = 0u + while (s < nsb) { + if (tid < 64u) { + let xrow = xt * 64u + tid + bxs[tid] = xrow < cnt ? axs[(row0 + xrow) * nsb + s] : 0.0 + } + for (bu in range(8)) { + let k0 = uint(bu) * 2u + let xrow = xt * 64u + tid / 2u + var xv = uint4(0u) + if (xrow < cnt) { + let xb4 = ((row0 + xrow) * nsb + s) * 64u + (k0 + tid % 2u) * 4u + xv = uint4(xqw[xb4], xqw[xb4 + 1u], xqw[xb4 + 2u], xqw[xb4 + 3u]) + } + bx4[tid] = xv + let wcol = wt * 128u + tid + var w0 = uint4(0u) + var w1 = uint4(0u) + var sa = 0.0 + var sbv = 0.0 + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let rb4 = wsb * 32u + uint(bu) * 4u + let r0 = wq[rb4] + let r1 = wq[rb4 + 1u] + let r2 = wq[rb4 + 2u] + let r3 = wq[rb4 + 3u] + w0 = uint4(r0 & 0x0F0F0F0F, r1 & 0x0F0F0F0F, r2 & 0x0F0F0F0F, r3 & 0x0F0F0F0F) + w1 = uint4((r0 >> 4u) & 0x0F0F0F0F, (r1 >> 4u) & 0x0F0F0F0F, (r2 >> 4u) & 0x0F0F0F0F, (r3 >> 4u) & 0x0F0F0F0F) + let dm = unpackHalf2x16(wsu[wsb * 5u]) + let sc = int((wsu[wsb * 5u + 1u + uint(bu) / 4u] >> ((uint(bu) % 4u) * 8u)) & 0xFF) + let mn = int((wsu[wsb * 5u + 3u + uint(bu) / 4u] >> ((uint(bu) % 4u) * 8u)) & 0xFF) + sa = dm.x * float(sc) + sbv = dm.y * float(mn) + } + bw4[tid * 2u] = w0 + bw4[tid * 2u + 1u] = w1 + w_dsc[tid] = sa + w_dmin[tid] = sbv + barrier() + var xw : uint4[8] + var bs : int[4] + for [unroll] (t in range(4)) { + let xr = (tg * 4u + uint(t)) * 2u + xw[t * 2] = bx4[xr] + xw[t * 2 + 1] = bx4[xr + 1u] + bs[t] = (sdot4(xw[t * 2].x, 0x01010101) + sdot4(xw[t * 2].y, 0x01010101) + sdot4(xw[t * 2].z, 0x01010101) + sdot4(xw[t * 2].w, 0x01010101) + + sdot4(xw[t * 2 + 1].x, 0x01010101) + sdot4(xw[t * 2 + 1].y, 0x01010101) + sdot4(xw[t * 2 + 1].z, 0x01010101) + sdot4(xw[t * 2 + 1].w, 0x01010101)) + } + for [unroll] (c in range(16)) { + let col = cg * 16u + uint(c) + let wa = bw4[col * 2u] + let wb = bw4[col * 2u + 1u] + let dsc = w_dsc[col] + let dmin = w_dmin[col] + for [unroll] (t in range(4)) { + let idot = (sdot4(wa.x, xw[t * 2].x) + sdot4(wa.y, xw[t * 2].y) + sdot4(wa.z, xw[t * 2].z) + sdot4(wa.w, xw[t * 2].w) + + sdot4(wb.x, xw[t * 2 + 1].x) + sdot4(wb.y, xw[t * 2 + 1].y) + sdot4(wb.z, xw[t * 2 + 1].z) + sdot4(wb.w, xw[t * 2 + 1].w)) + acc[t * 16 + c] += bxs[tg * 4u + uint(t)] * (dsc * float(idot) - dmin * float(bs[t])) + } + } + barrier() + } + s++ + } + for [unroll] (t in range(4)) { + let xrow = xt * 64u + tg * 4u + uint(t) + for [unroll] (c in range(16)) { + let col = wt * 128u + cg * 16u + uint(c) + if (xrow < cnt && col < pa.d) { + y[(row0 + xrow) * pa.d + col] = acc[t * 16 + c] + } + } + } + } +} + +//! eight sdot4 over one 32-quant block: a column's two staged chunks against a row's two +def dot8(wa, wb, xa, xb : uint4) : int { + return (sdot4(wa.x, xa.x) + sdot4(wa.y, xa.y) + sdot4(wa.z, xa.z) + sdot4(wa.w, xa.w) + + sdot4(wb.x, xb.x) + sdot4(wb.y, xb.y) + sdot4(wb.z, xb.z) + sdot4(wb.w, xb.w)) +} + +def bsum8(xa, xb : uint4) : int { + return (sdot4(xa.x, 0x01010101) + sdot4(xa.y, 0x01010101) + sdot4(xa.z, 0x01010101) + sdot4(xa.w, 0x01010101) + + sdot4(xb.x, 0x01010101) + sdot4(xb.y, 0x01010101) + sdot4(xb.z, 0x01010101) + sdot4(xb.w, 0x01010101)) +} + +[ |> template_struct_instance] +class template MmqPxK4CT { //! standalone: a harness template cannot derive from the module's plain base + @ssbo @binding = 0 @role = "alias" @readonly wq : array + @ssbo @binding = 1 @role = "alias" @readonly wsu : array + @ssbo @binding = 2 sched : array + @ssbo @binding = 3 @role = "alias" @readonly xqw : array + @ssbo @binding = 4 @role = "alias" @readonly axs : array + @ssbo @binding = 5 y : array + @push_constant pa : BatchArgs + @workgroup bx4 : uint4[128] //! 64 token rows x one block (2 uint4 chunks) + @workgroup bw4 : uint4[128] //! 64 columns x one block + @workgroup bxs : float[64] + @workgroup w_dsc : float[64] + @workgroup w_dmin : float[64] + @template_constant NOSTAGE : bool = false //! the compute ceiling: constants in place of the plane reads + + [spirv_kernel(local_size_x = 256)] + def run { // nolint:STYLE038 — one measurement body, the register block written out + let rb = sched[pa.map_off + gl_WorkGroupID.x] * 4u + let row0 = sched[rb + 1u] + let cnt = sched[rb + 2u] + let tix = gl_WorkGroupID.x - sched[rb + 3u] + let wtiles = (pa.d + 63u) / 64u + let xt = tix / wtiles + let wt = tix % wtiles + let nsb = pa.n / 256u + let tid = gl_LocalInvocationID.x + let tg = tid / 16u + let cg = tid % 16u + var a00 = 0.0 + var a01 = 0.0 + var a02 = 0.0 + var a03 = 0.0 + var a10 = 0.0 + var a11 = 0.0 + var a12 = 0.0 + var a13 = 0.0 + var a20 = 0.0 + var a21 = 0.0 + var a22 = 0.0 + var a23 = 0.0 + var a30 = 0.0 + var a31 = 0.0 + var a32 = 0.0 + var a33 = 0.0 + var s = 0u + while (s < nsb) { + if (tid < 64u) { + static_if (NOSTAGE) { + bxs[tid] = 0.0625 + } else { + let xrow = xt * 64u + tid + bxs[tid] = xrow < cnt ? axs[(row0 + xrow) * nsb + s] : 0.0 + } + } + for (bu in range(8)) { + if (tid < 128u) { + var xv = uint4(0u) + static_if (NOSTAGE) { + xv = uint4(0x01020304u + s, 0x05060708u, 0x090A0B0Cu, 0x0D0E0F10u + uint(bu)) + } else { + let xrow = xt * 64u + tid / 2u + if (xrow < cnt) { + let xb4 = ((row0 + xrow) * nsb + s) * 64u + (uint(bu) * 2u + tid % 2u) * 4u + xv = uint4(xqw[xb4], xqw[xb4 + 1u], xqw[xb4 + 2u], xqw[xb4 + 3u]) + } + } + bx4[tid] = xv + } else { + let ct = tid - 128u + var wv = uint4(0u) + var sa = 0.0 + var sbv = 0.0 + static_if (NOSTAGE) { + wv = uint4(0x01020304u + uint(bu), 0x05060708u, 0x090A0B0Cu + s, 0x0D0E0F10u) + sa = 0.001 + sbv = 0.0001 + } else { + let wcol = wt * 64u + ct / 2u + if (wcol < pa.d) { + let wsb = sched[rb] + wcol * nsb + s + let rb4 = wsb * 32u + uint(bu) * 4u + let sh = (ct % 2u) * 4u + wv = uint4((wq[rb4] >> sh) & 0x0F0F0F0F, (wq[rb4 + 1u] >> sh) & 0x0F0F0F0F, (wq[rb4 + 2u] >> sh) & 0x0F0F0F0F, (wq[rb4 + 3u] >> sh) & 0x0F0F0F0F) + let dm = unpackHalf2x16(wsu[wsb * 5u]) + let sc = int((wsu[wsb * 5u + 1u + uint(bu) / 4u] >> ((uint(bu) % 4u) * 8u)) & 0xFF) + let mn = int((wsu[wsb * 5u + 3u + uint(bu) / 4u] >> ((uint(bu) % 4u) * 8u)) & 0xFF) + sa = dm.x * float(sc) + sbv = dm.y * float(mn) + } + } + bw4[ct] = wv + if (ct % 2u == 0u) { + w_dsc[ct / 2u] = sa + w_dmin[ct / 2u] = sbv + } + } + barrier() + let xr = tg * 8u + let x0a = bx4[xr] + let x0b = bx4[xr + 1u] + let x1a = bx4[xr + 2u] + let x1b = bx4[xr + 3u] + let x2a = bx4[xr + 4u] + let x2b = bx4[xr + 5u] + let x3a = bx4[xr + 6u] + let x3b = bx4[xr + 7u] + let b0 = float(bsum8(x0a, x0b)) + let b1 = float(bsum8(x1a, x1b)) + let b2 = float(bsum8(x2a, x2b)) + let b3 = float(bsum8(x3a, x3b)) + let xs0 = bxs[tg * 4u] + let xs1 = bxs[tg * 4u + 1u] + let xs2 = bxs[tg * 4u + 2u] + let xs3 = bxs[tg * 4u + 3u] + let cr = cg * 8u + let w0a = bw4[cr] + let w0b = bw4[cr + 1u] + let w1a = bw4[cr + 2u] + let w1b = bw4[cr + 3u] + let w2a = bw4[cr + 4u] + let w2b = bw4[cr + 5u] + let w3a = bw4[cr + 6u] + let w3b = bw4[cr + 7u] + let c0 = cg * 4u + let ws0 = w_dsc[c0] + let ws1 = w_dsc[c0 + 1u] + let ws2 = w_dsc[c0 + 2u] + let ws3 = w_dsc[c0 + 3u] + let wm0 = w_dmin[c0] + let wm1 = w_dmin[c0 + 1u] + let wm2 = w_dmin[c0 + 2u] + let wm3 = w_dmin[c0 + 3u] + a00 += xs0 * (ws0 * float(dot8(w0a, w0b, x0a, x0b)) - wm0 * b0) + a01 += xs0 * (ws1 * float(dot8(w1a, w1b, x0a, x0b)) - wm1 * b0) + a02 += xs0 * (ws2 * float(dot8(w2a, w2b, x0a, x0b)) - wm2 * b0) + a03 += xs0 * (ws3 * float(dot8(w3a, w3b, x0a, x0b)) - wm3 * b0) + a10 += xs1 * (ws0 * float(dot8(w0a, w0b, x1a, x1b)) - wm0 * b1) + a11 += xs1 * (ws1 * float(dot8(w1a, w1b, x1a, x1b)) - wm1 * b1) + a12 += xs1 * (ws2 * float(dot8(w2a, w2b, x1a, x1b)) - wm2 * b1) + a13 += xs1 * (ws3 * float(dot8(w3a, w3b, x1a, x1b)) - wm3 * b1) + a20 += xs2 * (ws0 * float(dot8(w0a, w0b, x2a, x2b)) - wm0 * b2) + a21 += xs2 * (ws1 * float(dot8(w1a, w1b, x2a, x2b)) - wm1 * b2) + a22 += xs2 * (ws2 * float(dot8(w2a, w2b, x2a, x2b)) - wm2 * b2) + a23 += xs2 * (ws3 * float(dot8(w3a, w3b, x2a, x2b)) - wm3 * b2) + a30 += xs3 * (ws0 * float(dot8(w0a, w0b, x3a, x3b)) - wm0 * b3) + a31 += xs3 * (ws1 * float(dot8(w1a, w1b, x3a, x3b)) - wm1 * b3) + a32 += xs3 * (ws2 * float(dot8(w2a, w2b, x3a, x3b)) - wm2 * b3) + a33 += xs3 * (ws3 * float(dot8(w3a, w3b, x3a, x3b)) - wm3 * b3) + barrier() + } + s++ + } + let r0 = xt * 64u + tg * 4u + let cb = wt * 64u + cg * 4u + if (r0 < cnt && cb + 3u < pa.d) { //! the probe shapes are whole tiles + let yb = (row0 + r0) * pa.d + cb + y[yb] = a00 + y[yb + 1u] = a01 + y[yb + 2u] = a02 + y[yb + 3u] = a03 + y[yb + pa.d] = a10 + y[yb + pa.d + 1u] = a11 + y[yb + pa.d + 2u] = a12 + y[yb + pa.d + 3u] = a13 + y[yb + 2u * pa.d] = a20 + y[yb + 2u * pa.d + 1u] = a21 + y[yb + 2u * pa.d + 2u] = a22 + y[yb + 2u * pa.d + 3u] = a23 + y[yb + 3u * pa.d] = a30 + y[yb + 3u * pa.d + 1u] = a31 + y[yb + 3u * pa.d + 2u] = a32 + y[yb + 3u * pa.d + 3u] = a33 + } + } +} + +[vk_dispatch(name = "mmqpx_k4c", kernel = "run", grid = "wgs", params = "wgs : int64")] +class MmqPxK4Staged : MmqPxK4CT { +} + +[vk_dispatch(name = "mmqpx_k4d", kernel = "run", grid = "wgs", params = "wgs : int64")] +class MmqPxK4Ceil : MmqPxK4CT { + override NOSTAGE = true +} + +def private mmq_arm_set(v, fmt : int; bufs : uint64 const[6]; sizes : int64 const[6]; gbits : uint const[6]) : VkDescriptorSet { + if (v == 0) { + return kq_batch_cls_set_for(fmt, false, bufs, sizes, gbits) + } elif (v == 1) { + return set_mmqpx_k4(bufs, sizes, gbits) + } elif (v == 2) { + return set_mmqpx_k4b(bufs, sizes, gbits) + } elif (v == 3) { + return set_mmqpx_k4c(bufs, sizes, gbits) + } + return set_mmqpx_k4d(bufs, sizes, gbits) +} + +def private mmq_arm_enc(v, fmt : int; raw : VkCommandBuffer; var hz : VkHaz; var s : VkDescriptorSet; var pc : BatchArgs; wgs : int64) { + if (v == 0) { + kq_batch_cls_enc_for(fmt, false, raw, hz, s, pc, wgs) + } elif (v == 1) { + enc_mmqpx_k4(raw, hz, s, pc, wgs) + } elif (v == 2) { + enc_mmqpx_k4b(raw, hz, s, pc, wgs) + } elif (v == 3) { + enc_mmqpx_k4c(raw, hz, s, pc, wgs) + } else { + enc_mmqpx_k4d(raw, hz, s, pc, wgs) + } +} + +[arch(at="../ARCHITECTURE_MEASUREMENT.md#one-benchmark-rig")] +def private run_mmq_shape(name : string; d, n, cnt : int) { // nolint:STYLE038 — one linear measurement sweep + let fmt = int(KqFmt.k4) + let nsb = n / 256 + let totsb = d * nsb + let bb = arena_block_bytes(fmt) + let wq_words = int(bb.wq / 4l) + let ws_words = int(bb.ws / 4l) + var wqh : array + var wsuh : array + var xqh : array + var axsh : array + wqh |> resize(totsb * wq_words) + wsuh |> resize(totsb * ws_words) + xqh |> resize(cnt * nsb * 64) + axsh |> resize(cnt * nsb) + for (i in range(totsb * wq_words)) { + wqh[i] = hash_word(uint(i) + 29u) + } + for (sb in range(totsb)) { + wsuh[sb * ws_words] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.00005 * float(1 + sb % 5))) + for (wi in range(1, ws_words)) { + wsuh[sb * ws_words + wi] = hash_word(uint(sb * ws_words + wi) + 613u) + } + } + for (i in range(cnt * nsb * 64)) { + xqh[i] = hash_word(uint(i) * 13u + 1u) + } + for (i in range(cnt * nsb)) { + axsh[i] = 0.0625 + } + let wq_bytes = int64(totsb) * bb.wq + let ws_bytes = long_length(wsuh) * 4l + let xq_bytes = long_length(xqh) * 4l + let axs_bytes = int64(cnt * nsb) * 4l + let y_bytes = int64(cnt * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xqd = make_device_buf(xq_bytes) + let axsd = make_device_buf(axs_bytes) + let yd = make_device_buf(y_bytes) + let flop = 2.0lf * double(cnt) * double(d) * double(n) + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xqd, 0l, addr(xqh[0]), xq_bytes) + upload_region_at(axsd, 0l, addr(axsh[0]), axs_bytes) + let yd2 = make_device_buf(y_bytes) //! consecutive dispatches alternate outputs, as the served graph overlaps them + let vnames = fixed_array("kq ", "mmq ", "mmqb ", "mmqc ", "mmqd ") + let compared = fixed_array(false, true, true, true, false) //! kq is the reference; the ceiling twin reads constants: timing-only + var raws : array + raws |> reserve(MMQ_ARMS) + for (v in range(MMQ_ARMS)) { + let ttile = v == 0 ? 32 : 64 + let wtile = v == 2 ? 128 : (v >= 3 ? 64 : 32) + let wgs = ((cnt + ttile - 1) / ttile) * ((d + wtile - 1) / wtile) + var sched : array + sched |> resize(4 + wgs) + sched[2] = uint(cnt) + let sc_bytes = int64(4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + delete sched + let gbits = fixed_array(1u, 2u, 0u, 4u, 8u, 16u) + let asizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xq_bytes, axs_bytes, y_bytes) + var sa = mmq_arm_set(v, fmt, fixed_array(wqd, wsd, scd, xqd, axsd, yd), asizes, gbits) + var sb = mmq_arm_set(v, fmt, fixed_array(wqd, wsd, scd, xqd, axsd, yd2), asizes, gbits) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = 4u) + for (k in range(DISPATCHES)) { + var hz : VkHaz //! fresh per dispatch: no barrier between them, the served graph's shape + if (k % 2 == 0) { + mmq_arm_enc(v, fmt, raw, hz, sa, pc, int64(wgs)) + } else { + mmq_arm_enc(v, fmt, raw, hz, sb, pc, int64(wgs)) + } + } + vk_check(vkEndCommandBuffer(raw), null) + raws |> push(raw) + } + var best <- sweep_best_us(raws) + let last_out = DISPATCHES % 2 == 0 ? yd2 : yd + var y_ref : array + read_back_y(raws[0], last_out, y_bytes, y_ref) + var y_arm : array + for (v in range(MMQ_ARMS)) { + let tf = flop / (best[v] * 1000000.0lf) + var tail = "timing-only" + if (compared[v]) { + read_back_y(raws[v], last_out, y_bytes, y_arm) + tail = "max rel diff {max_rel_diff(y_arm, y_ref):.5f} vs kq" + } + print("{name} {vnames[v]}: {best[v] / 1000.0lf} ms/dispatch {tf} TFLOP/s {tail}\n") + } + delete y_ref + delete y_arm + delete best + delete raws + } + delete wqh + delete wsuh + delete xqh + delete axsh +} + // k6 (Q6_K) cm2 decode-in-load vs the serving kq tile - same instrument as run_k4_shape; // bisect adds the k6px decode-spelling variants [arch(at="../ARCHITECTURE_MEASUREMENT.md#one-benchmark-rig")] @@ -1371,7 +2552,7 @@ def private run_probe { // nolint:STYLE037,STYLE038 — the flat per-arg shape print("no Vulkan device\n") return } - if (g_gpu.coopmat_mode < 3) { + if (g_gpu.coopmat_mode < COOPMAT_MM) { print("coopmat mode {g_gpu.coopmat_mode} < 3 - mm tiles not routed on this device\n") return } @@ -1388,7 +2569,7 @@ def private run_probe { // nolint:STYLE037,STYLE038 — the flat per-arg shape var only = "" var cm2_fmt = -1 for (a in get_command_line_arguments()) { - if (a == "gate" || a == "down" || a == "q" || a == "kv" || a == "kvm" || a == "qkv" || a == "cm2x" || a == "ref" || a == "tl" || a == "k4" || a == "k6" || a == "k6x") { + if (a == "gate" || a == "down" || a == "q" || a == "kv" || a == "kvm" || a == "qkv" || a == "cm2x" || a == "ref" || a == "tl" || a == "k4" || a == "k6" || a == "k6x" || a == "khrx" || a == "mmqx") { only = a } elif (a |> starts_with("cm2:")) { only = "cm2:" @@ -1480,6 +2661,22 @@ def private run_probe { // nolint:STYLE037,STYLE038 — the flat per-arg shape run_shape("tl k/v ", 256, 2048, 512) return } + if (only == "mmqx") { + verify(ensure_mmqpx_k4() && ensure_mmqpx_k4b() && ensure_mmqpx_k4c() && ensure_mmqpx_k4d() && kq_batch_cls_ensure(int(KqFmt.k4), false), "integer probe pipelines must engage") + run_mmq_shape("q4k gate", 9728, 2560, 512) + run_mmq_shape("q4k down", 2560, 9728, 512) + run_mmq_shape("q4k q/wo", 4096, 2560, 512) + return + } + if (only == "khrx") { + verify(ensure_khrpx_nil() && ensure_khrpx_ship() && ensure_khrpx_inline() && ensure_kq_batch_k4_khr_cls() + && ensure_khrpx_nob() && ensure_khrpx_k64() && ensure_khrpx_slab() + && kq_batch_cls_ensure(int(KqFmt.k4), false), "KHR probe pipelines must engage") + run_khr_shape("q4k gate", 9728, 2560, 512) + run_khr_shape("q4k down", 2560, 9728, 512) + run_khr_shape("q4k q/wo", 4096, 2560, 512) + return + } if (only == "cm2x") { if (!g_gpu.has_coopmat2) { print("no coopmat2 on this device\n") diff --git a/modules/dasLLAMA/harness/vk_gemv_probe.das b/modules/dasLLAMA/harness/vk_gemv_probe.das new file mode 100644 index 0000000000..6a0b4f6b81 --- /dev/null +++ b/modules/dasLLAMA/harness/vk_gemv_probe.das @@ -0,0 +1,136 @@ +options gen2 +options stack = 524288 +options _dasllama_internal = true + +// ATTRIBUTION SWEEP - isolated per-format rates of the decode GEMV classes at one served shape: +// synthetic planes per kq format at [n x d], the shipped kq_gemv class dispatched over enough +// regions of d rows that one dispatch streams STREAM_MB or more (the decode reads a layer's +// planes once per token, so the rate wanted is the DRAM one, not an L2 replay of a small plane), +// timed over repeated submits with a dependency between dispatches. Prints the bytes a dispatch +// reads, its wall per dispatch and the GB/s - the number llama.cpp's per-type mat-vec rates +// (test-backend-ops perf -o MUL_MAT, the m=4096 n=1 k=14336 rows) are read against. +// Args: (default 4096 12288, the Qwen3.5-9B FFN gate; 5120 17408 is the 27B's). + +require strings +require vulkan +require vulkan/vulkan_boost +require dasllama/dasllama_vulkan_common +require dasllama/dasllama_vulkan_classes +require dasllama/dasllama_kqformat +require dasllama/dasllama_gemm_schema +require _vk_probe_fixture // hash_word, cool_f16_pair: the synthetic planes both probes build + +let private DISPATCHES = 20 +let private SUBMITS = 5 +let private STREAM_MB = 128 //! the least a dispatch streams: four times the 32 MB L2 of the reference card + +let private YBIT = 32u //! the y buffer's hazard bit in the binding order below + +//! one format at [n x d]: the shipped class over as many d-row region copies as one dispatch needs to stream STREAM_MB +def private probe_fmt(fmt : KqFmt; n, d : int) : tuple { + let nsb = n / 256 + let nwsb = d * nsb + let qsb_words = int(kq_qsb(fmt) / 4l) + let ssb_words = int(kq_dev_ssb(fmt) / 4l) + let plane_bytes = nwsb * (qsb_words + ssb_words) * 4 //! under 2^31 for every served shape (the 27B's largest plane is 89 MB) + let copies = (STREAM_MB * 1048576 + plane_bytes - 1) / plane_bytes + let wq_words = nwsb * qsb_words * copies + let ws_words = nwsb * ssb_words * copies + var wqh : array + var wsh : array + var xqh : array + var xsh : array + wqh |> reserve(wq_words) + wqh |> resize(wq_words) + wsh |> reserve(ws_words) + wsh |> resize(ws_words) + xqh |> resize(nsb * 64) + xsh |> resize(nsb) + for (i in range(wq_words)) { + wqh[i] = hash_word(uint(i) + uint(int(fmt)) * 77777u) + } + for (i in range(ws_words)) { + wsh[i] = cool_f16_pair(uint(i) + uint(int(fmt)) * 13u) + } + for (i in range(nsb * 64)) { + xqh[i] = hash_word(uint(i) * 31u + 5u) + } + for (i in range(nsb)) { + xsh[i] = 0.5 + float(i % 7) * 0.25 + } + let wq_bytes = int64(wq_words) * 4l + let ws_bytes = int64(ws_words) * 4l + let xq_bytes = int64(nsb) * 256l + let xs_bytes = int64(nsb) * 4l + let y_bytes = int64(d) * 4l * int64(copies) + let rg_bytes = 8l * int64(copies) + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xqd = make_device_buf(xq_bytes) + let xsd = make_device_buf(xs_bytes) + let yd = make_device_buf(y_bytes) + let rgd = make_device_buf(rg_bytes) + var regs <- [for (i in range(copies * 2)); (i % 2 == 0) ? uint(i / 2 * nwsb) : 0u] //! region r: its own plane copy, the one x + let wgs = (int64(d) * int64(copies) + g_gpu.rows_per_wg - 1l) / g_gpu.rows_per_wg + var raw = alloc_cmd() + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsh[0]), ws_bytes) + upload_region_at(xqd, 0l, addr(xqh[0]), xq_bytes) + upload_region_at(xsd, 0l, addr(xsh[0]), xs_bytes) + upload_region_at(rgd, 0l, addr(regs[0]), rg_bytes) + let bufs = fixed_array(wqd, wsd, rgd, xqd, xsd, yd) + let sizes = fixed_array(wq_bytes, ws_bytes, rg_bytes, xq_bytes, xs_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 4u, 8u, 16u, YBIT) + var s_cls = gemv_cls_set(int(fmt), bufs, sizes, gbits) + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var h : VkHaz + var pc = GemvArgs(n = uint(n), d = uint(d), nreg = uint(copies), ybase = 0u) + for (_i in range(DISPATCHES)) { + gemv_cls_enc(int(fmt), raw, h, s_cls, pc, wgs) + vhz_dep(raw, h, YBIT, 0u, true) //! the next dispatch waits on this one's y + } + vk_check(vkEndCommandBuffer(raw), null) + } + submit_wait(raw) //! the warm-up: pipeline compile, first-touch + let t0 = ref_time_ticks() + for (_i in range(SUBMITS)) { + submit_wait(raw) + } + let us = double(get_time_usec(t0)) / double(SUBMITS * DISPATCHES) + delete wqh + delete wsh + delete xqh + delete xsh + return (mb = double(wq_bytes + ws_bytes) / 1048576.0lf, us = us) +} + +[export] +def main : int { + let args <- get_command_line_arguments() + var n = 4096 + var d = 12288 + let shape_args <- [for (a in args); a; where to_int(a) > 0] + if (length(shape_args) >= 2) { + n = to_int(shape_args[length(shape_args) - 2]) + d = to_int(shape_args[length(shape_args) - 1]) + } + if (!ensure_q8_batch_cls()) { + print("vk_gemv_probe: no Vulkan device\n") + return 1 + } + let fmts = fixed_array(KqFmt.k4, KqFmt.q40, KqFmt.k5, KqFmt.k6, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2, KqFmt.iq2s, KqFmt.iq2xs, KqFmt.iq2xxs) + for (f in fmts) { + verify(gemv_cls_ensure(int(f)), "vk_gemv_probe: every kq gemv class rail must engage on a live device") + } + print("vk_gemv_probe: decode GEMV classes at [n {n} x d {d}], {g_gpu.rows_per_wg} rows/wg, plane copies to {STREAM_MB} MB per dispatch, {DISPATCHES} dispatches x {SUBMITS} submits after a warm-up\n") + print("| fmt | MB/dispatch | us/dispatch | GB/s | |\n") + print("| :----- | ------------: | -----------: | ------: | :---------- |\n") + for (f in fmts) { + let r = probe_fmt(f, n, d) + let gbs = r.mb * 1048576.0lf / (r.us * 1000.0lf) + print("| {f} | {r.mb:13.1f} | {r.us:12.1f} | {gbs:7.1f} | timing-only |\n") + } + return 0 +} diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index f348641d75..4b7f8f50d0 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -9,15 +9,15 @@ validate through `../dasllama/dasllama_exchange_schema.das` instead.** The engin `dasllama/` require beyond the lint macro module) is `REVIEW.das`'s to enforce; weakening that gate is a defect. -**Weakening `REVIEW.das`'s record-hygiene checks is a defect; the fix for a red is a re-mint -on a quiet, session-free box, never an edit.** What each check enforces is read from the gate -itself; each check's finding text states its own rule. +**Weakening any check in `REVIEW.das` - the conditions it fires on - is a defect; the fix for a +red is a re-mint on a quiet, session-free box, never an edit.** **Narrowing the scope of any `REVIEW.das` check - the files it walks, the names it does not flag - is a defect unless `../ARCHITECTURE_ENGINE.md` or `../ARCHITECTURE_MEASUREMENT.md` -ledgers the excluded scope in the same change.** The single-exchange-client check walks the -engine (`../dasllama/`); a measurement harness talking HTTP to a reference server is the -ledgered exclusion. +ledgers the excluded scope in the same change.** What each check enforces is +read from the gate itself; each check's finding text states its own rule. The +single-exchange-client check walks the engine (`../dasllama/`); a measurement harness talking +HTTP to a reference server is the ledgered exclusion. **A diff that writes a commit stamp anywhere under this folder naming a commit the branch under review cannot reach is a defect - re-mint, or re-stamp to a reachable commit whose @@ -39,8 +39,8 @@ checkout the record's provenance spells, the python legs by `../benchmarks/asr/requirements-*.txt`. **A diff that writes a records row, sidecar archive, or `defaults/` profile under this -folder whose `provenance.dasllama_version` differs from, or is absent where, the -`DASLLAMA_RELEASE` string (`../dasllama/dasllama_version.das`) is a defect - re-mint.** For a +folder whose `provenance.dasllama_version` differs from, or is absent where, +`DASLLAMA_RELEASE` (`../dasllama/dasllama_version.das`) is a defect - re-mint.** For a sidecar with an `engine_sha`, read the value at that commit; a `defaults/` profile compares against the branch under review. A ruler record pins its engines through `meta.das_sha`, and through `meta.lcpp_version` when a reference arm ran (`-` when none did). @@ -77,11 +77,14 @@ second tool's record carrying the wrong engine, and looks real. **Outside `model_specs()` (text, in `model_specs.das`) and `asr_catalog()` (audio, in `profile_common.das`), a `.das` function under this folder that lists model files, quants, -board membership, provenance, or parity fixtures is a defect.** Board membership is which -models the site results board shows. Write a new list as a view over those two functions: it -recomputes from them on every call, selects rows by one field whose value on the row states -membership, never by matching a field against a list of literal values - file names, name -prefixes, and recipe constants alike - and stores no `url`/`bytes`/`sha256` of its own. +board membership, provenance, or parity fixtures is a defect - write it as a view over those +two functions.** Board membership is which models the site results board shows. + +**A view over `model_specs()` or `asr_catalog()` recomputes from them on every call, selects +rows by one field whose value on the row states membership, and stores no +`url`/`bytes`/`sha256` of its own; a view that matches a field against a list of literal +values - file names, name prefixes, recipe constants - is a defect.** A literal list is a +second catalog that drifts from the first. **A diff that makes a recorded row or manifest under this folder pin a model file keeps that file's provenance on its own row.** The row is a row of `model_specs()` (`model_specs.das`) or @@ -113,6 +116,11 @@ row's model file, ending `0 pending, 0 failed` with the row reported `ok`, plus `fetch_models.das --` run in which no row the diff touched is `pending`.** A box stocks only some of the rows, so an unscoped run's `pending` count is the box's. +**A diff that changes the `recipe` of a row carrying no `sha256` records, in the PR +description, the conversion command as run and the produced file's identity - its byte size, +or a `file_identity` hash.** `fetch_models` reports such a row `ok` on presence alone, so its +run cannot tell a re-mint from the stale file the old recipe made. + **A diff that changes `fetch_models.das` beyond its comments records its settling evidence in the PR description: one unscoped `fetch_models.das --` run ending `0 failed`.** diff --git a/modules/dasLLAMA/performance/model_specs.das b/modules/dasLLAMA/performance/model_specs.das index a6bfc9bad4..a55e9bf3c0 100644 --- a/modules/dasLLAMA/performance/model_specs.das +++ b/modules/dasLLAMA/performance/model_specs.das @@ -305,9 +305,9 @@ def model_specs() : array { // nolint:STYLE038 — flat model-set t parity <- ev_qwen_count([dn_chunked = true])), // the K-quant twin of the hybrid for the resident driver's native-plane cells // (tests/test_gpu_resident_hybrid.das): qkv q6_K, z q4_K, beta/alpha q4_K, the deltanet - // out plane pinned q8_0 like the 9B UD file's (the step's o row is Q8_0) + // out plane q4_K - the step leaves the o row f32 for the superblock out plane's requant ModelSpec(file = "Qwen3.5-0.8B-Q4_K_M.gguf", - recipe = "llama-quantize --allow-requantize --tensor-type ssm_out=q8_0 Qwen3.5-0.8B-Q8_0.gguf Qwen3.5-0.8B-Q4_K_M.gguf Q4_K_M"), + recipe = "llama-quantize --allow-requantize Qwen3.5-0.8B-Q8_0.gguf Qwen3.5-0.8B-Q4_K_M.gguf Q4_K_M"), // the 9B hybrid with its in-file MTP block, the whole-model driver's parity carrier // (head 256 gated attention, partial rope, Q5_K/Q6_K deltanet planes) and the vulkan // serving census's head-256 row; the local name carries the MTP tag the release's does not diff --git a/modules/dasLLAMA/tests/CLAUDE.md b/modules/dasLLAMA/tests/CLAUDE.md index e1bed15b07..1c42b85471 100644 --- a/modules/dasLLAMA/tests/CLAUDE.md +++ b/modules/dasLLAMA/tests/CLAUDE.md @@ -207,7 +207,11 @@ GPU-less boxes), and the flavor image round-trips the plan verbatim. The `coverage` suite (test_kernel_coverage, arm `coverage`; arm `coverage-vk` = the vulkan SERVING census - needs a vulkan device + `DASLLAMA_GPU=1` + `DASLLAMA_MODELS_DIR`, MoE rows -under `DASLLAMA_PARITY_FULL=1`) is the KERNEL COVERAGE census (the census-row obligation is +under `DASLLAMA_PARITY_FULL=1`; every prefill tile family is reached through the qwen3 Q8_0 and +Q4_K_M and the 1B llama requants, machine-local like the other fixtures - the `-local` ones are +minted from the bartowski Q8_0 with `llama-quantize --allow-requantize ` - each +swept under the coopmat modes its planes have twins in: all five for q8 and q40, the box's mode, +mm and sdot4 for the other kq formats) is the KERNEL COVERAGE census (the census-row obligation is `REVIEW.md`'s): the small-model zoo swept across format/graph/batch/KV axes, then a report of per-kernel dispatch counts with LOUD WARNINGS for compiled-but-never-dispatched kernels - never an auto-dead verdict. A zero means "nothing THIS zoo runs dispatched it", @@ -255,11 +259,21 @@ suite: the runner disarms the guard that tripwire asserts. The map below is part two lists together are the census. `test_vulkan_dec_tail.das` - model-free (a Vulkan device, else skips): the per-op tier's decode era against a CPU reference - the decode attention block (K-quant and q8 quads, both rope -pairings, the hydrate arms), the decode FFN tail, and the whole-token decode span with its +pairings, a q8 pair carrying the q/k/v projection bias, the hydrate arms), the decode FFN tail, and the whole-token decode span with its device router + top-k against `moe_select_core`, plus the `vulkan_moe_span` override reached through its registry. `test_vulkan_moe_cm2.das` - model-free (a cm2 device, else skips): the cm2 expert chain over a device-side f16 gather, the streamed-group slot hand-off, and the streamed split's async head. +`test_vulkan_kernels.das` - model-free (a Vulkan device, else skips): the per-class CPU-oracle +units of the Vulkan kernel census (`_vkd_oracles.das` runs the class methods on the CPU as the +oracle; `_vkd_toy.das` is the `[vk_dispatch]` bring-up fixture). The thirteen per-format tile +cells (`test_vkd__cm2_batch`) run four arms: the cm2 l/m/s tiles in mode 4 on an +NV_coopmat2 device and the KHR 128x128 tile wherever the device has KHR coopmat at subgroup +32 - the cell skips only when the device has neither, so a KHR-only card still runs its arm; +`test_vkd_direct_decode` +proves a `[spirv_decode]` method called from a kernel body on the plane element (the KHR arm's +staging form: the index travels, the callee chains through the plane) is an ordinary call on +the device, against the same method run on the CPU. `test_bench_records_schema.das` - model-free: the record store's schema (round-trip, upsert identity with `workload` in the key, annotations landing only on the rows they select, the store lister admitting `records/{box}.json` alone) and the record rig's shared seams (the @@ -312,7 +326,9 @@ deltanet decode step, and skips otherwise. `test_gpu_serving_declines.das` - model-free: the whole-model driver's decline reasons decided from a Config or a synthetic Model shell (`resident_unserved_features`, `attn_chain_unserved_features`, `resident_layer_decline`) - every unserved feature and layer -shape is named in the text a user reads, a served one yields "". +shape is named in the text a user reads, a served one yields ""; plus the KV mirror's binding cap +(`resident_binding_ctx`) on a hybrid shell whose layer 0 is recurrent, its dense twin, and a +shell with no attention layer. `test_gpu_resident_hybrid.das` - stocked suite; the whole-model resident driver on a deltanet hybrid (Qwen3.5-0.8B-Q8_0, `DASLLAMA_GPU=1`): the resident window chain prefills (recurrent layers through conv + chunked scan on device state, gated partial-rope attention over the @@ -325,11 +341,19 @@ prompt one window plus 88 rows long), one-past-the-window (one row past one wind shortens the preceding window so the last one still carries the conv taps) and two-token (a prompt shorter than the conv taps: the conv history ring's leading rows are zero) cells; skips without the model or the armed tier. The K-quant twin (`Qwen3.5-0.8B-Q4_K_M.gguf`, minted from -the Q8_0 by the recipe its `../performance/model_specs.das` row carries, which pins one -tensor - the deltanet out projection (`ssm_out`) - to Q8_0) runs the same one-window and -two-window cells with the deltanet qkv (q6_K) and z (q4_K) planes in their file formats on the -driver, asserts the loader kept them so, and holds a 6% bar (the K-quant chain's device-vs-CPU -noise runs near double the Q8 file's, flat across steps). +the Q8_0 by the recipe its `../performance/model_specs.das` row carries) runs the same one-window +and two-window cells with the deltanet qkv (q6_K), z (q4_K) and out (q4_K) planes in their file +formats on the driver, asserts the loader kept them so, and holds a 6% bar (the K-quant chain's device-vs-CPU +noise runs near double the Q8 file's, flat across steps). Both files make their sessions on the +mirror codec the box arms, so under `DASLLAMA_VK_KV32=1` the Q8 cells run on the f32 mirrors and +the K-quant cells skip (their bar is calibrated on the f16 mirror). Run under `DASLLAMA_COOPMAT=mm` the +same file is the KHR arm's end-to-end gate: the K-quant twin's planes then prefill on the KHR kq +tile (mode 3), and the bars hold there too. +`test_gpu_resident_qwen2.das` - stocked suite; the whole-model resident driver on a qwen2 +(Qwen2.5-0.5B-Instruct-Q8_0, `DASLLAMA_GPU=1`): the q/k/v projection bias folded into the rope +stage on the device - the hybrid file's forced-feed logits-tolerance form (its K-quant 6% bar, +the one-step-off control) at one window and two windows, with the arm witnesses that the model +carries the bias and the driver armed on it; skips without the model or the armed tier. `test_gpu_model_swap.das` - stocked suite; two models through one process on the armed tier (Qwen3-0.6B, SmolLM2-135M, `DASLLAMA_GPU=1`): a model reloaded behind the other decodes its own weights, the pin on the upload rail dropping a still-installed model's device state first; skips @@ -408,10 +432,12 @@ against the committed `site/files/dasllama/bench_records.json` (what daslang.io/ renders); red means a records commit skipped `gen_site_records`. `test_tok_seed.das` - model-free: `lcpp_bench.das`'s `tok_read_seed` corpus-header walk, required by relative path (`../benchmarks/lcpp_bench.das`), so it pays the bench's full engine compile. -`test_tokenizer.das` - stocked suite; every cell is fixture-gated (the `ggml-vocab-*.gguf` +`test_tokenizer.das` - stocked suite; the corpus cells are fixture-gated (the `ggml-vocab-*.gguf` corpora under the models dir, machine-local): the seven vocab families' `.inp`/`.out` corpora through `load_tokenizer_auto` -> `encode` / `decode`, ids exact and the decode round-trip -lossless; reports SKIPPED where the vocab is not stocked. +lossless; reports SKIPPED where the vocab is not stocked. One cell is model-free: the BPE +`add_bos` default a GGUF without the key takes (upstream's per-pre table), then the qwen35 vocab +reading `add_bos == false` where that fixture is stocked. `test_exe_smoke.das` - stocked suite; model-gated (SmolLM2-135M, small tier): the standalone-exe context gate. Builds `_exe_smoke_root.das` with `-jit -exe` and runs the artifact - the one rail where globals restore as DATA, so a function-typed global with no diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 5554365ed0..0833f6c51e 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -44,12 +44,16 @@ a run of skips is not the coverage the suite owes. **A diff that registers a test file in this folder in a `CMakeLists.txt` is a defect - a `run.das` suite listing is the only registration these files get.** -**A diff that adds, removes or moves a gate (one test cell, or a file between suites), or -changes the contract a gate pins - what its asserts hold fixed - corrects, in the same change, -every clause in this folder that names that gate's suite, fixture, model, arm, skip condition -or pinned contract - a `CLAUDE.md` clause, this checklist's pinned-set entry, or a test file's -own header.** A clause that only names the file (a brace list, a suite roster) carries nothing -to correct. +**A diff that adds, removes or moves a gate - one test cell, or a file between suites - updates +the `CLAUDE.md` census entry of the file it lands in, in the same change.** A brace list or a +suite roster that only names the file carries nothing to correct. + +**A diff that changes the contract a gate pins - what its asserts hold fixed, an axis gained or +lost - updates that gate's entry in this checklist's pinned set in the same change.** + +**A diff that changes a gate's skip condition - the model, fixture, device or arm it needs - +updates the test file's own header and the `CLAUDE.md` clause that states it, in the same +change.** **A diff that adds, moves, or removes a `[test]` file outside `modules/dasLLAMA/` that carries a `require dasllama/...` line of its own adds, corrects, or drops its row, with the reason it @@ -88,20 +92,20 @@ wire-key pin read out of `../dasllama/dasllama_tune_scope.das`) and `test_bench_records_schema.das` (the `write_bench_records` output, corpus sweeps included); `test_scheduler.das`'s media-stream bypass check (no cached hit at `prefix_attach`, no donated pages at `donate_stream`); `test_vulkan_kernels.das`'s tile-pick cell (which tile the Vulkan -matmul picks for a given width and row count, and whether that dispatch splits its reduction -across partial planes); `utils/dasllama-server/test_worker_dispatch.das` (repo root: worker-local fork pools, shared queue policy). +matmul picks for a given width, row count and coopmat mode, and whether that dispatch splits its +reduction across partial planes); `utils/dasllama-server/test_worker_dispatch.das` (repo root: worker-local fork pools, shared queue policy). **A diff that adds a gate whose failure means a documented contract changed, rather than a kernel regressing, adds it to the pinned set in the same change** - as a file when every cell of it pins, as a named cell otherwise. **On every platform, a cell that neither asserts nor registers a skip is a defect.** A cell -that returns without asserting - the module is absent, its models are not stocked, no device -answered, a capability declined - registers `t |> skip` there; `feint` is a print, not a skip. +that returns without asserting - whatever the reason - registers `t |> skip` there; `feint` is a +print, not a skip. -**A cell whose claim needs a capability the box may lack - a window server, an audio device, a -module the build omits - registers `t |> skip` on that fact before it asserts; a cell that reds -on one instead is a defect.** +**A cell whose claim needs a capability the box may lack - a device, a window server, an audio +device, a module the build omits, a stocked model - registers `t |> skip` on that fact before +it asserts: never a bare return, never a red.** **A cell's skip condition keys on a fact the box owns - a device capability, a run-mode knob's value, a host toolchain's presence, a compile-time module-presence check diff --git a/modules/dasLLAMA/tests/REVIEW_KERNEL_CELLS.md b/modules/dasLLAMA/tests/REVIEW_KERNEL_CELLS.md index 2a832ad4e1..41d5b9392b 100644 --- a/modules/dasLLAMA/tests/REVIEW_KERNEL_CELLS.md +++ b/modules/dasLLAMA/tests/REVIEW_KERNEL_CELLS.md @@ -7,10 +7,11 @@ doc: `CLAUDE.md`. Planned work: `../followup_general.md`, `../followup_vulkan.md **Routed from `REVIEW.md` (beside this file): a diff that checklist routes here applies this list together with it.** -**A diff that changes a kernel's dispatch geometry - its grid divisor, threadgroup size, or -threadgroup-memory length - updates every gate that hand-dispatches that kernel, in the same -change.** A hand-dispatched gate encodes the geometry itself, so a moved divisor leaves the -gate dispatching the wrong shape with no error. +**A diff that changes any dispatch geometry a gate itself encodes - a grid divisor, a +threadgroup size, a threadgroup-memory length the gate sets - updates every gate that +hand-dispatches that kernel, in the same change.** A hand-dispatched gate encodes the geometry +itself, so a moved divisor leaves the gate dispatching the wrong shape with no error; a +`@workgroup` array's size is compiled into the kernel and no gate carries it. **A diff that changes a kernel's kargs - the kernel-argument struct, or any buffer binding - updates every gate that hand-binds that kernel, in the same change.** A stale hand bind reads @@ -18,9 +19,10 @@ the wrong buffer and passes on garbage that happens to compare. **A kernel that gains a kargs field whose non-default value changes what it computes or which elements it reads or writes - a branch selector, a row or element base, a stride - ships, in -the same change, a gate cell that sets that field to a non-default value.** At the default the -new field has no visible effect: a CPU oracle that ignores it and the kernel that honors it -agree. +the same change, a kernel-unit cell that sets that field to a non-default value.** At the +default the new field has no visible effect: a CPU oracle that ignores it and the kernel that +honors it agree. A model-level cell that reaches the field does not discharge this: it binds the +layer's own row, so a base stays at its default there. **A kernel-unit cell - a model-less cell that dispatches one kernel class and asserts on its output - missing a compare against a CPU oracle that can witness the cell's property is a diff --git a/modules/dasLLAMA/tests/_model_tier.das b/modules/dasLLAMA/tests/_model_tier.das index 99df961a57..9bb0524c49 100644 --- a/modules/dasLLAMA/tests/_model_tier.das +++ b/modules/dasLLAMA/tests/_model_tier.das @@ -26,6 +26,10 @@ require dasllama/dasllama_env // (incl. gemma-4-E2B at 4.97GB) from the 7.7GB+ heavies. let LARGE_TIER_BYTES = 6ul * 1024ul * 1024ul * 1024ul +//! The KV codec a session takes to be served by the armed mirror: f16, or f32 where DASLLAMA_VK_KV32 +//! arms f32 mirrors - a session on the other codec rides the CPU rails. +def mirror_kv_dtype() : KVDtype => g_env_vulkan.vk_kv32 != 0l ? KVDtype.f32 : KVDtype.f16 + //! Max |a[i] - b[i]| over two same-length float arrays - the compare behind a logits bar. def logits_maxdiff(a, b : array) : float { var m = 0.0 diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 3af9753c64..4c7f0d4304 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -260,9 +260,9 @@ def iq4xs_gemv_float_oracle(wq, ws, xq : array; xs : array; for (s in range(nsb)) { let sb = w0 + s let xsb = x0 + s - let dd = unpackHalf2x16(ws[sb * 5]).x + let dd = unpackHalf2x16(ws[sb * 2]).x for (blk in range(8)) { - let sc = byte_s8(ws, sb * 20 + 4 + blk) + let sc = iq4xs_sc(ws[sb * 2], ws[sb * 2 + 1], uint(blk)) var bacc = 0.0 for (m in range(16)) { let b = byte_u8(wq, sb * 128 + blk * 16 + m) @@ -332,9 +332,9 @@ def iq3s_gemv_float_oracle(wq, ws, xq : array; xs : array; for (s in range(nsb)) { let sb = w0 + s let xsb = x0 + s - let dd = half_at(ws, sb * 10) + let dd = unpackHalf2x16(ws[sb * 2]).x for (blk in range(8)) { - let sc = float(byte_s8(ws, sb * 20 + 4 + blk)) // the plane byte IS the signed (1 + 2s) sub-scale + let sc = float(grid_sc(ws[sb * 2 + 1], uint(blk))) // the row's 4-bit s field -> the signed (1 + 2s) sub-scale var bacc = 0.0 for (e in range(32)) { let l = e / 8 @@ -369,9 +369,9 @@ def iq3xxs_gemv_float_oracle(wq, ws, xq : array; xs : array; for (s in range(nsb)) { let sb = w0 + s let xsb = x0 + s - let dd = half_at(ws, sb * 10) + let dd = unpackHalf2x16(ws[sb * 2]).x for (blk in range(8)) { - let sc = float(byte_s8(ws, sb * 20 + 4 + blk)) // the plane byte IS the (2ls + 1) sub-scale + let sc = float(grid_sc(ws[sb * 2 + 1], uint(blk))) // the row's 4-bit field -> the (2ls + 1) sub-scale let aux = byte_u8(wq, sb * 96 + 64 + blk * 4) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 1) << 8) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 2) << 16) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 3) << 24) var bacc = 0.0 for (e in range(32)) { @@ -488,11 +488,11 @@ def iq2xxs_gemv_float_oracle(wq, ws, xq : array; xs : array; for (s in range(nsb)) { let sb = w0 + s let xsb = x0 + s - let dd = half_at(ws, sb * 10) + let dd = unpackHalf2x16(ws[sb * 2]).x for (b in range(8)) { let gw = wq[sb * 16 + b * 2] let aux = int(wq[sb * 16 + b * 2 + 1]) - let sc = float(byte_s8(ws, sb * 20 + 4 + b)) // SIGNED like the class/batch stage - real (2ls+1) strips stay positive + let sc = float(grid_sc(ws[sb * 2 + 1], uint(b))) // the row's 4-bit field -> the (2ls + 1) sub-scale, as the class stage decodes it var bacc = 0.0 for (l in range(4)) { let idx = int((gw >> uint(8 * l)) & 0xFFu) @@ -531,9 +531,9 @@ def iq3sf16_gemm_oracle(wq : array; wsu : array; xf : array; var acc = 0.0 for (s in range(nsb)) { let sb = wsb0 + c * nsb + s - let dv = unpackHalf2x16(wsu[sb * 5]).x + let dv = unpackHalf2x16(wsu[sb * 2]).x for (g in range(8)) { - let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + let sc = float(grid_sc(wsu[sb * 2 + 1], uint(g))) for (e in range(32)) { let wj = e / 4 let qi = byte_u8(wq, sb * 104 + g * 8 + wj) | ((byte_u8(wq, sb * 104 + 64 + g) << (8 - wj)) & 256) @@ -565,9 +565,9 @@ def iq3xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; var acc = 0.0 for (s in range(nsb)) { let sb = wsb0 + c * nsb + s - let dv = unpackHalf2x16(wsu[sb * 5]).x + let dv = unpackHalf2x16(wsu[sb * 2]).x for (g in range(8)) { - let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + let sc = float(grid_sc(wsu[sb * 2 + 1], uint(g))) let aux = byte_u8(wq, sb * 96 + 64 + g * 4) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 1) << 8) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 2) << 16) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 3) << 24) for (e in range(32)) { let qi = byte_u8(wq, sb * 96 + g * 8 + e / 4) @@ -674,7 +674,7 @@ def iq2xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; var acc = 0.0 for (s in range(nsb)) { let sb = wsb0 + c * nsb + s - let dv = unpackHalf2x16(wsu[sb * 5]).x + let dv = unpackHalf2x16(wsu[sb * 2]).x for (e in range(256)) { let g = e / 32 let rr2 = e % 32 @@ -688,7 +688,7 @@ def iq2xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; tt = tt ^ (tt >> 2) tt = tt ^ (tt >> 1) let sgb = sidx | ((tt & 1) << 7) - let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - (g % 4)) * 8)) >> 24) + let sc = float(grid_sc(wsu[sb * 2 + 1], uint(g))) let gb = float(int(IQ2XXS_GRID[idx * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) let w = float(float16(dv * sc * (((sgb >> j) & 1) != 0 ? -gb : gb))) acc += w * half_at(xf, (row0 + r) * n + s * 256 + e) @@ -867,9 +867,9 @@ def iq4xsf16_gemm_oracle(wq : array; wsu : array; xf : array; var acc = 0.0 for (s in range(nsb)) { let sb = wsb0 + c * nsb + s - let dv = unpackHalf2x16(wsu[sb * 5]).x + let dv = unpackHalf2x16(wsu[sb * 2]).x for (g in range(8)) { - let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + let sc = float(iq4xs_sc(wsu[sb * 2], wsu[sb * 2 + 1], uint(g))) for (e in range(32)) { let by = byte_u8(wq, sb * 128 + g * 16 + e % 16) let q = (by >> ((e / 16) * 4)) & 0xF diff --git a/modules/dasLLAMA/tests/_vkd_toy.das b/modules/dasLLAMA/tests/_vkd_toy.das index 7d8be8a260..14593d0f60 100644 --- a/modules/dasLLAMA/tests/_vkd_toy.das +++ b/modules/dasLLAMA/tests/_vkd_toy.das @@ -37,6 +37,36 @@ class VkdToy { } } +struct VdBlk { + qs : int16[16] //! 32 int8 quants as 16-bit lanes (the q8 block shape) +} + +struct VdArgs { + n : uint +} + +[vk_dispatch(name = "vkd_direct_decode", grid = "wgs", params = "wgs : int64")] +class VkdDirectDecode { + @ssbo @binding = 0 wq : array + @ssbo @binding = 1 ws : array + @ssbo @binding = 2 y : array + @push_constant pa : VdArgs + + [spirv_decode] + def decode(blk : VdBlk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return ws[bc.y] * float16(float(int(q))) + } + + [spirv_kernel(local_size_x = 64, name = "vkd_direct_decode_spv")] + def run { + let gid = gl_GlobalInvocationID.x + if (gid < pa.n) { + y[gid] = float(decode(wq[gid >> 5u], uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + } + } +} + // Two stamps of the served q8 cm2 l-tile, for the four-wide decode's device pair: the test // ensures one with `g_gpu.decvec_on` off (its words stripped to the scalar callback) and one // with it on (the twin served), and holds both to one CPU oracle. Stamps, so the served class's diff --git a/modules/dasLLAMA/tests/run.das b/modules/dasLLAMA/tests/run.das index e718672e4f..55b91b51bf 100644 --- a/modules/dasLLAMA/tests/run.das +++ b/modules/dasLLAMA/tests/run.das @@ -184,6 +184,7 @@ def suite_files(name : string) : array { // nolint:STYLE038 - a flat s "modules/dasLLAMA/tests/test_gemma4v.das", "modules/dasLLAMA/tests/test_gpu_model_swap.das", "modules/dasLLAMA/tests/test_gpu_resident_hybrid.das", + "modules/dasLLAMA/tests/test_gpu_resident_qwen2.das", "modules/dasLLAMA/tests/test_gpu_slot_swap.das", "modules/dasLLAMA/tests/test_kquant.das", "modules/dasLLAMA/tests/test_kv_codec.das", @@ -255,7 +256,7 @@ def area_tests(area : string) : array { return <- [ "test_accel_backend.das", "test_affinity_predicates.das", "test_arch_registry.das", "test_batch_decode.das", "test_batch_grid.das", "test_chat.das", "test_deltanet.das", "test_dispatch_shaping.das", "test_facade.das", "test_flash.das", "test_forward.das", "test_fused_decode.das", "test_gguf_quant.das", "test_gguf_shards.das", - "test_gpu_model_swap.das", "test_gpu_resident_hybrid.das", "test_gpu_serving_declines.das", "test_gpu_slot_swap.das", "test_gpu_tier.das", "test_groupn.das", + "test_gpu_model_swap.das", "test_gpu_resident_hybrid.das", "test_gpu_resident_qwen2.das", "test_gpu_serving_declines.das", "test_gpu_slot_swap.das", "test_gpu_tier.das", "test_groupn.das", "test_jobque_tripwire.das", "test_kernel_backend.das", "test_kgroup_repack.das", "test_kqformat.das", "test_kquant.das", "test_kv_codec.das", "test_kv_paged.das", "test_kv_prefix.das", "test_math_activations.das", "test_math_elementwise.das", "test_matmul.das", "test_matmul_batch.das", diff --git a/modules/dasLLAMA/tests/test_gguf_quant.das b/modules/dasLLAMA/tests/test_gguf_quant.das index c71dc34289..a66eaab543 100644 --- a/modules/dasLLAMA/tests/test_gguf_quant.das +++ b/modules/dasLLAMA/tests/test_gguf_quant.das @@ -6,6 +6,7 @@ require dasllama/dasllama_convert // the codec layer under test — direct requ require dasllama/dasllama_gguf require dasllama/dasllama_common // dequant_kq_plane_sb - the plane dequant the f32 fallback must match require math +require daslib/enum_trait // Byte-level GGUF quant-format gates on a synthetic in-memory file: the dequant arm must match an // independently hand-rolled reference, and the exact-transcode rails must be bit-exact against @@ -78,6 +79,36 @@ def private synth_q5_0_bytes : array { return <- data } +[test] +def test_gguf_type_sizes(t : T?) { + t |> run("ggml_type_bytes follows the ggml block sizes and ggml_type_name the upstream spelling") @(t : T?) { + t |> equal(144l, ggml_type_bytes(GGML_TYPE_Q4_K, 256l)) + t |> equal(210l, ggml_type_bytes(GGML_TYPE_Q6_K, 256l)) + t |> equal(136l, ggml_type_bytes(GGML_TYPE_IQ4_XS, 256l)) + t |> equal(110l, ggml_type_bytes(GGML_TYPE_IQ3_S, 256l)) + t |> equal(82l, ggml_type_bytes(GGML_TYPE_IQ2_S, 256l)) + t |> equal(34l, ggml_type_bytes(GGML_TYPE_Q8_0, 32l)) + t |> equal(18l, ggml_type_bytes(GGML_TYPE_IQ4_NL, 32l)) + t |> equal(2048l, ggml_type_bytes(GGML_TYPE_F16, 1024l)) + t |> equal(0l, ggml_type_bytes(99, 256l)) + t |> equal("IQ4_XS", ggml_type_name(GGML_TYPE_IQ4_XS)) + t |> equal("Q8_0", ggml_type_name(GGML_TYPE_Q8_0)) + t |> equal("type 99", ggml_type_name(99)) + } + t |> run("every kq format names its ggml type, and kq_disk_bytes is that type's block bytes") @(tt : T?) { + tt |> equal(GGML_TYPE_Q4_K, kq_ggml_type(KqFmt.k4)) + tt |> equal(GGML_TYPE_IQ2_XXS, kq_ggml_type(KqFmt.iq2xxs)) + tt |> equal(GGML_TYPE_Q8_0, kq_ggml_type(KqFmt.q8)) + tt |> equal(144l, kq_disk_bytes(KqFmt.k4, 256l)) + tt |> equal(34l, kq_disk_bytes(KqFmt.q8, 32l)) + tt |> equal(18l, kq_disk_bytes(KqFmt.iq4nl, 32l)) + for (f in type) { + tt |> success(kq_ggml_type(f) >= 0, "{f} names a ggml type") + tt |> success(kq_disk_bytes(f, 256l) > 0l, "{f} has block bytes") + } + } +} + [test] def test_gguf_q5_0(t : T?) { // The dequant arm (gguf_read_tensor_f32) must match the hand-rolled reference exactly. diff --git a/modules/dasLLAMA/tests/test_gpu_resident_hybrid.das b/modules/dasLLAMA/tests/test_gpu_resident_hybrid.das index f551ba42c4..b671da8f78 100644 --- a/modules/dasLLAMA/tests/test_gpu_resident_hybrid.das +++ b/modules/dasLLAMA/tests/test_gpu_resident_hybrid.das @@ -16,6 +16,7 @@ require daslib/fio require dasllama/dasllama_math // setup_dasllama_jobque_ (the engine-standard jobque config) require dasllama/dasllama_gpu_tier // moe_gpu_tier_installed / gpu_want_auto: is the tier armed require dasllama/dasllama_gpu_resident // moe_gpu_resident_active: did the whole-model driver take the hybrid +require ?vulkan dasllama/dasllama_vulkan_common // PF_WINDOW and the resolved coopmat mode, where the module exists require _model_tier // models_dir() + model_available() presence/size-tier gate // The whole-model resident driver on a deltanet hybrid (Qwen3.5-0.8B-Q8_0 under DASLLAMA_GPU=1): @@ -30,10 +31,24 @@ require _model_tier // models_dir() + model_available() presence/size-tier gat // (both conv-tail edges: the last window shorter than the taps). let private DN_LOGIT_BAR_REL = 0.04 // of the step's max |logit| - the deltanet bar test_deltanet.das holds too -//! the K-quant bar: CPU-vs-device rounding sits flat at 0.39-0.57 of a ~12.5 max logit at every step (the Q8 file 0.19-0.36), with no growth and the one-step-off control well past it +//! the K-quant bar on the f16 mirror: CPU-vs-device rounding sits flat at 0.39-0.57 of a ~12.5 max logit at every step (the Q8 file 0.19-0.36), with no growth and the one-step-off control well past it let private DN_LOGIT_BAR_REL_KQ = 0.06 -let private PF_WINDOW_ROWS = 512l // dasllama_vulkan_prefill's PF_WINDOW: a prompt past it runs as sequential windows +let private PF_WINDOW_ROWS = 512l //! the engine's PF_WINDOW, asserted equal by the two-window cells: a prompt past it runs as sequential windows +//! the served tile arm, so a red names which prefill tile it came from +def private tile_arm_tag() : string { + static_if (typeinfo builtin_module_exists(vulkan)) { + return "coopmat mode {g_gpu.coopmat_mode}" + } else { + return "no vulkan module" + } +} + +def private window_pin(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + t |> equal(PF_WINDOW, PF_WINDOW_ROWS, "the prompt is sized past the engine's own window") + } +} def private argmax_of(v : array) : int64 { var best = -1e30 var bi = 0l @@ -61,8 +76,7 @@ def private feed_logits(tr : Model; var s : Session; prompt : array; toks } // prefill `n` tokens + 8 fed steps on the resident driver, then the same on the all-CPU chain; the -// prefill logits and every step's within the deltanet bar, the one-step-off control past it. A -// K-quant file (`kq`) must keep its deltanet qkv/z planes in their file formats on the driver. +// prefill logits and every step's within the deltanet bar, the one-step-off control past it. def private hybrid_forced_feed(t : T?; n : int64; file : string; kq : bool) { if (!jit_enabled()) { t |> skip("interpreted (dasLLAMA model tests are JIT-only)") @@ -74,29 +88,34 @@ def private hybrid_forced_feed(t : T?; n : int64; file : string; kq : bool) { t |> skip("the GPU tier is not armed - needs DASLLAMA_GPU=1 on a device that serves the whole-model driver") return } + if (kq && mirror_kv_dtype() == KVDtype.f32) { + t |> skip("the K-quant bar is calibrated on the f16 mirror the tier arms; the f32 A/B mirror (DASLLAMA_VK_KV32=1) is no gate for it") + return + } var tr <- load_model_(path, QuantMode.q8) t |> success(moe_gpu_resident_active(), "the whole-model driver armed on the hybrid (the load log names any decline)") if (kq) { var native = 0l for (l in range64(tr.config.n_layers)) { - if (layer_is_recurrent(tr.config, l) && fmt_at(tr.dnqkv_fmt, l) != KqFmt.q8 && fmt_at(tr.dngate_fmt, l) != KqFmt.q8) { + if (layer_is_recurrent(tr.config, l) && fmt_at(tr.dnqkv_fmt, l) != KqFmt.q8 && fmt_at(tr.dngate_fmt, l) != KqFmt.q8 + && fmt_at(tr.dnout_fmt, l) != KqFmt.q8) { native++ } } - t |> success(native > 0l, "the loader kept the deltanet qkv/z planes in their K-quant file formats ({native} recurrent layers)") + t |> success(native > 0l, "the loader kept the deltanet qkv/z/out planes in their K-quant file formats ({native} recurrent layers)") } with_job_que() { setup_dasllama_jobque_() var prompt <- [for (i in range64(n)); 1000l + (i * 37l) % 5000l] var toks <- [for (i in range64(8l)); 2000l + (i * 53l) % 4000l] - var sg <- create_session(tr, KVDtype.f16) // the armed mirror codec + var sg <- create_session(tr, mirror_kv_dtype()) let served0 = moe_gpu_resident_prefills() var gpu <- feed_logits(tr, sg, prompt, toks) t |> success(moe_gpu_resident_active(), "the whole-model driver stayed armed through the run") t |> equal(moe_gpu_resident_prefills(), served0 + 1l, "the resident window chain served the {n}-token prompt") delete sg moe_gpu_drop_model() // every later step is the CPU chain - var sc <- create_session(tr, KVDtype.f16) + var sc <- create_session(tr, mirror_kv_dtype()) var cpu <- feed_logits(tr, sc, prompt, toks) delete sc let rel = kq ? DN_LOGIT_BAR_REL_KQ : DN_LOGIT_BAR_REL @@ -105,7 +124,7 @@ def private hybrid_forced_feed(t : T?; n : int64; file : string; kq : bool) { let bar = rel * logits_maxabs(cpu[st]) let ag = argmax_of(gpu[st]) let ac = argmax_of(cpu[st]) - let what = st == 0l ? "prefill ({n} tokens)" : "step {st - 1l} (fed {toks[st - 1l]})" + let what = (st == 0l ? "prefill ({n} tokens)" : "step {st - 1l} (fed {toks[st - 1l]})") + " [{tile_arm_tag()}]" to_log(LOG_INFO, "{what}: argmax cpu {ac} ({cpu[st][ac]}), resident {ag} ({gpu[st][ag]}); logits maxdiff {diff} (bar {bar})\n") t |> success(diff <= bar, "{what} logits: resident within {bar} of the CPU chain (maxdiff {diff})") if (st > 0l) { // the control: the previous step's CPU logits are a disconnected mechanism the bar must RED @@ -154,6 +173,7 @@ def test_gpu_resident_hybrid_kq(t : T?) { hybrid_forced_feed(t, 40l, "Qwen3.5-0.8B-Q4_K_M.gguf", true) } t |> run("Qwen3.5-0.8B-Q4_K_M: the same across two windows") @(t : T?) { + window_pin(t) hybrid_forced_feed(t, PF_WINDOW_ROWS + 88l, "Qwen3.5-0.8B-Q4_K_M.gguf", true) } } diff --git a/modules/dasLLAMA/tests/test_gpu_resident_qwen2.das b/modules/dasLLAMA/tests/test_gpu_resident_qwen2.das new file mode 100644 index 0000000000..2595d96e00 --- /dev/null +++ b/modules/dasLLAMA/tests/test_gpu_resident_qwen2.das @@ -0,0 +1,128 @@ +options gen2 +options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) +options persistent_heap // + explicit deletes below: the model and sessions free before exit +options _dasllama_internal = true + +require dastest/testing_boost public +require dasllama/dasllama_transformer // umbrella fires each arch [init] registration (qwen2) +require dasllama/dasllama // create_session, moe_gpu_drop_model +require daslib/jobque_boost +require daslib/fio +require dasllama/dasllama_math // setup_dasllama_jobque_ (the engine-standard jobque config) +require dasllama/dasllama_gpu_tier // moe_gpu_tier_installed / gpu_want_auto: is the tier armed +require dasllama/dasllama_gpu_resident // moe_gpu_resident_active: did the whole-model driver take the model +require ?vulkan dasllama/dasllama_vulkan_common // PF_WINDOW and the resolved coopmat mode, where the module exists +require _model_tier // models_dir() + model_available() presence/size-tier gate + +// The whole-model resident driver on a qwen2 (Qwen2.5-0.5B-Instruct-Q8_0 under DASLLAMA_GPU=1): the +// q/k/v projection bias folds into the rope stage on the device. The forced-feed logits-tolerance +// form of test_gpu_resident_hybrid: the same fixed tokens through the resident chain and the +// all-CPU chain, the prefill logits and every step's within the bar, the one-step-off control past +// it - a dropped or misplaced bias moves the logits far past the bar (the bias is a large share of +// a 0.5B's k rows), so the bar is the bias arm's gate. Two prompt lengths: one window, two windows. + +[init] +def private allow_cpu_prefill_intent() { + allow_cpu_prefill() // the control arm (the model dropped off the device) prefills on the CPU by design +} + +//! of the step's max |logit|: the 0.5B's CPU-vs-device rounding sits flat at 0.30-0.50 of a 9.5-18 max logit +//! at every step, while a dropped bias lands whole logits off - the hybrid file's K-quant bar. +let private LOGIT_BAR_REL = 0.06 +let private PF_WINDOW_ROWS = 512l //! the engine's PF_WINDOW, asserted equal by the two-window cell: a prompt past it runs as sequential windows + +//! the served tile arm, so a red names which prefill tile it came from +def private tile_arm_tag() : string { + static_if (typeinfo builtin_module_exists(vulkan)) { + return "coopmat mode {g_gpu.coopmat_mode}" + } else { + return "no vulkan module" + } +} + +def private argmax_of(v : array) : int64 { + var best = -1e30 + var bi = 0l + for (i in range64(long_length(v))) { + if (v[i] > best) { + best = v[i] + bi = i + } + } + return bi +} + +def private feed_logits(tr : Model; var s : Session; prompt : array; toks : array) : array> { + let n = long_length(prompt) + var out : array> + out |> reserve(long_length(toks) + 1l) + forward_prefill(tr, s, prompt, n, 0l) + out |> emplace <| [for (i in range64(tr.config.vocab_size)); s.logits[i]] + for (st in range64(long_length(toks))) { + forward(tr, s, toks[st], n + st) + out |> emplace <| [for (i in range64(tr.config.vocab_size)); s.logits[i]] + } + return <- out +} + +def private qwen2_forced_feed(t : T?; n : int64) { + if (!jit_enabled()) { + t |> skip("interpreted (dasLLAMA model tests are JIT-only)") + return + } + let path = path_join(models_dir(), "Qwen2.5-0.5B-Instruct-Q8_0.gguf") + if (!model_available(t, path)) return + if (!moe_gpu_tier_installed() || !gpu_want_auto()) { + t |> skip("the GPU tier is not armed - needs DASLLAMA_GPU=1 on a device that serves the whole-model driver") + return + } + var tr <- load_model_(path, QuantMode.q8) + t |> success(tr.config.attn_qkv_bias, "the qwen2 carries the q/k/v projection bias the arm exists for") + t |> success(moe_gpu_resident_active(), "the whole-model driver armed on the biased model (the load log names any decline)") + with_job_que() { + setup_dasllama_jobque_() + var prompt <- [for (i in range64(n)); 1000l + (i * 37l) % 5000l] + var toks <- [for (i in range64(8l)); 2000l + (i * 53l) % 4000l] + var sg <- create_session(tr, mirror_kv_dtype()) + let served0 = moe_gpu_resident_prefills() + var gpu <- feed_logits(tr, sg, prompt, toks) + t |> success(moe_gpu_resident_active(), "the whole-model driver stayed armed through the run") + t |> equal(moe_gpu_resident_prefills(), served0 + 1l, "the resident window chain served the {n}-token prompt") + delete sg + moe_gpu_drop_model() //! every later step is the CPU chain + var sc <- create_session(tr, mirror_kv_dtype()) + var cpu <- feed_logits(tr, sc, prompt, toks) + delete sc + for (st in range64(long_length(cpu))) { + let diff = logits_maxdiff(gpu[st], cpu[st]) + let bar = LOGIT_BAR_REL * logits_maxabs(cpu[st]) + let ag = argmax_of(gpu[st]) + let ac = argmax_of(cpu[st]) + let what = (st == 0l ? "prefill ({n} tokens)" : "step {st - 1l} (fed {toks[st - 1l]})") + " [{tile_arm_tag()}]" + to_log(LOG_INFO, "{what}: argmax cpu {ac} ({cpu[st][ac]}), resident {ag} ({gpu[st][ag]}); logits maxdiff {diff} (bar {bar})\n") + t |> success(diff <= bar, "{what} logits: resident within {bar} of the CPU chain (maxdiff {diff})") + if (st > 0l) { //! the control: the previous step's CPU logits are a disconnected mechanism the bar must RED + let off = logits_maxdiff(gpu[st], cpu[st - 1l]) + t |> success(off > bar, "{what} control: the bar discriminates a one-step-off compare (maxdiff {off} vs bar {bar})") + } + } + delete gpu + delete cpu + delete prompt + delete toks + } + delete tr +} + +[test] +def test_gpu_resident_qwen2_bias(t : T?) { + t |> run("Qwen2.5-0.5B: resident prefill + decode with the q/k/v bias on device vs the CPU chain, one window") @(t : T?) { + qwen2_forced_feed(t, 40l) + } + t |> run("Qwen2.5-0.5B: the same across two windows") @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + t |> equal(PF_WINDOW, PF_WINDOW_ROWS, "the prompt is sized past the engine's own window") + } + qwen2_forced_feed(t, PF_WINDOW_ROWS + 88l) + } +} diff --git a/modules/dasLLAMA/tests/test_gpu_serving_declines.das b/modules/dasLLAMA/tests/test_gpu_serving_declines.das index f835c71c73..bc2ece2ca0 100644 --- a/modules/dasLLAMA/tests/test_gpu_serving_declines.das +++ b/modules/dasLLAMA/tests/test_gpu_serving_declines.das @@ -4,9 +4,13 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_common require dasllama/dasllama_gpu_resident +require dasllama/dasllama_gpu_tier // install_moe_gpu_resident_bias - the bias seat a served qwen2 needs require strings require daslib/strings_boost +[unused_argument(rows)] +def private test_upload_bias(rows : array) {} + // The whole-model driver's decline reasons, decided from a Config or a synthetic Model shell // (no weights on disk): every feature and layer shape the driver has no arm for is named in // the text a user reads, and a shape it serves yields no text at all. @@ -53,6 +57,20 @@ def test_resident_unserved_features(t : T?) { c.attn_sinks = true t |> equal(resident_unserved_features(c), "attention sinks") } + t |> run("the q/k/v projection bias is named only while no bias seat is installed; a seat serves the qwen2 shape") @(t : T?) { + var c = Config() + c.attn_qkv_bias = true + c.rope_neox = true + let saved_seat = g_rdec_upload_bias + let saved_installed = g_rdec_bias_installed + g_rdec_bias_installed = false + t |> equal(resident_unserved_features(c), "attention qkv bias") + install_moe_gpu_resident_bias(@@test_upload_bias) + t |> equal(resident_unserved_features(c), "") + t |> equal(attn_chain_unserved_features(c), "", "the per-op chain carries the bias through its hooks - no seat to install") + g_rdec_upload_bias = saved_seat + g_rdec_bias_installed = saved_installed + } } [test] @@ -62,6 +80,10 @@ def test_attn_chain_unserved_features(t : T?) { t |> equal(attn_chain_unserved_features(c), "interleaved (non-NEOX) rope") c.rope_neox = true t |> equal(attn_chain_unserved_features(c), "") + c.attn_qkv_bias = true + t |> equal(attn_chain_unserved_features(c), "", "the q/k/v projection bias is served (qwen2moe)") + c.attn_out_bias = true + t |> equal(attn_chain_unserved_features(c), "attention output bias") var p = Config() p.head_size = 256l p.rope_dim = 64l @@ -91,9 +113,12 @@ def test_resident_layer_decline(t : T?) { m.dnout_fmt <- [KqFmt.q8, KqFmt.q8, KqFmt.q8] let sq = resident_layer_decline(m, 1l, hid0, hs, kvd) t |> success(!says(sq, "kq-tagged") && !says(sq, "do not serve"), "a K-quant qkv/z pair is no format decline: '{sq}'") - // only the OUT plane kq-tagged: the step's o row feeds a q8 GEMV, so the layer declines by number m.dnout_fmt <- [KqFmt.q8, KqFmt.k4, KqFmt.q8] - t |> success(says(resident_layer_decline(m, 1l, hid0, hs, kvd), "layer 1's deltanet out plane is kq-tagged"), "kq-tagged out plane named") + let so = resident_layer_decline(m, 1l, hid0, hs, kvd) + t |> success(!says(so, "out plane"), "a K-quant out plane is no format decline: '{so}'") + //! an inner width off the format's block leaves the out plane untileable - the decline names it + m.config.ssm_d_inner = 100l + t |> success(says(resident_layer_decline(m, 1l, hid0, hs, kvd), "layer 1's deltanet out plane carries a format"), "an untileable out plane is named") delete m } t |> run("shared K/V, a missing plane and a differing FFN width each name the layer") @(t : T?) { @@ -110,3 +135,47 @@ def test_resident_layer_decline(t : T?) { delete m } } + +[test] +def test_resident_binding_ctx(t : T?) { + t |> run("the KV mirror's binding cap counts the attention layers' rows, whatever layer 0 is") @(t : T?) { + let bcap = 4_294_967_295l //! the 4 GiB maxStorageBufferRange of the NV/AMD drivers + //! 32 layers, attention every fourth (3, 7, ..., 31), kv width 8 x 128 in f16: one side is 8 x 2048 B per position + var hy <- shell(32l, 0x7777_7777ul) + hy.config.n_kv_heads = 8l + hy.config.head_size = 128l + t |> success(layer_is_recurrent(hy.config, 0l) && layer_kv_dim(hy.config, 0l) == 0l, "layer 0 is recurrent: its kv width reads 0") + t |> equal(resident_binding_ctx(hy.config, KVDtype.f16, bcap), 262143l, "the hybrid's ctx is the range over eight attention rows, not skipped for layer 0's zero width") + t |> equal(resident_binding_ctx(hy.config, KVDtype.f32, bcap), 131071l, "f32 rows halve it") + t |> equal(resident_binding_ctx(hy.config, KVDtype.f16, 0l), 0l, "no cap known reads 0") + delete hy + var dense <- shell(32l, 0ul) + dense.config.n_kv_heads = 8l + dense.config.head_size = 128l + t |> equal(resident_binding_ctx(dense.config, KVDtype.f16, bcap), 65535l, "the dense twin counts all 32 rows") + delete dense + var recr <- shell(4l, 0xFul) + t |> equal(resident_binding_ctx(recr.config, KVDtype.f16, bcap), 0l, "no attention layer reads 0") + delete recr + } +} + +[test] +def test_plan_room(t : T?) { + t |> run("the plan's room: the tier cap less headroom, capped by the OS's room when it answers") @(t : T?) { + let gib = 1_073_741_824l + let silent = (budget = 0l, usage = 0l, adapter_dedicated = 0l, adapter_shared = 0l) + let no_os = plan_room(16l * gib, false, silent) + t |> equal(16l * gib * 27l / 100l, no_os.headroom, "no OS answer: 27% of a 16 GiB cap, past the 2 GiB floor, is the headroom") + t |> equal(16l * gib - no_os.headroom, no_os.usable, "no OS answer: the cap less the headroom") + t |> equal(0l, no_os.others, "no OS answer: nobody else is counted") + let os = (budget = 14l * gib, usage = 1l * gib, adapter_dedicated = 5l * gib, adapter_shared = 0l) + let answered = plan_room(16l * gib, false, os) + t |> equal(0l, answered.headroom, "an OS answer takes no headroom") + t |> equal(4l * gib, answered.others, "the other processes hold the adapter's dedicated usage less ours") + t |> equal(14l * gib - 4l * gib - 1l * gib, answered.usable, "the OS budget less the others and the 1 GiB reserve") + let pinned = plan_room(8l * gib, true, os) + t |> equal(8l * gib, pinned.usable, "a pinned cap fills itself and reads no room") + t |> equal(0l, pinned.others, "a pinned cap counts nobody else") + } +} diff --git a/modules/dasLLAMA/tests/test_gpu_tier.das b/modules/dasLLAMA/tests/test_gpu_tier.das index 54c603b894..a7061a49fc 100644 --- a/modules/dasLLAMA/tests/test_gpu_tier.das +++ b/modules/dasLLAMA/tests/test_gpu_tier.das @@ -13,6 +13,7 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_gpu_tier require dasllama/dasllama_config +require dasllama/dasllama_env // g_env_gpu: an exported DASLLAMA_GPU answers gpu_want_auto ahead of the want state // ===== rdec doubles: record what arrives, answer distinctively ===== @@ -195,7 +196,7 @@ def test_gpu_tier_contract(t : T?) { t |> success(gpu_want_attn(), "attn wanted") t |> equal(gpu_want_heat(), 5l) t |> success(gpu_want_qkv(), "qkv wanted") - t |> success(!gpu_want_auto(), "auto not requested") + t |> equal(gpu_want_auto(), g_env_gpu.gpu ?? false, "auto follows an exported DASLLAMA_GPU, else the set state (not requested)") t |> success(gpu_want_arms_tier(), "arms-tier reflects the set state") // membership witness: ONE rail alone must flip the predicate (the aggregate case // above cannot see a dropped member) diff --git a/modules/dasLLAMA/tests/test_kernel_coverage.das b/modules/dasLLAMA/tests/test_kernel_coverage.das index e1f667eaef..6f40323305 100644 --- a/modules/dasLLAMA/tests/test_kernel_coverage.das +++ b/modules/dasLLAMA/tests/test_kernel_coverage.das @@ -683,12 +683,34 @@ def test_vulkan_kernel_coverage_serving(tst : T?) { vk_cov_model(t, path_join(models_dir(), "tinyllama-1.1b-chat-v1.0.Q8_0.gguf"), "llama", [2]) // gemma3 q8: the per-op arena rails + qk_norm vk_cov_model(t, path_join(models_dir(), "gemma-3-1b-it-Q8_0.gguf"), "gemma3", none) - // qwen3 q8: the resident fused qk_norm rail (qkn_rope) — the kq row below rides the - // split rail by construction, so this is the fused twin's only carrier - vk_cov_model(t, path_join(models_dir(), "Qwen3-0.6B-Q8_0.gguf"), "qwen3", none) - // qwen3 Q4_K_M: the native K-quant formats (k4/k6 GEMV + batch tiles) + the Q8_K - // activation quant forms + qk_norm on the split rail - vk_cov_model(t, path_join(models_dir(), "Qwen3-4B-Instruct-2507-Q4_K_M.gguf"), "qwen3", none) + // every prefill tile family under the coopmat modes its planes have twins in - the + // box's resolved mode, mm, sdot4, and for the q8 and q40 planes the f16 and int8 + // coopmat twins too; the carriers are the qwen3 files (the Q8_0 also carries the + // resident fused qk_norm rail; the Q4_K_M k4/k6 with the Q8_K activation quant forms + // and qk_norm on the split rail) and the 1B requants the format arc used + let vehicles = fixed_array( + (file = "Qwen3-0.6B-Q8_0.gguf", family = "qwen3", nmodes = 5), + (file = "Llama-3.2-1B-Instruct-Q4_0-local.gguf", family = "llama", nmodes = 5), + (file = "Qwen3-4B-Instruct-2507-Q4_K_M.gguf", family = "qwen3", nmodes = 3), // k4 + k6 + (file = "Llama-3.2-1B-Instruct-IQ4_XS.gguf", family = "llama", nmodes = 3), + (file = "Llama-3.2-1B-Instruct-Q3_K_L.gguf", family = "llama", nmodes = 3), // k3 + k5 + (file = "Llama-3.2-1B-Instruct-IQ3_M.gguf", family = "llama", nmodes = 3), // iq3s + (file = "Llama-3.2-1B-Instruct-IQ3_XXS-local.gguf", family = "llama", nmodes = 3), + (file = "Llama-3.2-1B-Instruct-IQ4_NL-local.gguf", family = "llama", nmodes = 3), + (file = "Llama-3.2-1B-Instruct-Q2_K-local.gguf", family = "llama", nmodes = 3), + (file = "Llama-3.2-1B-Instruct-IQ2_XS-local.gguf", family = "llama", nmodes = 3), + (file = "Llama-3.2-1B-Instruct-IQ2_XXS-local.gguf", family = "llama", nmodes = 3), + (file = "Llama-3.2-1B-Instruct.i1-IQ3_XXS.gguf", family = "llama", nmodes = 3)) // iq2s on the attention planes + let box_mode = g_gpu.coopmat_mode + let modes = fixed_array(box_mode, COOPMAT_MM, COOPMAT_SDOT4, COOPMAT_F16, COOPMAT_INT8) + for (v in vehicles) { + let vpath = path_join(models_dir(), v.file) + for (mi in range(v.nmodes)) { + g_gpu.coopmat_mode = modes[mi] + vk_cov_model(t, vpath, v.family, none) + } + g_gpu.coopmat_mode = box_mode + } // qwen3.5 q8: the DeltaNet chains (conv/step/scan + the decode step) vk_cov_model(t, path_join(models_dir(), "Qwen3.5-0.8B-Q8_0.gguf"), "qwen35", none) // qwen3.5 9B (large tier): head-256 gated attention on the cm2 flash stamps, the diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index aa2c92f4f4..97e2abb617 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -2697,6 +2697,120 @@ def private kq_gpu_gather_gate(t : T?; fmt : int) { t |> equal(sdiff, 0, "kq {fmt}: scale device bytes identical from disk-order and grp planes") } +//! the decoded sub-scale byte a synthetic plane row carries at (row, blk): iq4xs's ls - 32, the iq3 family's 1 + 2s +def private iq_plane_sc(fmt : KqFmt; r, blk : int64) : int { + return fmt == KqFmt.iq4xs ? int((r * 5l + blk * 7l) % 64l) - 32 : 1 + 2 * int((r * 3l + blk * 5l) % 16l) +} + +def private iq_plane_d(r : int64) : float => 0.25 + 0.5 * float(r) + +def private kq_gpu_gather_iq_gate(t : T?; fmt : KqFmt) { + let n = 512l + let d = 16l + let nsb = n / 256l + let mr = kq_layout_of(int(fmt)) + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + var kq : array + var ks : array + kq |> resize(d * nsb * qsb) + ks |> resize(d * nsb * ssb) + var st = 0xC0FFEEu + for (i in range64(d * nsb * qsb)) { + st = st * 1664525u + 1013904223u + kq[i] = uint8(st >> 24u) + } + for (r in range64(d)) { + for (sbi in range64(nsb)) { + let ro = (r * nsb + sbi) * ssb + let dh = f32_to_f16(iq_plane_d(r + sbi)) + ks[ro] = uint8(dh & 0xFFu) + ks[ro + 1l] = uint8((dh >> 8u) & 0xFFu) + for (blk in range64(8l)) { + ks[ro + 4l + blk] = uint8(iq_plane_sc(fmt, r + sbi, blk) & 0xFF) + } + } + } + var m = Model() + if (fmt == KqFmt.iq4xs) { + m.iq4xsq := kq + m.iq4xss := ks + } elif (fmt == KqFmt.iq3s) { + m.iq3sq := kq + m.iq3ss := ks + } elif (fmt == KqFmt.iq3xxs) { + m.iq3xxsq := kq + m.iq3xxss := ks + } else { + m.iq2xxsq := kq + m.iq2xxss := ks + } + var wqDisk : array + var wsDisk : array + moe_gpu_gather_stack_kq(m, fmt, 0l, n, d, d, false, mr, wqDisk, wsDisk) + // the device row is two words: word 0 = the f16 d with iq4xs's high scale bits above it, word 1 = eight 4-bit fields + var bad = 0 + for (r in range64(d)) { + for (sbi in range64(nsb)) { + let wo = (r * nsb + sbi) * 8l + let w0 = uint(wsDisk[wo]) | (uint(wsDisk[wo + 1l]) << 8u) | (uint(wsDisk[wo + 2l]) << 16u) | (uint(wsDisk[wo + 3l]) << 24u) + let w1 = uint(wsDisk[wo + 4l]) | (uint(wsDisk[wo + 5l]) << 8u) | (uint(wsDisk[wo + 6l]) << 16u) | (uint(wsDisk[wo + 7l]) << 24u) + if (f16_to_f32(w0 & 0xFFFFu) != iq_plane_d(r + sbi)) { + bad++ + } + for (blk in range64(8l)) { + let f = int((w1 >> uint(4l * blk)) & 15u) | (int((w0 >> uint(16l + 2l * blk)) & 3u) << 4) + let got = fmt == KqFmt.iq4xs ? f - 32 : 1 + 2 * f + if (got != iq_plane_sc(fmt, r + sbi, blk)) { + bad++ + } + } + } + } + t |> equal(bad, 0, "kq {fmt}: the two-word device scale rows decode back to the plane's d and sub-scales") + t |> equal(long_length(wqDisk), d * nsb * qsb, "kq {fmt}: the quant plane gathers whole") + if (fmt == KqFmt.iq4xs || fmt == KqFmt.iq3s) { + unsafe { + if (fmt == KqFmt.iq4xs) { + repack_iq4xs_grp(addr(m.iq4xsq[0]), addr(m.iq4xss[0]), n, d, mr) + } else { + repack_iq3s_grp(addr(m.iq3sq[0]), addr(m.iq3ss[0]), n, d, mr) + } + } + var wqGrp : array + var wsGrp : array + moe_gpu_gather_stack_kq(m, fmt, 0l, n, d, d, true, mr, wqGrp, wsGrp) + var qdiff = 0 + for (a, b in wqDisk, wqGrp) { + if (a != b) { + qdiff++ + } + } + var sdiff = 0 + for (a, b in wsDisk, wsGrp) { + if (a != b) { + sdiff++ + } + } + t |> equal(qdiff, 0, "kq {fmt}: quant device bytes identical from disk-order and grp planes") + t |> equal(sdiff, 0, "kq {fmt}: scale device bytes identical from disk-order and grp planes") + delete wqGrp + delete wsGrp + } + delete wqDisk + delete wsDisk + delete m +} + +[test] +def test_kq_gpu_gather_iq(t : T?) { + for (fmt in [KqFmt.iq4xs, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq2xxs]) { + t |> run("moe_gpu_gather_stack_kq {fmt}: the codebook format's two-word device scale rows") @(t : T?) { + kq_gpu_gather_iq_gate(t, fmt) + } + } +} + [test] def test_kq_gpu_gather_grp(t : T?) { for (fmt in [3, 6]) { diff --git a/modules/dasLLAMA/tests/test_tokenizer.das b/modules/dasLLAMA/tests/test_tokenizer.das index a83cc3c809..6c8c324bbe 100644 --- a/modules/dasLLAMA/tests/test_tokenizer.das +++ b/modules/dasLLAMA/tests/test_tokenizer.das @@ -106,6 +106,26 @@ def test_tokenizer_qwen35(t : T?) { corpus_case(t, "qwen35 BPE (mark-absorbing letter runs) vs the llama.cpp corpus", "ggml-vocab-qwen35.gguf") } +[test] +def test_tokenizer_bos_default(t : T?) { + t |> run("a BPE vocab without add_bos_token prepends BOS only for upstream's BOS families") @(t : T?) { + t |> success(bpe_pre_adds_bos("llama-bpe") && bpe_pre_adds_bos("tekken"), "the llama3 family and tekken default to a BOS") + t |> success(!bpe_pre_adds_bos("qwen35") && !bpe_pre_adds_bos("qwen2") && !bpe_pre_adds_bos("gpt-4o"), "qwen and gpt-4o default to none") + //! the upstream vocab fixture carries bos_token_id and no add_bos_token key - the unsloth Qwen3.8 shape + let path = path_join(models_dir(), "ggml-vocab-qwen35.gguf") + if (!stat(path).is_valid) { + t |> skip("{path} not present") + return + } + var tk <- load_tokenizer_auto(path) + t |> success(!tk.bpe.add_bos, "qwen35 without the key adds no BOS") + var ids <- encode(tk, "Once upon a time", true) + t |> equal(4, length(ids), "four BPE ids and no BOS in front: {ids}") + delete ids + delete tk + } +} + [test] def test_tokenizer_qwen2(t : T?) { corpus_case(t, "qwen2 byte-level BPE vs the llama.cpp corpus", "ggml-vocab-qwen2.gguf") diff --git a/modules/dasLLAMA/tests/test_vulkan_dec_tail.das b/modules/dasLLAMA/tests/test_vulkan_dec_tail.das index c1f43375d1..beb11b63f8 100644 --- a/modules/dasLLAMA/tests/test_vulkan_dec_tail.das +++ b/modules/dasLLAMA/tests/test_vulkan_dec_tail.das @@ -160,10 +160,16 @@ struct private Quad { wv : array wo : array norm : array // the layer's attention rms weight row + bq : array // the q/k/v projection bias rows (empty = the model has none) + bk : array + bv : array kmir : array // the reference mirror (f16-rounded rows at absolute positions) vmir : array } +//! a bias row's pointer for the block hook: null when the quad carries no bias +def private bias_ptr(b : array) : float const? => !empty(b) ? unsafe(addr(b[0])) : null + // the expert stack pair (dequantized for the reference), the FFN norm row, the routing struct private Ffn { w1 : array @@ -189,10 +195,15 @@ def private plane_up(fmt : int; rows, n, seed, woff : int64; var w : array; pos : int64; cossin : arra gemv_ref(q.wk, KVD, N, xd, kr) gemv_ref(q.wv, KVD, N, xd, vr) kraw := kr + if (!empty(q.bq)) { + for (i in range64(QD)) { + qr[i] += q.bq[i] + } + for (i in range64(KVD)) { + kr[i] += q.bk[i] + vr[i] += q.bv[i] + } + } rope_ref(qr, NH, cossin, q.neox) rope_ref(kr, 1l, cossin, q.neox) for (i in range64(KVD)) { @@ -317,8 +337,10 @@ def private token_cell(t : T?; var qa : Quad; var qb : Quad; f : Ffn; pos : int6 var xq : array var xs : array feed_ref(x, qa.norm, qa.fmt, xd, xq, xs) - matmul_moe_gpu_attn_dec(xb2, kout, vout, qa.woq, qa.wok, qa.wov, qa.woo, qa.fmt, qa.fmt, qa.fmt, qa.fmt, xq, xs, cossin, null, null, - pos, N, QD, KVD, HS, NH, KVM, ATTN_SCALE, EPS, qa.neox, false) + unsafe { + matmul_moe_gpu_attn_dec(xb2, kout, vout, qa.woq, qa.wok, qa.wov, qa.woo, qa.fmt, qa.fmt, qa.fmt, qa.fmt, xq, xs, cossin, null, null, + bias_ptr(qa.bq), bias_ptr(qa.bk), bias_ptr(qa.bv), pos, N, QD, KVD, HS, NH, KVM, ATTN_SCALE, EPS, qa.neox, false) + } var xb2_ref : array var kraw_ref : array block_ref(qa, x, pos, cossin, xb2_ref, kraw_ref) @@ -344,8 +366,10 @@ def private token_cell(t : T?; var qa : Quad; var qb : Quad; f : Ffn; pos : int6 let pair <- [0l, 1l] // the tail arm routes slot j to expert j ffn_ref_sel(f, fd, pair, f.rw, x1_ref) // layer B on the block: its feed came from the tail; the host acts are ignored - matmul_moe_gpu_attn_dec(xb2, kout, vout, qb.woq, qb.wok, qb.wov, qb.woo, qb.fmt, qb.fmt, qb.fmt, qb.fmt, xq, xs, cossin, null, null, - pos, N, QD, KVD, HS, NH, KVM, ATTN_SCALE, EPS, qb.neox, false) + unsafe { + matmul_moe_gpu_attn_dec(xb2, kout, vout, qb.woq, qb.wok, qb.wov, qb.woo, qb.fmt, qb.fmt, qb.fmt, qb.fmt, xq, xs, cossin, null, null, + bias_ptr(qb.bq), bias_ptr(qb.bk), bias_ptr(qb.bv), pos, N, QD, KVD, HS, NH, KVM, ATTN_SCALE, EPS, qb.neox, false) + } t |> success(!moe_gpu_ffn_tail_pending(), "{tag} pos {pos}: layer B's attention collected the tail") check_row(t, "{tag} pos {pos} residual row after FFN A (the tail's combine)", x, x1_ref) var xb2b_ref : array @@ -361,13 +385,13 @@ def private token_cell(t : T?; var qa : Quad; var qb : Quad; f : Ffn; pos : int6 } // one quad pair in one plane format through two positions; false = no device -def private pair_cell(t : T?; f : Ffn; fmt : int; base : int64; tag : string; neox : bool) : bool { +def private pair_cell(t : T?; f : Ffn; fmt : int; base : int64; tag : string; neox, hasb : bool) : bool { var qa : Quad var qb : Quad - if (!quad_up(qa, fmt, base, 21l + int64(fmt), 0.3, neox)) { + if (!quad_up(qa, fmt, base, 21l + int64(fmt), 0.3, neox, hasb)) { return false } - t |> success(quad_up(qb, fmt, base + 10l * STRIDE, 33l + int64(fmt), 0.7, neox), "{tag}: layer B's quad uploaded") + t |> success(quad_up(qb, fmt, base + 10l * STRIDE, 33l + int64(fmt), 0.7, neox, hasb), "{tag}: layer B's quad uploaded") let gen = moe_gpu_attn_dec_claim(true) // a layer the block has never seen hydrates first (zero rows at position 0 creates it), then serves t |> success(moe_gpu_attn_dec_ok(qa.woq, 0l, gen) == DAT_HYDRATE, "{tag}: an unseen layer asks for a hydrate at position 0") @@ -515,6 +539,9 @@ def private span_args(qa, qb : Quad; f : Ffn) : MoeSpanDec { f1 <- [1, 1], f3 <- [1, 1], f2 <- [3, 3]) a.rmsq |> resize(2) // no qk-norm: null rows a.rmsk |> resize(2) + a.bq |> resize(2) // no projection bias: null rows + a.bk |> resize(2) + a.bv |> resize(2) return <- a } @@ -554,8 +581,8 @@ def test_vulkan_dec_span(t : T?) { } var qa : Quad var qb : Quad - t |> success(quad_up(qa, 1, 60l * STRIDE, 51l, 0.4, true), "dec span: layer A's quad uploaded") - t |> success(quad_up(qb, 1, 70l * STRIDE, 63l, 0.8, true), "dec span: layer B's quad uploaded") + t |> success(quad_up(qa, 1, 60l * STRIDE, 51l, 0.4, true, false), "dec span: layer A's quad uploaded") + t |> success(quad_up(qb, 1, 70l * STRIDE, 63l, 0.8, true, false), "dec span: layer B's quad uploaded") let gen = moe_gpu_attn_dec_claim(true) matmul_moe_gpu_attn_dec_hydrate(qa.woq, null, null, 0l, KVD, true) matmul_moe_gpu_attn_dec_hydrate(qb.woq, null, null, 0l, KVD, true) @@ -634,8 +661,9 @@ def test_vulkan_dec_tail(t : T?) { t |> skip("dec tail: no Vulkan device (or probe/budget refused the upload)") return } - t |> success(pair_cell(t, f, 1, 10l * STRIDE, "k4 quads (Q8_K feed, neox rope)", true), "dec tail: the k4 pair uploaded") - t |> success(pair_cell(t, f, 0, 40l * STRIDE, "q8 quads (Q8_0 feed, classic rope)", false), "dec tail: the q8 pair uploaded") + t |> success(pair_cell(t, f, 1, 10l * STRIDE, "k4 quads (Q8_K feed, neox rope)", true, false), "dec tail: the k4 pair uploaded") + t |> success(pair_cell(t, f, 0, 40l * STRIDE, "q8 quads (Q8_0 feed, classic rope)", false, false), "dec tail: the q8 pair uploaded") + t |> success(pair_cell(t, f, 0, 80l * STRIDE, "q8 quads with the q/k/v projection bias (qwen2moe, neox rope)", true, true), "dec tail: the biased q8 pair uploaded") } else { t |> skip("dasVulkan not installed; the decode block and tail are untestable here") } diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index bd8976bc7c..c0f77aba4c 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -16,6 +16,7 @@ require ?vulkan dasllama/dasllama_vulkan_classes require ?vulkan dasllama/dasllama_vulkan_prefill require ?vulkan dasllama/dasllama_vulkan_seams require ?vulkan dasllama/dasllama_kqformat +require ?vulkan dasllama/dasllama_gemm_schema // kq_dev_ssb_i - the fixtures size their device scale planes per format require ?vulkan daslib/shader_lingua_franca // gl_GlobalInvocationID — the CPU replay drives it per lane (only vulkan-guarded arms replay) // The model-less Vulkan kernel-unit suite ([vk_dispatch] bring-up): a class kernel dispatched @@ -87,6 +88,86 @@ def test_vkd_class_dispatch(t0 : T?) { } } +[test, arch(at="../ARCHITECTURE_GPU_VULKAN_GEMM.md#cm2-tile-pick-and-default")] +def test_vkd_direct_decode(t0 : T?) { + t0 |> run("a [spirv_decode] method called directly from the kernel body == the same method on the CPU") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_vkd_direct_decode()) { + t |> skip("no Vulkan device") + return + } + let n = 64 //! two q8-shaped blocks of 32 quants + var k = new VkdDirectDecode() + k.wq |> resize(2) + k.ws <- [float16(0.5), float16(-0.25)] + k.y |> resize(n) + k.pa = VdArgs(n = uint(n)) + for (b in range(2)) { + for (i in range(16)) { + let q0 = (b * 32 + i * 2) % 23 - 11 + let q1 = (b * 32 + i * 2 + 1) % 23 - 11 + k.wq[b].qs[i] = int16((q0 & 0xFF) | ((q1 & 0xFF) << 8)) + } + } + for (i in range(n)) { + gl_GlobalInvocationID = uint3(uint(i), 0u, 0u) + k->run() + } + let wq_bytes = 2l * 32l + let ws_bytes = 2l * 2l + let y_bytes = int64(n) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let yd = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var gpu : array + gpu |> resize(n) + unsafe { + upload_region_at(wqd, 0l, addr(k.wq[0].qs[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(k.ws[0]), ws_bytes) + var s = set_vkd_direct_decode(fixed_array(wqd, wsd, yd), fixed_array(wq_bytes, ws_bytes, y_bytes), fixed_array(1u, 2u, 4u)) + var raw = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw, begin), null) + var h : VkHaz + var pc = VdArgs(n = uint(n)) + enc_vkd_direct_decode(raw, h, s, pc, 1l) + vhz_dep(raw, h, 4u, 0u, true) + cmd_copy_whole(raw, yd, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw), null) + submit_wait(raw) + memcpy(addr(gpu[0]), host.mapped, y_bytes) + } + var bad = 0 + var nonzero = 0 + for (i in range(n)) { + if (gpu[i] != k.y[i]) { + bad++ + } + if (k.y[i] != 0.0) { + nonzero++ + } + } + t |> success(nonzero > 48, "the oracle is non-trivial ({nonzero} of {n} lanes non-zero)") + t |> success(bad == 0, "GPU == CPU on all {n} lanes through the direct decode call ({bad} diverge)") + var poisoned = 0 // the control: one oracle lane pushed off must red the same compare + gpu[3]++ + for (i in range(n)) { + if (gpu[i] != k.y[i]) { + poisoned++ + } + } + t |> success(poisoned == 1, "the exact compare reds a poisoned lane ({poisoned} diverge)") + delete gpu + unsafe { + delete k + } + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_ar_class(t0 : T?) { t0 |> run("ar_add_rms class twin == the vk_add_rms seam (bit-exact, same reduce order)") <| @(t : T?) { @@ -633,7 +714,7 @@ def test_vkd_kq_gemv_family(t0 : T?) { let stride4 = stride4s[fi] let nwsb = rows * nsb // region r's weights at r*d*nsb let wq_words = nwsb * stride4 - let ws_words_n = nwsb * 5 + let ws_words_n = nwsb * int(kq_dev_ssb_i(fmt) / 4l) var wqh : array var wsh : array var xqh : array @@ -681,11 +762,10 @@ def test_vkd_kq_gemv_family(t0 : T?) { upload_region_at(xqd, 0l, addr(xqh[0]), xq_bytes) upload_region_at(xsd, 0l, addr(xsh[0]), xs_bytes) upload_region_at(rgd, 0l, addr(regs[0]), int64(nreg) * 8l) - // the class kernel — regions SSBO + push params (one family set serves all four) let cb = fixed_array(wqd, wsd, rgd, xqd, xsd, yd2) let cs = fixed_array(wq_bytes, ws_bytes, int64(nreg) * 8l, xq_bytes, xs_bytes, y_bytes) let cg = fixed_array(1u, 2u, 4u, 8u, 16u, 32u) - var s_cls = set_kq_gemv_cls(cb, cs, cg) + var s_cls = gemv_cls_set(fmt, cb, cs, cg) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -975,12 +1055,20 @@ def test_vkd_rope_family(t0 : T?) { csh[i] = 0.3 + float(i % 9) * 0.07 // cos row csh[half + i] = 0.1 + float(i % 7) * 0.11 // sin row } + let bpad = 64 // the bias rows sit past a pad, so boff is exercised off its default + var bh : array + bh |> resize(bpad + qd + 2 * kvd) + for (i in range(bpad + qd + 2 * kvd)) { + bh[i] = i < bpad ? 55.0 : float((i * 13) % 29) * 0.05 - 0.7 + } + let b_bytes = long_length(bh) * 4l let q_bytes = int64(qd) * 4l let kv_bytes = int64(qd + 2 * kvd) * 4l let m_bytes = int64(kvd) * 4l let cs_bytes = int64(hs) * 4l let kvd_dev = make_device_buf(kv_bytes) let csd = make_device_buf(cs_bytes) + let bd = make_device_buf(b_bytes) let q2 = make_device_buf(q_bytes) let km2 = make_device_buf(m_bytes) let vm2 = make_device_buf(m_bytes) @@ -996,18 +1084,19 @@ def test_vkd_rope_family(t0 : T?) { upload_region_at(kvd_dev, 0l, addr(kvh[0]), kv_bytes) upload_region_at(csd, 0l, addr(csh[0]), cs_bytes) upload_region_at(q2, 0l, addr(qh[0]), q_bytes) - // the class kernel: q@0, kvsrc@1, kmir@2, vmir@3, cossin@4, tok@5 (pos 0 -> row 0) + upload_region_at(bd, 0l, addr(bh[0]), b_bytes) + //! the class kernel: q@0, kvsrc@1, kmir@2, vmir@3, cossin@4, tok@5 (pos 0 -> row 0), bias@6 var tokb = make_host_buf(16l, true) var tm = reinterpret(tokb.mapped) tm.pos = 0u - var sc = set_rope_kv_cls(fixed_array(q2, kvd_dev, km2, vm2, csd, tokb.buf), - fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, 16l), - fixed_array(16u, 1u, 2u, 4u, 8u, 0u)) + var sc = set_rope_kv_cls(fixed_array(q2, kvd_dev, km2, vm2, csd, tokb.buf, bd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, 16l, b_bytes), + fixed_array(16u, 1u, 2u, 4u, 8u, 0u, 0u)) var raw2 = alloc_cmd() vk_check(vkBeginCommandBuffer(raw2, begin), null) var h2 : VkHaz var pc = RopeKvArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(half), - neox = 1u, kpair0 = uint(kpair0), npairs = uint(npairs), layerbase = 0u) + neox = 1u, kpair0 = uint(kpair0), npairs = uint(npairs), layerbase = 0u, boff = uint(bpad), hasb = 1u) enc_rope_kv_cls(raw2, h2, sc, pc, wgs) vhz_dep(raw2, h2, 2u | 4u | 16u, 0u, true) cmd_copy_whole(raw2, q2, host.buf, q_bytes) @@ -1027,8 +1116,7 @@ def test_vkd_rope_family(t0 : T?) { submit_wait(raw2c) memcpy(addr(cls_v[0]), host.mapped, m_bytes) } - // oracle: the same per-pair rotation sequentially (NEOX pairs (j, j+half)); - // v copies raw — approx bars ride out the device's fma contraction + //! oracle: bias first, then the sequential NEOX (j, j+half) rotation; v is raw plus bias — the approx bars ride out the device's fma contraction var ref_q : array var ref_k : array var ref_v : array @@ -1036,7 +1124,7 @@ def test_vkd_rope_family(t0 : T?) { ref_k |> resize(kvd) ref_v |> resize(kvd) for (i in range(qd)) { - ref_q[i] = qh[i] + ref_q[i] = qh[i] + bh[bpad + i] } for (gid in range(npairs)) { let is_k = gid >= kpair0 @@ -1046,24 +1134,25 @@ def test_vkd_rope_family(t0 : T?) { let fcr = csh[j] let fci = csh[half + j] if (is_k) { - let a0 = kvh[qd + hh * hs + j] - let a1 = kvh[qd + hh * hs + j + half] + let a0 = kvh[qd + hh * hs + j] + bh[bpad + qd + hh * hs + j] + let a1 = kvh[qd + hh * hs + j + half] + bh[bpad + qd + hh * hs + j + half] ref_k[hh * hs + j] = a0 * fcr - a1 * fci ref_k[hh * hs + j + half] = a0 * fci + a1 * fcr - ref_v[hh * hs + j] = kvh[qd + kvd + hh * hs + j] - ref_v[hh * hs + j + half] = kvh[qd + kvd + hh * hs + j + half] + ref_v[hh * hs + j] = kvh[qd + kvd + hh * hs + j] + bh[bpad + qd + kvd + hh * hs + j] + ref_v[hh * hs + j + half] = kvh[qd + kvd + hh * hs + j + half] + bh[bpad + qd + kvd + hh * hs + j + half] } else { - let a0 = qh[hh * hs + j] - let a1 = qh[hh * hs + j + half] + let a0 = qh[hh * hs + j] + bh[bpad + hh * hs + j] + let a1 = qh[hh * hs + j + half] + bh[bpad + hh * hs + j + half] ref_q[hh * hs + j] = a0 * fcr - a1 * fci ref_q[hh * hs + j + half] = a0 * fci + a1 * fcr } } let bad = mismatch_approx(cls_q, ref_q) + mismatch_approx(cls_k, ref_k) + mismatch_approx(cls_v, ref_v) - t |> success(bad == 0, "rope_kv_store matches the CPU oracle ({bad} elements off)") + t |> success(bad == 0, "rope_kv_store with the q/k/v bias matches the CPU oracle ({bad} elements off)") delete qh delete kvh delete csh + delete bh delete ref_q delete ref_k delete ref_v @@ -1314,14 +1403,15 @@ def test_vkd_kq_batch_family(t0 : T?) { var wsh : array var xqh : array var xsh : array + let wswords = totsb * int(kq_dev_ssb_i(fmt) / 4l) wqh |> resize(totsb * wstride) - wsh |> resize(totsb * 5) + wsh |> resize(wswords) xqh |> resize(rows * nsb * 64) xsh |> resize(rows * nsb) for (i in range(totsb * wstride)) { wqh[i] = hash_word(uint(i) + uint(fmt) * 131u) } - for (i in range(totsb * 5)) { + for (i in range(wswords)) { wsh[i] = ws_word(uint(i) + uint(fmt) * 17u) } for (i in range(rows * nsb * 64)) { @@ -1331,7 +1421,7 @@ def test_vkd_kq_batch_family(t0 : T?) { xsh[i] = 0.3 + float(i % 11) * 0.05 } let wq_bytes = int64(totsb * wstride) * 4l - let ws_bytes = int64(totsb * 5) * 4l + let ws_bytes = int64(wswords) * 4l let xq_bytes = int64(rows * nsb) * 256l let xs_bytes = int64(rows * nsb) * 4l let y_bytes = int64(rows * d) * 4l @@ -1605,7 +1695,7 @@ def test_vkd_kq_q40_cm_batch(t0 : T?) { feint("no Vulkan device - skipping\n") return } - if (g_gpu.coopmat_mode != 1) { + if (g_gpu.coopmat_mode != COOPMAT_F16) { feint("q40 rides the f16 coopmat twin only in mode 1 (DASLLAMA_COOPMAT=f16) - skipping\n") return } @@ -1719,7 +1809,7 @@ def test_vkd_cm2l_batch(t0 : T?) { feint("no Vulkan device - skipping\n") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + if (!(g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2)) { feint("the cm2 l-tile serves only in mode 4 on an NV_coopmat2 device (DASLLAMA_COOPMAT=cm2) - skipping\n") return } @@ -1824,7 +1914,7 @@ def test_vkd_cm2_decode_vector_pair(t0 : T?) { t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + if (!(g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2)) { t |> skip("the cm2 l-tile serves only in mode 4 on an NV_coopmat2 device (DASLLAMA_COOPMAT=cm2)") return } @@ -1944,17 +2034,24 @@ def test_vkd_cm2_decode_vector_pair(t0 : T?) { [test] def test_vkd_k4_cm2_batch(t0 : T?) { - t0 |> run("cm2 Q4_K l/m tiles == the CPU oracle (decode-in-load over the repacked planes)") <| @(t : T?) { + t0 |> run("cm2 Q4_K l/m/s tiles and the KHR tile == the CPU oracle (decode-in-load over the repacked planes; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 k4 tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the k4 tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_k4_cm2l_cls() && ensure_kq_batch_k4_cm2m_cls() && ensure_kq_batch_k4_cm2s_cls(), "k4 cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_k4_cm2l_cls() && ensure_kq_batch_k4_cm2m_cls() && ensure_kq_batch_k4_cm2s_cls(), "k4 cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_k4_khr_cls(), "the k4 KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -1995,8 +2092,12 @@ def test_vkd_k4_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -2018,20 +2119,17 @@ def test_vkd_k4_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_k4_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_k4_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_k4_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_k4_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_k4_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_k4_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.k4), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -2041,8 +2139,10 @@ def test_vkd_k4_cm2_batch(t0 : T?) { enc_kq_batch_k4_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_k4_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_k4_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.k4), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -2052,8 +2152,8 @@ def test_vkd_k4_cm2_batch(t0 : T?) { } k4f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 k4 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 k4 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "k4 {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "k4 {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k4 bar reds a poisoned element") @@ -2073,17 +2173,24 @@ def test_vkd_k4_cm2_batch(t0 : T?) { [test] def test_vkd_k5_cm2_batch(t0 : T?) { - t0 |> run("cm2 Q5_K l/m/s tiles == the CPU oracle (5th-bit compose decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 Q5_K l/m/s tiles and the KHR tile == the CPU oracle (5th-bit compose decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 k5 tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the k5 tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_k5_cm2l_cls() && ensure_kq_batch_k5_cm2m_cls() && ensure_kq_batch_k5_cm2s_cls(), "k5 cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_k5_cm2l_cls() && ensure_kq_batch_k5_cm2m_cls() && ensure_kq_batch_k5_cm2s_cls(), "k5 cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_k5_khr_cls(), "the k5 KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -2124,8 +2231,12 @@ def test_vkd_k5_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -2147,20 +2258,17 @@ def test_vkd_k5_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_k5_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_k5_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_k5_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_k5_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_k5_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_k5_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.k5), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -2170,8 +2278,10 @@ def test_vkd_k5_cm2_batch(t0 : T?) { enc_kq_batch_k5_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_k5_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_k5_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.k5), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -2181,8 +2291,8 @@ def test_vkd_k5_cm2_batch(t0 : T?) { } k5f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 k5 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 k5 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "k5 {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "k5 {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k5 bar reds a poisoned element") @@ -2202,17 +2312,24 @@ def test_vkd_k5_cm2_batch(t0 : T?) { [test] def test_vkd_q40_cm2_batch(t0 : T?) { - t0 |> run("cm2 Q4_0 l/m/s tiles == the CPU oracle (per-block f16 d decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 Q4_0 l/m/s tiles and the KHR tile == the CPU oracle (per-block f16 d decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 q40 tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the q40 tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_q40_cm2l_cls() && ensure_kq_batch_q40_cm2m_cls() && ensure_kq_batch_q40_cm2s_cls(), "q40 cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_q40_cm2l_cls() && ensure_kq_batch_q40_cm2m_cls() && ensure_kq_batch_q40_cm2s_cls(), "q40 cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_q40_khr_cls(), "the q40 KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -2252,8 +2369,12 @@ def test_vkd_q40_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -2275,20 +2396,17 @@ def test_vkd_q40_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_q40_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_q40_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_q40_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_q40_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_q40_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_q40_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.q40), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -2298,8 +2416,10 @@ def test_vkd_q40_cm2_batch(t0 : T?) { enc_kq_batch_q40_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_q40_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_q40_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.q40), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -2309,8 +2429,8 @@ def test_vkd_q40_cm2_batch(t0 : T?) { } q40f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 q40 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 q40 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "q40 {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "q40 {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the q40 bar reds a poisoned element") @@ -2330,17 +2450,24 @@ def test_vkd_q40_cm2_batch(t0 : T?) { [test] def test_vkd_k2_cm2_batch(t0 : T?) { - t0 |> run("cm2 Q2_K l/m/s tiles == the CPU oracle (pair-byte nibble scales decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 Q2_K l/m/s tiles and the KHR tile == the CPU oracle (pair-byte nibble scales decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 k2 tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the k2 tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_k2_cm2l_cls() && ensure_kq_batch_k2_cm2m_cls() && ensure_kq_batch_k2_cm2s_cls(), "k2 cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_k2_cm2l_cls() && ensure_kq_batch_k2_cm2m_cls() && ensure_kq_batch_k2_cm2s_cls(), "k2 cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_k2_khr_cls(), "the k2 KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -2379,8 +2506,12 @@ def test_vkd_k2_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -2402,20 +2533,17 @@ def test_vkd_k2_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_k2_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_k2_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_k2_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_k2_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_k2_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_k2_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.k2), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -2425,8 +2553,10 @@ def test_vkd_k2_cm2_batch(t0 : T?) { enc_kq_batch_k2_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_k2_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_k2_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.k2), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -2436,8 +2566,8 @@ def test_vkd_k2_cm2_batch(t0 : T?) { } k2f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 k2 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 k2 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "k2 {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "k2 {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k2 bar reds a poisoned element") @@ -2457,17 +2587,24 @@ def test_vkd_k2_cm2_batch(t0 : T?) { [test] def test_vkd_iq4nl_cm2_batch(t0 : T?) { - t0 |> run("cm2 IQ4_NL l/m/s tiles == the CPU oracle (codebook + per-block f16 d decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 IQ4_NL l/m/s tiles and the KHR tile == the CPU oracle (codebook + per-block f16 d decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 iq4nl tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the iq4nl tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_iq4nl_cm2l_cls() && ensure_kq_batch_iq4nl_cm2m_cls() && ensure_kq_batch_iq4nl_cm2s_cls(), "iq4nl cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_iq4nl_cm2l_cls() && ensure_kq_batch_iq4nl_cm2m_cls() && ensure_kq_batch_iq4nl_cm2s_cls(), "iq4nl cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_iq4nl_khr_cls(), "the iq4nl KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -2507,8 +2644,12 @@ def test_vkd_iq4nl_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -2530,20 +2671,17 @@ def test_vkd_iq4nl_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_iq4nl_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_iq4nl_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_iq4nl_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_iq4nl_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_iq4nl_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_iq4nl_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.iq4nl), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -2553,8 +2691,10 @@ def test_vkd_iq4nl_cm2_batch(t0 : T?) { enc_kq_batch_iq4nl_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_iq4nl_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_iq4nl_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.iq4nl), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -2564,8 +2704,8 @@ def test_vkd_iq4nl_cm2_batch(t0 : T?) { } iq4nlf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 iq4nl {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 iq4nl {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "iq4nl {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "iq4nl {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq4nl bar reds a poisoned element") @@ -2585,17 +2725,24 @@ def test_vkd_iq4nl_cm2_batch(t0 : T?) { [test] def test_vkd_iq4xs_cm2_batch(t0 : T?) { - t0 |> run("cm2 IQ4_XS l/m/s tiles == the CPU oracle (workgroup-staged codebook decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 IQ4_XS l/m/s tiles and the KHR tile == the CPU oracle (workgroup-staged codebook decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 iq4xs tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the iq4xs tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_iq4xs_cm2l_cls() && ensure_kq_batch_iq4xs_cm2m_cls() && ensure_kq_batch_iq4xs_cm2s_cls(), "iq4xs cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_iq4xs_cm2l_cls() && ensure_kq_batch_iq4xs_cm2m_cls() && ensure_kq_batch_iq4xs_cm2s_cls(), "iq4xs cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_iq4xs_khr_cls(), "the iq4xs KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -2607,17 +2754,14 @@ def test_vkd_iq4xs_cm2_batch(t0 : T?) { var wsuh : array var xfh : array wqh |> resize(totsb * 32) - wsuh |> resize(totsb * 5) + wsuh |> resize(totsb * 2) xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack for (i in range(totsb * 32)) { wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes } for (sb in range(totsb)) { - wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d - wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes - wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) - wsuh[sb * 5 + 3] = 0u - wsuh[sb * 5 + 4] = 0u + wsuh[sb * 2] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) | (hash_word(uint(sb * 2) + 613u) << 16u) //! cool d | the row's high bits + wsuh[sb * 2 + 1] = hash_word(uint(sb * 2 + 1) + 613u) //! the eight 4-bit sub-scale fields } for (i in range((rows + 32) * n / 2)) { xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs @@ -2635,8 +2779,12 @@ def test_vkd_iq4xs_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -2658,20 +2806,17 @@ def test_vkd_iq4xs_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_iq4xs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_iq4xs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_iq4xs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_iq4xs_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_iq4xs_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_iq4xs_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.iq4xs), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -2681,8 +2826,10 @@ def test_vkd_iq4xs_cm2_batch(t0 : T?) { enc_kq_batch_iq4xs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_iq4xs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_iq4xs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.iq4xs), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -2692,8 +2839,8 @@ def test_vkd_iq4xs_cm2_batch(t0 : T?) { } iq4xsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 iq4xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 iq4xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "iq4xs {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "iq4xs {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq4xs bar reds a poisoned element") @@ -2713,17 +2860,24 @@ def test_vkd_iq4xs_cm2_batch(t0 : T?) { [test] def test_vkd_k3_cm2_batch(t0 : T?) { - t0 |> run("cm2 Q3_K l/m/s tiles == the CPU oracle (2-bit + hmask compose decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 Q3_K l/m/s tiles and the KHR tile == the CPU oracle (2-bit + hmask compose decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 k3 tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the k3 tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_k3_cm2l_cls() && ensure_kq_batch_k3_cm2m_cls() && ensure_kq_batch_k3_cm2s_cls(), "k3 cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_k3_cm2l_cls() && ensure_kq_batch_k3_cm2m_cls() && ensure_kq_batch_k3_cm2s_cls(), "k3 cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_k3_khr_cls(), "the k3 KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -2762,8 +2916,12 @@ def test_vkd_k3_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -2785,20 +2943,17 @@ def test_vkd_k3_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_k3_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_k3_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_k3_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_k3_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_k3_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_k3_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.k3), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -2808,8 +2963,10 @@ def test_vkd_k3_cm2_batch(t0 : T?) { enc_kq_batch_k3_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_k3_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_k3_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.k3), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -2819,8 +2976,8 @@ def test_vkd_k3_cm2_batch(t0 : T?) { } k3f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 k3 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 k3 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "k3 {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "k3 {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k3 bar reds a poisoned element") @@ -2840,17 +2997,24 @@ def test_vkd_k3_cm2_batch(t0 : T?) { [test] def test_vkd_iq3s_cm2_batch(t0 : T?) { - t0 |> run("cm2 IQ3_S l/m/s tiles == the CPU oracle (workgroup-staged grid decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 IQ3_S l/m/s tiles and the KHR tile == the CPU oracle (workgroup-staged grid decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 iq3s tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the iq3s tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_iq3s_cm2l_cls() && ensure_kq_batch_iq3s_cm2m_cls() && ensure_kq_batch_iq3s_cm2s_cls(), "iq3s cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_iq3s_cm2l_cls() && ensure_kq_batch_iq3s_cm2m_cls() && ensure_kq_batch_iq3s_cm2s_cls(), "iq3s cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_iq3s_khr_cls(), "the iq3s KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -2862,17 +3026,14 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { var wsuh : array var xfh : array wqh |> resize(totsb * 26) - wsuh |> resize(totsb * 5) + wsuh |> resize(totsb * 2) xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack for (i in range(totsb * 26)) { wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes } for (sb in range(totsb)) { - wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d - wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes - wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) - wsuh[sb * 5 + 3] = 0u - wsuh[sb * 5 + 4] = 0u + wsuh[sb * 2] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) | (hash_word(uint(sb * 2) + 613u) << 16u) //! cool d | the row's high bits + wsuh[sb * 2 + 1] = hash_word(uint(sb * 2 + 1) + 613u) //! the eight 4-bit sub-scale fields } for (i in range((rows + 32) * n / 2)) { xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs @@ -2890,8 +3051,12 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -2913,20 +3078,17 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_iq3s_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_iq3s_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_iq3s_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_iq3s_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_iq3s_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_iq3s_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.iq3s), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -2936,8 +3098,10 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { enc_kq_batch_iq3s_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_iq3s_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_iq3s_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.iq3s), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -2947,8 +3111,8 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { } iq3sf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 iq3s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 iq3s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "iq3s {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "iq3s {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq3s bar reds a poisoned element") @@ -2968,17 +3132,24 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { [test] def test_vkd_iq2s_cm2_batch(t0 : T?) { - t0 |> run("cm2 IQ2_S l/m/s tiles == the CPU oracle (workgroup-staged u64 grid decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 IQ2_S l/m/s tiles and the KHR tile == the CPU oracle (workgroup-staged u64 grid decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 iq2s tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the iq2s tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_iq2s_cm2l_cls() && ensure_kq_batch_iq2s_cm2m_cls() && ensure_kq_batch_iq2s_cm2s_cls(), "iq2s cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_iq2s_cm2l_cls() && ensure_kq_batch_iq2s_cm2m_cls() && ensure_kq_batch_iq2s_cm2s_cls(), "iq2s cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_iq2s_khr_cls(), "the iq2s KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -3018,8 +3189,12 @@ def test_vkd_iq2s_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -3041,20 +3216,17 @@ def test_vkd_iq2s_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_iq2s_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_iq2s_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_iq2s_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_iq2s_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_iq2s_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_iq2s_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.iq2s), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -3064,8 +3236,10 @@ def test_vkd_iq2s_cm2_batch(t0 : T?) { enc_kq_batch_iq2s_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_iq2s_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_iq2s_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.iq2s), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -3075,8 +3249,8 @@ def test_vkd_iq2s_cm2_batch(t0 : T?) { } iq2sf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 iq2s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 iq2s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "iq2s {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "iq2s {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2s bar reds a poisoned element") @@ -3096,17 +3270,24 @@ def test_vkd_iq2s_cm2_batch(t0 : T?) { [test] def test_vkd_iq2xs_cm2_batch(t0 : T?) { - t0 |> run("cm2 IQ2_XS l/m/s tiles == the CPU oracle (u64 grid + ksigns-by-parity decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 IQ2_XS l/m/s tiles and the KHR tile == the CPU oracle (u64 grid + ksigns-by-parity decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 iq2xs tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the iq2xs tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_iq2xs_cm2l_cls() && ensure_kq_batch_iq2xs_cm2m_cls() && ensure_kq_batch_iq2xs_cm2s_cls(), "iq2xs cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_iq2xs_cm2l_cls() && ensure_kq_batch_iq2xs_cm2m_cls() && ensure_kq_batch_iq2xs_cm2s_cls(), "iq2xs cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_iq2xs_khr_cls(), "the iq2xs KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -3146,8 +3327,12 @@ def test_vkd_iq2xs_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -3169,20 +3354,17 @@ def test_vkd_iq2xs_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_iq2xs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_iq2xs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_iq2xs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_iq2xs_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_iq2xs_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_iq2xs_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.iq2xs), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -3192,8 +3374,10 @@ def test_vkd_iq2xs_cm2_batch(t0 : T?) { enc_kq_batch_iq2xs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_iq2xs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_iq2xs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.iq2xs), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -3203,8 +3387,8 @@ def test_vkd_iq2xs_cm2_batch(t0 : T?) { } iq2xsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 iq2xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 iq2xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "iq2xs {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "iq2xs {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2xs bar reds a poisoned element") @@ -3224,17 +3408,24 @@ def test_vkd_iq2xs_cm2_batch(t0 : T?) { [test] def test_vkd_iq2xxs_cm2_batch(t0 : T?) { - t0 |> run("cm2 IQ2_XXS l/m/s tiles == the CPU oracle (byte-grid + aux32 ksigns decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 IQ2_XXS l/m/s tiles and the KHR tile == the CPU oracle (byte-grid + aux32 ksigns decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 iq2xxs tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the iq2xxs tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_iq2xxs_cm2l_cls() && ensure_kq_batch_iq2xxs_cm2m_cls() && ensure_kq_batch_iq2xxs_cm2s_cls(), "iq2xxs cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_iq2xxs_cm2l_cls() && ensure_kq_batch_iq2xxs_cm2m_cls() && ensure_kq_batch_iq2xxs_cm2s_cls(), "iq2xxs cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_iq2xxs_khr_cls(), "the iq2xxs KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -3246,17 +3437,14 @@ def test_vkd_iq2xxs_cm2_batch(t0 : T?) { var wsuh : array var xfh : array wqh |> resize(totsb * 16) - wsuh |> resize(totsb * 5) + wsuh |> resize(totsb * 2) xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack for (i in range(totsb * 16)) { wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes } for (sb in range(totsb)) { - wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d8 - wsuh[sb * 5 + 1] = hash_word(uint(sb * 4) + 613u) // 16 unsigned strip bytes - wsuh[sb * 5 + 2] = hash_word(uint(sb * 4 + 1) + 613u) - wsuh[sb * 5 + 3] = hash_word(uint(sb * 4 + 2) + 613u) - wsuh[sb * 5 + 4] = hash_word(uint(sb * 4 + 3) + 613u) + wsuh[sb * 2] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) | (hash_word(uint(sb * 2) + 613u) << 16u) //! cool d8 | the row's high bits + wsuh[sb * 2 + 1] = hash_word(uint(sb * 2 + 1) + 613u) //! the eight 4-bit (2ls + 1) fields } for (i in range((rows + 32) * n / 2)) { xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs @@ -3274,8 +3462,12 @@ def test_vkd_iq2xxs_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -3297,20 +3489,17 @@ def test_vkd_iq2xxs_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_iq2xxs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_iq2xxs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_iq2xxs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_iq2xxs_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_iq2xxs_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_iq2xxs_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.iq2xxs), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -3320,8 +3509,10 @@ def test_vkd_iq2xxs_cm2_batch(t0 : T?) { enc_kq_batch_iq2xxs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_iq2xxs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_iq2xxs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.iq2xxs), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -3331,8 +3522,8 @@ def test_vkd_iq2xxs_cm2_batch(t0 : T?) { } iq2xxsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 iq2xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 iq2xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "iq2xxs {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "iq2xxs {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2xxs bar reds a poisoned element") @@ -3352,17 +3543,24 @@ def test_vkd_iq2xxs_cm2_batch(t0 : T?) { [test] def test_vkd_iq3xxs_cm2_batch(t0 : T?) { - t0 |> run("cm2 IQ3_XXS l/m/s tiles == the CPU oracle (halved grid + parity signs decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 IQ3_XXS l/m/s tiles and the KHR tile == the CPU oracle (halved grid + parity signs decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 iq3xxs tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the iq3xxs tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_iq3xxs_cm2l_cls() && ensure_kq_batch_iq3xxs_cm2m_cls() && ensure_kq_batch_iq3xxs_cm2s_cls(), "iq3xxs cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_iq3xxs_cm2l_cls() && ensure_kq_batch_iq3xxs_cm2m_cls() && ensure_kq_batch_iq3xxs_cm2s_cls(), "iq3xxs cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_iq3xxs_khr_cls(), "the iq3xxs KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -3374,17 +3572,14 @@ def test_vkd_iq3xxs_cm2_batch(t0 : T?) { var wsuh : array var xfh : array wqh |> resize(totsb * 24) - wsuh |> resize(totsb * 5) + wsuh |> resize(totsb * 2) xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack for (i in range(totsb * 24)) { wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes } for (sb in range(totsb)) { - wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d - wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes - wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) - wsuh[sb * 5 + 3] = 0u - wsuh[sb * 5 + 4] = 0u + wsuh[sb * 2] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) | (hash_word(uint(sb * 2) + 613u) << 16u) //! cool d | the row's high bits + wsuh[sb * 2 + 1] = hash_word(uint(sb * 2 + 1) + 613u) //! the eight 4-bit sub-scale fields } for (i in range((rows + 32) * n / 2)) { xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs @@ -3402,8 +3597,12 @@ def test_vkd_iq3xxs_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -3425,20 +3624,17 @@ def test_vkd_iq3xxs_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_iq3xxs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_iq3xxs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_iq3xxs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_iq3xxs_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_iq3xxs_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_iq3xxs_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.iq3xxs), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -3448,8 +3644,10 @@ def test_vkd_iq3xxs_cm2_batch(t0 : T?) { enc_kq_batch_iq3xxs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_iq3xxs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_iq3xxs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.iq3xxs), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -3459,8 +3657,8 @@ def test_vkd_iq3xxs_cm2_batch(t0 : T?) { } iq3xxsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 iq3xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 iq3xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "iq3xxs {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "iq3xxs {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq3xxs bar reds a poisoned element") @@ -3480,17 +3678,24 @@ def test_vkd_iq3xxs_cm2_batch(t0 : T?) { [test] def test_vkd_k6_cm2_batch(t0 : T?) { - t0 |> run("cm2 Q6_K l/m tiles == the CPU oracle (6-bit compose decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 Q6_K l/m/s tiles and the KHR tile == the CPU oracle (6-bit compose decode-in-load; the KHR arm stages the same decode)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { - t |> skip("the cm2 k6 tiles serve only in mode 4 on an NV_coopmat2 device") + let cm2_on = g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2 + let khr_on = khr_kq_tile_on() + if (!cm2_on && !khr_on) { + t |> skip("the k6 tiles need cooperative matrices (cm2 in mode 4 on an NV_coopmat2 device, or KHR coopmat at subgroup 32)") return } - verify(ensure_kq_batch_k6_cm2l_cls() && ensure_kq_batch_k6_cm2m_cls() && ensure_kq_batch_k6_cm2s_cls(), "k6 cm2 class rails must engage in cm2 mode") + if (cm2_on) { + verify(ensure_kq_batch_k6_cm2l_cls() && ensure_kq_batch_k6_cm2m_cls() && ensure_kq_batch_k6_cm2s_cls(), "k6 cm2 class rails must engage in cm2 mode") + } + if (khr_on) { + verify(ensure_kq_batch_k6_khr_cls(), "the k6 KHR class rail must engage on a coopmat device") + } let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds let nsb = n / 256 let d = 160 // one full 128-tile + a 32 edge @@ -3529,8 +3734,12 @@ def test_vkd_k6_cm2_batch(t0 : T?) { y_cls |> resize(rows * d) var y_ref : array y_ref |> resize(rows * d) - for (ml in range(3)) { - let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + for (ml in range(4)) { + if ((ml < 3 && !cm2_on) || (ml == 3 && !khr_on)) { + continue + } + let tname = ml == 0 ? "l" : (ml == 1 ? "m" : (ml == 2 ? "s" : "khr")) + let ttile = ml == 0 ? 256 : (ml == 2 ? 32 : 128) let wtiles = (d + 127) / 128 let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles @@ -3552,20 +3761,17 @@ def test_vkd_k6_cm2_batch(t0 : T?) { upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + nan_sentinel_upload(yd2, y_bytes) let sc_bytes = int64(2 * 4 + wgs) * 4l let scd = make_device_buf(sc_bytes) upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) - var sc = (ml == 0 - ? set_kq_batch_k6_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : (ml == 1 - ? set_kq_batch_k6_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)) - : set_kq_batch_k6_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), - fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), - fixed_array(1u, 2u, 0u, 4u, 16u)))) + let bufs = fixed_array(wqd, wsd, scd, xfd, yd2) + let sizes = fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes) + let gbits = fixed_array(1u, 2u, 0u, 4u, 16u) + var sc = (ml == 0 ? set_kq_batch_k6_cm2l_cls(bufs, sizes, gbits) + : (ml == 1 ? set_kq_batch_k6_cm2m_cls(bufs, sizes, gbits) + : (ml == 2 ? set_kq_batch_k6_cm2s_cls(bufs, sizes, gbits) + : khr_cls_set(int(KqFmt.k6), bufs, sizes, gbits)))) var raw2 = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw2, begin), null) @@ -3575,8 +3781,10 @@ def test_vkd_k6_cm2_batch(t0 : T?) { enc_kq_batch_k6_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) } elif (ml == 1) { enc_kq_batch_k6_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) - } else { + } elif (ml == 2) { enc_kq_batch_k6_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + khr_cls_enc(int(KqFmt.k6), raw2, h2, sc, pc, int64(wgs)) } vhz_dep(raw2, h2, 16u, 0u, true) cmd_copy_whole(raw2, yd2, host.buf, y_bytes) @@ -3586,8 +3794,8 @@ def test_vkd_k6_cm2_batch(t0 : T?) { } k6f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) - to_log(LOG_INFO, "cm2 k6 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") - t |> success(bad == 0, "cm2 k6 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + to_log(LOG_INFO, "k6 {tname}-tile batch: {bad} of {rows * d} off the oracle, {g_gpu.decvec_on ? "four-wide twin" : "scalar callback"}\n") + t |> success(bad == 0, "k6 {tname}-tile matches the CPU oracle ({bad} of {rows * d} off)") var y_poison := y_cls // the bar's control: one element pushed past both bars must red y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k6 bar reds a poisoned element") @@ -3613,7 +3821,7 @@ def test_vkd_cm2m_batch(t0 : T?) { feint("no Vulkan device - skipping\n") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + if (!(g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2)) { feint("the cm2 m-tile serves only in mode 4 on an NV_coopmat2 device (DASLLAMA_COOPMAT=cm2) - skipping\n") return } @@ -3717,7 +3925,7 @@ def test_vkd_cm2s_batch(t0 : T?) { t |> skip("no Vulkan device - skipping") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + if (!(g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2)) { t |> skip("the cm2 s-tile serves only in mode 4 on an NV_coopmat2 device (DASLLAMA_COOPMAT=cm2) - skipping") return } @@ -3825,7 +4033,7 @@ def test_vkd_cm2l_split(t0 : T?) { feint("no Vulkan device - skipping\n") return } - if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + if (!(g_gpu.coopmat_mode == COOPMAT_CM2 && g_gpu.has_coopmat2)) { feint("the cm2 l-tile serves only in mode 4 on an NV_coopmat2 device (DASLLAMA_COOPMAT=cm2) - skipping\n") return } @@ -5096,13 +5304,22 @@ def test_vkd_rope_b_pair(t0 : T?) { q_cls |> resize(npos * qd) m_cls |> resize(2 * mirn) let wgs = (npos * ppp + 255) / 256 + let bpad = 96 // the bias rows sit past a pad, so boff is exercised off its default + var bh : array + bh |> resize(bpad + qd + 2 * kvd) + for (i in range(bpad + qd + 2 * kvd)) { + bh[i] = i < bpad ? 77.0 : float((i * 19) % 31) * 0.02 - 0.3 + } + let b_bytes = long_length(bh) * 4l + let bd = make_device_buf(b_bytes) unsafe { upload_region_at(kvdv, 0l, addr(kvh[0]), kv_bytes) upload_region_at(csd, 0l, addr(csh[0]), cs_bytes) upload_region_at(qd2, 0l, addr(q0h[0]), q_bytes) - var sc = set_rope_kv_b_cls(fixed_array(qd2, kvdv, km2, vm2, csd), - fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes), - fixed_array(1u, 16u, 2u, 4u, 8u)) + upload_region_at(bd, 0l, addr(bh[0]), b_bytes) + var sc = set_rope_kv_b_cls(fixed_array(qd2, kvdv, km2, vm2, csd, bd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, b_bytes), + fixed_array(1u, 16u, 2u, 4u, 8u, 0u)) var raw = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw, begin), null) @@ -5110,7 +5327,7 @@ def test_vkd_rope_b_pair(t0 : T?) { var pc = RopeKvBArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(half), neox = uint(neox), kpair0 = uint(kpair0), ppp = uint(ppp), layerbase = uint(layerbase), npos = uint(npos), - pos0 = uint(pos0), voff = uint(voff), kstride = uint(kvd)) + pos0 = uint(pos0), voff = uint(voff), kstride = uint(kvd), boff = uint(bpad), hasb = 1u) enc_rope_kv_b_cls(raw, h, sc, pc, int64(wgs)) vhz_dep(raw, h, 8u, 0u, true) cmd_copy_whole(raw, qd2, host.buf, q_bytes) @@ -5121,8 +5338,8 @@ def test_vkd_rope_b_pair(t0 : T?) { memcpy(addr(q_cls[0]), host.mapped, q_bytes) memcpy(addr(m_cls[0]), reinterpret(intptr(host.mapped) + uint64(q_bytes)), 2l * m_bytes) } - // oracle: per-position NEOX pairs off per-position cos rows; k roped + v raw at - // absolute mirror rows — only the written region compares + // oracle: the bias row at boff first, then per-position NEOX pairs off per-position cos + // rows; k roped + v raw (plus bias) at absolute mirror rows — only the written region compares var q_ref : array var mk_ref : array var mv_ref : array @@ -5142,17 +5359,20 @@ def test_vkd_rope_b_pair(t0 : T?) { let fci = csh[p * hs + half + j] if (is_k) { let kb = p * kvd + hh * hs - let a0 = kvh[kb + j] - let a1 = kvh[kb + j + half] + let bk = bpad + qd + hh * hs + let bv = bpad + qd + kvd + hh * hs + let a0 = kvh[kb + j] + bh[bk + j] + let a1 = kvh[kb + j + half] + bh[bk + j + half] let mo = layerbase + (pos0 + p) * kvd + hh * hs mk_ref[mo + j] = a0 * fcr - a1 * fci mk_ref[mo + j + half] = a0 * fci + a1 * fcr - mv_ref[mo + j] = kvh[voff + kb + j] - mv_ref[mo + j + half] = kvh[voff + kb + j + half] + mv_ref[mo + j] = kvh[voff + kb + j] + bh[bv + j] + mv_ref[mo + j + half] = kvh[voff + kb + j + half] + bh[bv + j + half] } else { let qb = p * qd + hh * hs - let a0 = q0h[qb + j] - let a1 = q0h[qb + j + half] + let bq = bpad + hh * hs + let a0 = q0h[qb + j] + bh[bq + j] + let a1 = q0h[qb + j + half] + bh[bq + j + half] q_ref[qb + j] = a0 * fcr - a1 * fci q_ref[qb + j + half] = a0 * fci + a1 * fcr } @@ -5189,9 +5409,9 @@ def test_vkd_rope_b_pair(t0 : T?) { upload_region_at(kvdv, 0l, addr(kvm_h[0]), kv_bytes) upload_region_at(km2, 0l, addr(msent[0]), m_bytes) upload_region_at(vm2, 0l, addr(msent[0]), m_bytes) - var scm = set_rope_kv_b_cls(fixed_array(qd2, kvdv, km2, vm2, csd), - fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes), - fixed_array(1u, 16u, 2u, 4u, 8u)) + var scm = set_rope_kv_b_cls(fixed_array(qd2, kvdv, km2, vm2, csd, bd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, b_bytes), + fixed_array(1u, 16u, 2u, 4u, 8u, 0u)) var raw = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw, begin), null) @@ -5199,7 +5419,7 @@ def test_vkd_rope_b_pair(t0 : T?) { var pc = RopeKvBArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(half), neox = uint(neox), kpair0 = uint(kpair0), ppp = uint(ppp), layerbase = uint(layerbase), npos = uint(npos), - pos0 = uint(pos0), voff = uint(kvd), kstride = uint(2 * kvd)) + pos0 = uint(pos0), voff = uint(kvd), kstride = uint(2 * kvd), boff = uint(bpad), hasb = 1u) enc_rope_kv_b_cls(raw, h, scm, pc, int64(wgs)) vhz_dep(raw, h, 8u, 0u, true) cmd_copy_whole(raw, qd2, host.buf, q_bytes) @@ -5225,6 +5445,7 @@ def test_vkd_rope_b_pair(t0 : T?) { delete q0h delete kvh delete csh + delete bh delete q_ref delete mk_ref delete mv_ref @@ -5278,17 +5499,26 @@ def test_vkd_rope_b_pair(t0 : T?) { q_cls |> resize(qd) m_cls |> resize(2 * mirn2) let wgs = nh + nkvh + let bpad = 96 // the bias rows sit past a pad, so boff is exercised off its default + var bh : array + bh |> resize(bpad + qd + 2 * kvd) + for (i in range(bpad + qd + 2 * kvd)) { + bh[i] = i < bpad ? 77.0 : float((i * 17) % 29) * 0.02 - 0.28 + } + let b_bytes = long_length(bh) * 4l + let bd = make_device_buf(b_bytes) unsafe { upload_region_at(kvdv, 0l, addr(kvh[0]), kv_bytes) upload_region_at(csd, 0l, addr(csh[0]), cs_bytes) upload_region_at(wnd, 0l, addr(wnh[0]), wn_bytes) upload_region_at(qd2, 0l, addr(q0h[0]), q_bytes) + upload_region_at(bd, 0l, addr(bh[0]), b_bytes) var tokb = make_host_buf(16l, true) var tmq = reinterpret(tokb.mapped) tmq.pos = 0u // mirror row = layerbase alone - var sc = set_qkn_rope_cls(fixed_array(qd2, kvdv, km2, vm2, csd, wnd, tokb.buf), - fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, wn_bytes, 16l), - fixed_array(16u, 1u, 2u, 4u, 8u, 8u, 0u)) + var sc = set_qkn_rope_cls(fixed_array(qd2, kvdv, km2, vm2, csd, wnd, tokb.buf, bd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, wn_bytes, 16l, b_bytes), + fixed_array(16u, 1u, 2u, 4u, 8u, 8u, 0u, 0u)) var raw = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw, begin), null) @@ -5296,7 +5526,7 @@ def test_vkd_rope_b_pair(t0 : T?) { var pc = QknRopeArgs(qd = uint(qd), kvd = uint(kvd), hs = uint(hs), half = uint(half), neox = uint(neox), nh = uint(nh), layerbase = uint(krow), qwoff = 0u, - kwoff = uint(hs), eps = eps) + kwoff = uint(hs), eps = eps, boff = uint(bpad), hasb = 1u) enc_qkn_rope_cls(raw, h, sc, pc, int64(wgs)) vhz_dep(raw, h, 8u, 0u, true) cmd_copy_whole(raw, qd2, host.buf, q_bytes) @@ -5307,8 +5537,8 @@ def test_vkd_rope_b_pair(t0 : T?) { memcpy(addr(q_cls[0]), host.mapped, q_bytes) memcpy(addr(m_cls[0]), reinterpret(intptr(host.mapped) + uint64(q_bytes)), 2l * m_bytes) } - // oracle: per-head rms-norm with its weight row, then NEOX rope off the normed - // row; k lands at the mirror row, v copies raw + // oracle: the bias row at boff, then per-head rms-norm with its weight row, then NEOX + // rope off the normed row; k lands at the mirror row, v copies raw plus its bias var q_ref : array var mk_ref : array var mv_ref : array @@ -5321,14 +5551,14 @@ def test_vkd_rope_b_pair(t0 : T?) { let sbase = is_k ? qd + hh * hs : hh * hs var ss = 0.0 for (i in range(hs)) { - let v = is_k ? kvh[sbase + i] : q0h[sbase + i] + let v = (is_k ? kvh[sbase + i] : q0h[sbase + i]) + bh[bpad + sbase + i] ss += v * v } let inv = 1.0 / sqrt(ss / float(hs) + eps) var nrow : array nrow |> resize(hs) for (i in range(hs)) { - let v = is_k ? kvh[sbase + i] : q0h[sbase + i] + let v = (is_k ? kvh[sbase + i] : q0h[sbase + i]) + bh[bpad + sbase + i] nrow[i] = wnh[(is_k ? hs : 0) + i] * (v * inv) } for (j in range(half)) { @@ -5346,7 +5576,7 @@ def test_vkd_rope_b_pair(t0 : T?) { } if (is_k) { for (i in range(hs)) { - mv_ref[krow + hh * hs + i] = kvh[qd + kvd + hh * hs + i] + mv_ref[krow + hh * hs + i] = kvh[qd + kvd + hh * hs + i] + bh[bpad + qd + kvd + hh * hs + i] } } delete nrow @@ -5379,7 +5609,7 @@ def test_vkd_rope_b_pair(t0 : T?) { [test] def test_vkd_at_prep(t0 : T?) { - t0 |> run("attention prep class (q+k collapse) == the CPU oracle (gated+qknorm+partial rope)") <| @(t : T?) { + t0 |> run("attention prep class (q+k collapse) == the CPU oracle (gated+qknorm+partial rope+projection bias)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!vk_moe_init()) { feint("no Vulkan device - skipping\n") @@ -5409,9 +5639,11 @@ def test_vkd_at_prep(t0 : T?) { var srcqh : array var srckh : array var smh : array + var bh : array //! the projection bias: [q (qd) | k (kvd)], the k pass reads at boff = qd srcqh |> resize(rows * 2 * qd) srckh |> resize(rows * kvd) smh |> resize(sm_n) + bh |> resize(qd + kvd) for (i in range(rows * 2 * qd)) { srcqh[i] = float((i * 19) % 79) * 0.045 - 1.7 } @@ -5421,15 +5653,20 @@ def test_vkd_at_prep(t0 : T?) { for (i in range(sm_n)) { smh[i] = float((i * 11) % 37) * 0.05 - 0.8 } + for (i in range(qd + kvd)) { + bh[i] = float((i * 7) % 23) * 0.03 - 0.3 + } smh[0] = eps let srcq_bytes = int64(rows * 2 * qd) * 4l let srck_bytes = int64(rows * kvd) * 4l let sm_bytes = int64(sm_n) * 4l + let b_bytes = int64(qd + kvd) * 4l let qp_bytes = int64(rows * qd) * 4l let kp_bytes = int64((w0 + rows) * kvd) * 4l let srcqd = make_device_buf(srcq_bytes) let srckd = make_device_buf(srck_bytes) let smd = make_device_buf(sm_bytes) + let bd = make_device_buf(b_bytes) let qp2 = make_device_buf(qp_bytes) let g2 = make_device_buf(qp_bytes) let kp2 = make_device_buf(kp_bytes) @@ -5440,13 +5677,14 @@ def test_vkd_at_prep(t0 : T?) { upload_region_at(srcqd, 0l, addr(srcqh[0]), srcq_bytes) upload_region_at(srckd, 0l, addr(srckh[0]), srck_bytes) upload_region_at(smd, 0l, addr(smh[0]), sm_bytes) + upload_region_at(bd, 0l, addr(bh[0]), b_bytes) // the class collapse: ONE kernel, two dispatches with different push values - var scq = set_at_prep_cls(fixed_array(srcqd, qp2, g2, smd), - fixed_array(srcq_bytes, qp_bytes, qp_bytes, sm_bytes), - fixed_array(8u, 1u, 2u, 4u)) - var sck = set_at_prep_cls(fixed_array(srckd, kp2, g2, smd), - fixed_array(srck_bytes, kp_bytes, qp_bytes, sm_bytes), - fixed_array(8u, 16u, 0u, 4u)) + var scq = set_at_prep_cls(fixed_array(srcqd, qp2, g2, smd, bd), + fixed_array(srcq_bytes, qp_bytes, qp_bytes, sm_bytes, b_bytes), + fixed_array(8u, 1u, 2u, 4u, 32u)) + var sck = set_at_prep_cls(fixed_array(srckd, kp2, g2, smd, bd), + fixed_array(srck_bytes, kp_bytes, qp_bytes, sm_bytes, b_bytes), + fixed_array(8u, 16u, 0u, 4u, 32u)) var raw = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw, begin), null) @@ -5454,12 +5692,12 @@ def test_vkd_at_prep(t0 : T?) { var pcq = AtPrepArgs(d = uint(qd), nh = uint(nh), hs = uint(hs), half = uint(half), flags = uint(flags), rms_off = uint(rmsq_off), cos_off = uint(cos_off), sin_off = uint(sin_off), obase = 0u, - qsrc = 1u, eps = eps) + qsrc = 1u, boff = 0u, hasb = 1u, eps = eps) enc_at_prep_cls(raw, h, scq, pcq, int64(rows)) var pck = AtPrepArgs(d = uint(kvd), nh = uint(nkvh), hs = uint(hs), half = uint(half), flags = uint(flags), rms_off = uint(rmsk_off), cos_off = uint(cos_off), sin_off = uint(sin_off), obase = uint(w0), - qsrc = 0u, eps = eps) + qsrc = 0u, boff = uint(qd), hasb = 1u, eps = eps) enc_at_prep_cls(raw, h, sck, pck, int64(rows)) vhz_dep(raw, h, 1u | 2u | 16u, 0u, true) cmd_copy_whole(raw, qp2, host.buf, qp_bytes) @@ -5469,8 +5707,7 @@ def test_vkd_at_prep(t0 : T?) { submit_wait(raw) memcpy(addr(o_cls[0]), host.mapped, 2l * qp_bytes + kp_bytes) } - // oracle: deinterleave (gated q), per-head rms with the pass's weight row, partial - // NEOX rope over dims [0, 2*half), panel rows at obase + p + //! oracle: deinterleave (gated q), bias, per-head rms with the pass's weight row, partial NEOX rope over dims [0, 2*half), panel rows at obase + p var o_ref : array o_ref |> resize(2 * rows * qd + (w0 + rows) * kvd) let kb0 = 2 * rows * qd + w0 * kvd @@ -5480,16 +5717,17 @@ def test_vkd_at_prep(t0 : T?) { let d = isq ? qd : kvd let nhh = isq ? nh : nkvh let rms_off = isq ? rmsq_off : rmsk_off + let boff = isq ? 0 : qd var rrow : array rrow |> resize(d) for (i in range(d)) { let hh = i / hs let j = i % hs if (isq) { - rrow[i] = srcqh[p * 2 * qd + hh * 2 * hs + j] + rrow[i] = srcqh[p * 2 * qd + hh * 2 * hs + j] + bh[i] o_ref[rows * qd + p * qd + i] = srcqh[p * 2 * qd + hh * 2 * hs + hs + j] } else { - rrow[i] = srckh[p * kvd + i] + rrow[i] = srckh[p * kvd + i] + bh[boff + i] } } for (hh in range(nhh)) { @@ -5536,6 +5774,7 @@ def test_vkd_at_prep(t0 : T?) { delete srcqh delete srckh delete smh + delete bh delete o_ref delete o_cls } else { @@ -5695,6 +5934,8 @@ def test_vkd_dn_family(t0 : T?) { let sm2 = make_device_buf(sm_bytes) let scd2 = make_device_buf(sc_bytes) let qd2 = make_device_buf(q_bytes) + let of_bytes = int64(nvh * ds) * 4l + let ofd2 = make_device_buf(of_bytes) var host = make_host_buf(st_bytes + sm_bytes + sc_bytes + q_bytes, true, [cached = true]) var st_cls : array var sm_cls : array @@ -5711,9 +5952,9 @@ def test_vkd_dn_family(t0 : T?) { var tokb = make_host_buf(16l, true) var tmd = reinterpret(tokb.mapped) tmd.parity = uint(par) - var sc = set_dn_step_cls(fixed_array(projd, st2, sm2, scd2, qd2, tokb.buf), - fixed_array(proj_bytes, st_bytes, sm_bytes, sc_bytes, q_bytes, 16l), - fixed_array(16u, 2u, 8u, 1u, 4u, 0u)) + var sc = set_dn_step_cls(fixed_array(projd, st2, sm2, scd2, qd2, tokb.buf, ofd2), + fixed_array(proj_bytes, st_bytes, sm_bytes, sc_bytes, q_bytes, 16l, of_bytes), + fixed_array(16u, 2u, 8u, 1u, 4u, 0u, 32u)) var raw = alloc_cmd() let begin = VkCommandBufferBeginInfo() vk_check(vkBeginCommandBuffer(raw, begin), null) @@ -5834,6 +6075,50 @@ def test_vkd_dn_family(t0 : T?) { let bad = (mismatch_approx(st_cls, st_ref) + mismatch_approx(sm_cls, sm_ref) + mismatch_approx(sc_cls, s_ref) + mismatch_qbytes(q_cls, q_ref, 1)) t |> success(bad == 0, "dn_step_fused matches the CPU oracle ({bad} elements off)") + //! the f32 arm (oform 1): the same step leaves the o row as floats - the pre-quant oracle row - and advances state and ring alike + var of_f32 : array + var st_f32 : array + var sm_f32 : array + of_f32 |> resize(nvh * ds) + st_f32 |> resize(nvh * ds * ds) + sm_f32 |> resize(sm_n) + unsafe { + upload_region_at(st2, 0l, addr(sth[0]), st_bytes) + upload_region_at(sm2, 0l, addr(smh[0]), sm_bytes) + nan_sentinel_upload(ofd2, of_bytes) + var tokf = make_host_buf(16l, true) + var tmf = reinterpret(tokf.mapped) + tmf.parity = uint(par) + var scf = set_dn_step_cls(fixed_array(projd, st2, sm2, scd2, qd2, tokf.buf, ofd2), + fixed_array(proj_bytes, st_bytes, sm_bytes, sc_bytes, q_bytes, 16l, of_bytes), + fixed_array(16u, 2u, 8u, 1u, 4u, 0u, 32u)) + var hostf = make_host_buf(of_bytes + st_bytes + sm_bytes, true, [cached = true]) + var rawf = alloc_cmd() + let beginf = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(rawf, beginf), null) + var hf : VkHaz + var pcf = DnStepArgs(cd = uint(cd), kd = uint(kd), nkh = uint(nkh), + ds = uint(ds), dconv = uint(dconv), beta_off = uint(beta_off), + g_off = uint(g_off), taps_off = uint(taps_off), + wnorm_off = uint(wnorm_off), a_off = uint(a_off), + dt_off = uint(dt_off), hist_off = uint(hist_off), + eps = eps, qscale = qscale, out_f32 = 1u) + enc_dn_step_cls(rawf, hf, scf, pcf, int64(nvh)) + vhz_dep(rawf, hf, 2u | 8u | 32u, 0u, true) + cmd_copy_whole(rawf, ofd2, hostf.buf, of_bytes) + cmd_copy_range(rawf, st2, 0l, hostf.buf, of_bytes, st_bytes) + cmd_copy_range(rawf, sm2, 0l, hostf.buf, of_bytes + st_bytes, sm_bytes) + vk_check(vkEndCommandBuffer(rawf), null) + submit_wait(rawf) + memcpy(addr(of_f32[0]), hostf.mapped, of_bytes) + memcpy(addr(st_f32[0]), reinterpret(intptr(hostf.mapped) + uint64(of_bytes)), st_bytes) + memcpy(addr(sm_f32[0]), reinterpret(intptr(hostf.mapped) + uint64(of_bytes + st_bytes)), sm_bytes) + } + let badf = mismatch_approx(of_f32, out_ref) + mismatch_approx(st_f32, st_ref) + mismatch_approx(sm_f32, sm_ref) + t |> success(badf == 0, "dn_step_fused's f32 o row (oform 1) matches the pre-quant oracle with state and ring advanced alike ({badf} elements off)") + delete of_f32 + delete st_f32 + delete sm_f32 delete projh delete sth delete smh @@ -6517,9 +6802,9 @@ def test_vkd_kv16_writers(t0 : T?) { var tokb = make_host_buf(16l, true) var tm = reinterpret(tokb.mapped) tm.pos = 0u - var sc = set_rope_kv_f16_cls(fixed_array(q2, kvd_dev, km2, vm2, csd, tokb.buf), - fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, 16l), - fixed_array(16u, 1u, 2u, 4u, 8u, 0u)) + var sc = set_rope_kv_f16_cls(fixed_array(q2, kvd_dev, km2, vm2, csd, tokb.buf, csd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, 16l, cs_bytes), + fixed_array(16u, 1u, 2u, 4u, 8u, 0u, 0u)) var raw2 = alloc_cmd() vk_check(vkBeginCommandBuffer(raw2, begin), null) var h2 : VkHaz @@ -6625,9 +6910,9 @@ def test_vkd_kv16_writers(t0 : T?) { upload_region_at(kvdv, 0l, addr(kvh[0]), kv_bytes) upload_region_at(csd, 0l, addr(csh[0]), cs_bytes) upload_region_at(qd2, 0l, addr(q0h[0]), q_bytes) - var sc = set_rope_kv_b_f16_cls(fixed_array(qd2, kvdv, km2, vm2, csd), - fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes), - fixed_array(1u, 16u, 2u, 4u, 8u)) + var sc = set_rope_kv_b_f16_cls(fixed_array(qd2, kvdv, km2, vm2, csd, csd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, cs_bytes), + fixed_array(1u, 16u, 2u, 4u, 8u, 0u)) var raw = alloc_cmd() vk_check(vkBeginCommandBuffer(raw, begin), null) var h : VkHaz @@ -6700,9 +6985,9 @@ def test_vkd_kv16_writers(t0 : T?) { upload_region_at(kvdv, 0l, addr(kvm_h[0]), kv_bytes) upload_region_at(km2, 0l, addr(msent[0]), m_bytes) upload_region_at(vm2, 0l, addr(msent[0]), m_bytes) - var scm = set_rope_kv_b_f16_cls(fixed_array(qd2, kvdv, km2, vm2, csd), - fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes), - fixed_array(1u, 16u, 2u, 4u, 8u)) + var scm = set_rope_kv_b_f16_cls(fixed_array(qd2, kvdv, km2, vm2, csd, csd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, cs_bytes), + fixed_array(1u, 16u, 2u, 4u, 8u, 0u)) var raw = alloc_cmd() vk_check(vkBeginCommandBuffer(raw, begin), null) var h : VkHaz @@ -6791,9 +7076,9 @@ def test_vkd_kv16_writers(t0 : T?) { var tokb = make_host_buf(16l, true) var tmq = reinterpret(tokb.mapped) tmq.pos = 0u - var sc = set_qkn_rope_f16_cls(fixed_array(qd2, kvdv, km2, vm2, csd, wnd, tokb.buf), - fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, wn_bytes, 16l), - fixed_array(16u, 1u, 2u, 4u, 8u, 8u, 0u)) + var sc = set_qkn_rope_f16_cls(fixed_array(qd2, kvdv, km2, vm2, csd, wnd, tokb.buf, csd), + fixed_array(q_bytes, kv_bytes, m_bytes, m_bytes, cs_bytes, wn_bytes, 16l, cs_bytes), + fixed_array(16u, 1u, 2u, 4u, 8u, 8u, 0u, 0u)) var raw = alloc_cmd() vk_check(vkBeginCommandBuffer(raw, begin), null) var h : VkHaz @@ -7394,7 +7679,13 @@ def test_vk_coopmat_default_and_tile_pick(t0 : T?) { return } let saved_sm = g_gpu.sm_count + let saved_mode = g_gpu.coopmat_mode g_gpu.sm_count = 36 + g_gpu.coopmat_mode = COOPMAT_MM + t |> equal(cm2_tile_cols(8192l, 512l), 128l) // the KHR arm has one tile geometry + t |> equal(cm2_tile_cols(768l, 32l), 128l) // ... the bucket rule included + t |> equal(cm2_split_k(512l, 512l, 2048l, 128l).nsplit, 1l) // ... and no split-k scratch + g_gpu.coopmat_mode = COOPMAT_CM2 t |> equal(cm2_tile_cols(3072l, 128l), 128l) // a short window never takes the half-empty l column t |> equal(cm2_tile_cols(3072l, 512l), 128l) // the 3B q/down shape: m on a strict wave win (96 tiles in 3 waves vs 48 in 2) t |> equal(cm2_tile_cols(8192l, 512l), 256l) // the 3B gate/up shape: a wave tie goes to l @@ -7417,6 +7708,7 @@ def test_vk_coopmat_default_and_tile_pick(t0 : T?) { t |> equal(cm2_tile_cols(3072l, 100l), 128l) // unknown SM count: the short-window rule still fires t |> equal(cm2_tile_cols(3072l, 512l), 256l) // unknown SM count: l, never a split g_gpu.sm_count = saved_sm + g_gpu.coopmat_mode = saved_mode } else { t |> skip("dasVulkan not present") } diff --git a/modules/dasLLAMA/tests/test_vulkan_tier.das b/modules/dasLLAMA/tests/test_vulkan_tier.das index dc4d3a9e50..6fd8dbbac2 100644 --- a/modules/dasLLAMA/tests/test_vulkan_tier.das +++ b/modules/dasLLAMA/tests/test_vulkan_tier.das @@ -2014,11 +2014,24 @@ def test_vulkan_attention(t : T?) { rmsq[i] = 1.0 + synth_f(int64(i), 71l, 0.3) rmsk[i] = 1.0 + synth_f(int64(i), 73l, 0.3) } - // arms: (npos, rot, gated, k4 wo) - for (arm in [(npos = 70l, rot = 64l, gated = true, o4 = false), - (npos = 70l, rot = 128l, gated = false, o4 = false), - (npos = 70l, rot = 64l, gated = true, o4 = true), - (npos = 1100l, rot = 64l, gated = true, o4 = false)]) { + var bq : array + var bk : array + var bv : array + bq |> resize(int(ATQD)) + bk |> resize(int(ATKVD)) + bv |> resize(int(ATKVD)) + for (i in range(int(ATQD))) { + bq[i] = synth_f(int64(i), 83l, 0.4) + } + for (i in range(int(ATKVD))) { + bk[i] = synth_f(int64(i), 89l, 0.4) + bv[i] = synth_f(int64(i), 97l, 0.4) + } + for (arm in [(npos = 70l, rot = 64l, gated = true, wo_k4 = false, bias = false), + (npos = 70l, rot = 128l, gated = false, wo_k4 = false, bias = false), + (npos = 70l, rot = 64l, gated = true, wo_k4 = true, bias = false), + (npos = 1100l, rot = 64l, gated = true, wo_k4 = false, bias = false), + (npos = 70l, rot = 128l, gated = false, wo_k4 = false, bias = true)]) { let npos = arm.npos let rot = arm.rot let half = rot / 2l @@ -2052,14 +2065,15 @@ def test_vulkan_attention(t : T?) { unsafe { matmul_moe_gpu_attn(y, kout, vout, arm.gated ? ATQ_WOFF : ATQ_WOFF + 500000l, ATK_WOFF, ATV_WOFF, - arm.o4 ? ATO4_WOFF : ATO_WOFF, - 0, 0, 0, arm.o4 ? 1 : 0, xq, xs, + arm.wo_k4 ? ATO4_WOFF : ATO_WOFF, + 0, 0, 0, arm.wo_k4 ? 1 : 0, xq, xs, addr(rmsq[0]), addr(rmsk[0]), + arm.bias ? addr(bq[0]) : null, arm.bias ? addr(bk[0]) : null, arm.bias ? addr(bv[0]) : null, addr(cosr[0]), addr(sinr[0]), - npos, ATD, ATQD, ATKVD, ATHS, rot, ATNH, ATNH / ATNKV, + npos, ATD, ATQD, ATKVD, ATHS, rot, ATNH, 1.0 / sqrt(float(ATHS)), AT_EPS, arm.gated, true) } - // the CPU mirror: GEMMs, prep, attention, requant, wo + // the CPU mirror: GEMMs (+ the projection bias), prep, attention, requant, wo var qgy : array var ky : array var vy : array @@ -2070,6 +2084,17 @@ def test_vulkan_attention(t : T?) { } ref_q8_gemm(wk, wsk, xq, xs, ATD, ATKVD, npos, ky) ref_q8_gemm(wv, wsv, xq, xs, ATD, ATKVD, npos, vy) + if (arm.bias) { //! ungated: the q rows are plain [npos x ATQD] + for (p in range64(npos)) { + for (i in range64(ATQD)) { + qgy[int(p * ATQD + i)] += bq[int(i)] + } + for (i in range64(ATKVD)) { + ky[int(p * ATKVD + i)] += bk[int(i)] + vy[int(p * ATKVD + i)] += bv[int(i)] + } + } + } var qp : array var gatep : array var kp : array @@ -2080,7 +2105,7 @@ def test_vulkan_attention(t : T?) { ref_at_attn(qp, kp, vy, gatep, npos, arm.gated, op) // requant the o rows in the wo stack's form, then the wo GEMM var refy : array - if (arm.o4) { + if (arm.wo_k4) { var oq : array var os : array oq |> resize(int(npos * ATQD)) @@ -2135,7 +2160,7 @@ def test_vulkan_attention(t : T?) { delete oq delete os } - let tag = "npos={npos} rot={rot} gated={arm.gated} o4={arm.o4}" + let tag = "npos={npos} rot={rot} gated={arm.gated} o4={arm.wo_k4} bias={arm.bias}" let cy = compare_gout(y, refy) t |> success(cy.maxref > 0.0, "{tag}: chain produced non-trivial outputs") t |> success(cy.maxdiff <= AT_REL_TOL * cy.maxref, diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 6dd7d6a318..e324d02c7c 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -133,3 +133,7 @@ the detected host features.** LLVM's `SubtargetFeatures` takes the last occurrence of a name, so a forced feature placed first is silently overridden by detection. A wasm machine has no force knob and no detected features, so the rule does not reach it. + +**A `[test]` file under this module that carries a `require dasllama/...` line of its own answers +to `modules/dasLLAMA/tests/REVIEW.md` (repo root) as well** - its out-of-folder ledger row lives +there. diff --git a/modules/dasMetal/ARCHITECTURE.md b/modules/dasMetal/ARCHITECTURE.md index 5663b1a435..fe99109fe0 100644 --- a/modules/dasMetal/ARCHITECTURE.md +++ b/modules/dasMetal/ARCHITECTURE.md @@ -166,10 +166,17 @@ than a second GPU. Cross-GPU parity of one source is secondary. Current entries: - **Grid-literal validation is asymmetric; pending, not deliberate.** Vulkan rejects a non-int64 ceil-div grid parameter at macro apply; the Metal emitter accepts it and infers a type silently. -- **cm2 decode-in-load is Vulkan-only by hardware.** `[spirv_decode]` lowers +- **cm2 decode-in-load is Vulkan-only by hardware.** `[spirv_decode]`'s callback form lowers SPV_NV_cooperative_matrix2 tensor loads. Metal-4 tensors have no decode-callback analogue, so the Metal quant GEMMs stage dequant through threadgroup memory instead. Deliberate, target-specific - not a pending port. +- **A kernel body calling a `[spirv_decode]` method directly is Vulkan-only; pending, not + deliberate.** dasSpirv emits the call as an ordinary function - on a plane element the + callee takes the element's index and chains through the plane, on a copy the struct + parameter spills to a local (`dasSpirv/ARCHITECTURE.md` sec.3.5). An MSL function takes a + struct by value with its members addressable as written, so Metal needs no spill; the + direct-call form itself has no MSL fixture yet, and a Metal kernel wanting one decode body + for a staged tile is what would land it. - **Literal fixed-array hoisting is Metal-only; pending, not deliberate.** `msl_emit` lowers a `let` fixed-array local whose elements are all literals to a program-scope `constant` table (renamed on a same-name/different-content collision); `spirv_emit` keeps such a local in diff --git a/modules/dasSpirv/ARCHITECTURE.md b/modules/dasSpirv/ARCHITECTURE.md index e0c66f5f71..04c08e56a9 100644 --- a/modules/dasSpirv/ARCHITECTURE.md +++ b/modules/dasSpirv/ARCHITECTURE.md @@ -45,8 +45,10 @@ compute test as a ready-made end-to-end gate. path), lavapipe advertises >= 1.2, and at `<= 1.3` the entry-point interface lists only Input/Output globals (1.4 requires *all* globals, which would churn every stage's interface). The version is a per-module field, `SpirvModule.version`: a mesh or task stage - raises it to 1.4 because `SPV_EXT_mesh_shader` requires it, and a few subgroup and - cooperative-matrix ops raise it to 1.5. Every other stage stays at 1.3. + raises it to 1.4 because `SPV_EXT_mesh_shader` requires it, a call that re-types a + Block-laid-out struct with `OpCopyLogical` (sec.3.5) raises it to 1.4 because that opcode + requires it, a few subgroup ops raise it to 1.5, and the cooperative-matrix ops raise it to + 1.6. Every other stage stays at 1.3. ## 3. Files and emission mechanism {#files-and-emission} @@ -89,7 +91,7 @@ without macro plumbing. **`[spirv_decode]` method form.** The decode callback's SPIR-V signature is a rigid three parameters. The method form erases the das-level `self` from it, so the decode body still reads its class members - a separate scale plane, push constants, `@workgroup` staging. The four-wide -twin of that callback is section 3.3. +twin of that callback is section 3.3; a kernel body calling the method directly is section 3.5. **Cooperative-matrix element loops carry `Unroll`.** `coopmatClamp` walks a coopmat local element by element through a hand-emitted structured loop bounded by @@ -182,6 +184,45 @@ device an out-of-range load is a fault that surfaces only when the overshoot lea so it tracks allocation layout, not the kernel's inputs. A local fixed array stays eager: its index is register arithmetic, not a device address. +### 3.5 A kernel calls a decode method directly {#direct-decode-call} + +A kernel body may CALL a `[spirv_decode]` method directly: that is an ordinary user function - +its own OpFunction, registered beside the callback form and pulling no cooperative-matrix +capability - which is what lets one decode body serve a tensor load on a cm2 device and a +hand-staged tile on a KHR one. The block argument takes one of two forms, fixed per method at +discovery (a method called both ways is refused). Called on the plane element itself +(`decode(wq[i], bc, cib)`), the block parameter is emitted as the element's `uint` INDEX and the +function's entry chains `OpAccessChain plane, 0, index` once, binding the parameter as a memory +local in the plane's own storage class, so the body's member reads chain through the plane +exactly as the callback form's chain through its block pointer - the element is never loaded as +a value. A pointer could not travel instead: under logical addressing an SSBO pointer is not a +legal OpFunctionCall argument without the VariablePointersStorageBuffer capability, and the +index needs no capability at all. Called on a copy (`let blk = wq[i]; decode(blk, bc, cib)`), +the block goes by value and lands in the spill local of the next paragraph - the whole block +loaded and stored per call, which the KHR tile measured at a third of the element form's rate. + +**A struct value parameter with an aggregate member is a memory local.** A read-only parameter +of a plain data struct that carries a fixed-array or nested-struct member (not a coopmat tile, a +tensor object, a ray query or a sampler marker) binds its SSA OpFunctionParameter and is stored +at entry into a Function-storage OpVariable. The variable is declared first in the entry block, +ahead of the body's locals and call temps; the store that fills it follows the block's last +OpVariable, because SPIR-V requires every OpVariable of a block to lead the block. The parameter +is thereafter that local: its members access-chain like a `var` struct local's. The reason is a +fixed-array member indexed at run time (`blk.qs[cib.y >> 1]`): an SSA composite offers no +pointer, and `OpCompositeExtract` takes literal indices only. A scalar-only struct parameter +keeps the value path (its members extract), and a `let` copy of a block element in a body stays +a composite (scalar and vector members extract; its array and struct members are still refused), +because reading it whole and then storing it would double every element load; the parameter +form pays the store once per call. The Metal emitter needs no arm for this: an MSL function +takes a struct by value and its members are addressable as written. + +**A struct loaded out of a `Block` re-types at the call.** A struct loaded out of a `Block` - +an ssbo element - carries the Block's laid-out `OpTypeStruct`, a different type id from the +plain struct a parameter takes, so a call passing one re-types the value with `OpCopyLogical` +first, and the module's version floor rises to SPIR-V 1.4, which that opcode requires. The +emitter remembers the struct type each loaded id was read as and copies only when the two ids +differ, so a struct already at the plain type passes through untouched. + ## 4. Test architecture - "every emitted instruction has a test" The behavioral layers, then the enforcement gates (all in main-tree `tests/spirv/` except the diff --git a/modules/dasSpirv/REVIEW.md b/modules/dasSpirv/REVIEW.md index 3cabf71480..7ee967eb39 100644 --- a/modules/dasSpirv/REVIEW.md +++ b/modules/dasSpirv/REVIEW.md @@ -27,3 +27,10 @@ only a device run shows the words compute. **A diff under `modules/dasSpirv` that edits a file under `modules/dasGlsl` or `modules/dasOpenGL` is a defect** - dasSpirv copies dasGlsl's design, not its code. + +**A diff that adds an `OpVariable` emit to an emitted function's entry block puts it ahead of +every non-`OpVariable` instruction of that block, and an instruction it emits into that block +goes after the block's last `OpVariable` - in `emit_user_function`, after `collect_locals` and +`alloc_call_temps`.** SPIR-V requires every `OpVariable` of a block to lead the block +(`ARCHITECTURE.md` section 3.5), and CI runs no `spirv-val` to catch an invalid module +(section 4). diff --git a/modules/dasSpirv/spirv/spirv_emit.das b/modules/dasSpirv/spirv/spirv_emit.das index 37d23a1eb3..473e4e10a5 100644 --- a/modules/dasSpirv/spirv/spirv_emit.das +++ b/modules/dasSpirv/spirv/spirv_emit.das @@ -149,8 +149,9 @@ struct EmitCtx { kernel_class : Structure const? // the kernel class (resolve_kernel_method's dispatch table); null = module-globals frontend member_vars : table // member name -> its synthetic Variable (key into globals) member_syms : array // the synthetic Variables (rooted here for gc_node) - invoke_fields : table // intptr(ExprField) of a self-method invoke's method-field read — visitExprField skips these (the invoke arm consumes them structurally) + invoke_fields : table //!< intptr(ExprField / ExprRef2Value) a call arm consumes structurally: the visitor skips them (a self-method invoke's method-field read, the block argument of a plane-element decode call) invoke_targets : table // intptr(ExprInvoke) -> intptr(devirtualized Function), resolved at discovery + decode_elem_calls : table //!< intptr(a [spirv_decode] method called on plane elements) -> intptr(that plane's Variable) decode_methods : table // intptr(ExprField of coopmatLoadTensorDecode's self. argument) -> the devirtualized decode Function, resolved at discovery callback_funcs : table // intptr(Function) -> registered [spirv_combine] / [spirv_per_element] function (referenced via @@ from coopmatReduce / coopmatPerElement) callback_order : array // callback_funcs keys in stable emission order (first-reference order) @@ -167,6 +168,8 @@ struct UserFunc { valid : bool // false = signature rejected at registration (error already emitted) is_method : bool // a kernel-class method: self (arg 0) is dropped from the signature — // members are module-scope OpVariables, so the body needs no receiver + block_arg : int = -1 //!< a directly called [spirv_decode] method whose block argument is a plane element: the das index of that parameter (its uint INDEX travels, the body chains through the plane); -1 = the value form + block_ssbo : uint64 //!< intptr(Variable) of the plane the element form indexes } // Per ref-argument copy-in/copy-out scratch: a Function-storage OpVariable hoisted into the calling @@ -3172,6 +3175,7 @@ class SpirvEmit : AstVisitor { e2ptr : table // intptr(Expression) -> lvalue pointer result-id e2pty : table // intptr(Expression) -> pointee type-id (paired with e2ptr) e2align : table // intptr(Expression) -> Aligned literal for the OpLoad (set only for PhysicalStorageBuffer-rooted chains — PSB loads REQUIRE the Aligned memory operand) + id2sty : table //!< struct value id -> the OpTypeStruct it was loaded as ite_ids : table // intptr(ExprIfThenElse) -> its {merge, then, else} labels lazy_ids : table //!< intptr(ExprOp3 / ExprOp2 &&,||) lowered as a branch (ctx.lazy_temps) -> its labels loop_ids : table // intptr(ExprWhile/ExprFor) -> its 5 loop-skeleton labels @@ -3203,6 +3207,9 @@ class SpirvEmit : AstVisitor { } else { emit(bm, SEC_FUNCS, SpvOp.Load, pty, res, pid) } + if (e._type != null && e._type.baseType == Type.tStructure) { + id2sty[res] = pty + } return res } errs |> push("internal: no rvalue available for {e.__rtti}") @@ -4473,10 +4480,35 @@ class SpirvEmit : AstVisitor { emit_call_core(bm, uf, expr.arguments, 0, 0, intptr(expr)) } + def elem_index_of(var bm : SpirvModule; arg : ExpressionPtr; fname : string) : uint { + var pe = arg + let r2v = pe ?as ExprRef2Value + if (r2v != null) { + pe = r2v.subexpr + } + let at = pe ?as ExprAt + if (at == null) { + errs |> push("'{fname}': a [spirv_decode] method called directly on plane elements takes the element itself (decode(plane[i], ...)) at every call") + return 0u + } + let bt = at.index._type.baseType + if (bt != Type.tUInt && bt != Type.tInt) { + errs |> push("'{fname}': a plane element index must be int or uint, not {bt}") + return 0u + } + let iv = value_of(at.index) + if (iv == 0u) return 0u + if (bt == Type.tUInt) return iv + let res = alloc_id(bm) + emit(bm, SEC_FUNCS, SpvOp.Bitcast, type_uint(bm), res, iv) + return res + } + // Shared OpFunctionCall core. `skip` = leading arguments to drop, `fshift` = how far the // remaining arguments sit AHEAD of f.arguments — a method invoke is [Cls.method field, // receiver, args...] against f.arguments [self, params...], so skip=2 / fshift=1 pairs // args[i] with f.arguments[i-1] (and the callee's self was dropped at registration). + [arch(at="../ARCHITECTURE.md#direct-decode-call")] def emit_call_core(var bm : SpirvModule; uf : UserFunc; arguments : dasvector`ptr`Expression; skip, fshift : int; resKey : uint64) { let f = uf.fn let res = alloc_id(bm) @@ -4506,13 +4538,33 @@ class SpirvEmit : AstVisitor { emit(bm, SEC_FUNCS, SpvOp.Store, ct.ptr_id, v) ops |> push(ct.ptr_id) ref_temps |> push((temp = ct.ptr_id, orig = p.id, vt = ct.value_type)) + } elif (fi == uf.block_arg) { + let iv = elem_index_of(bm, arg, "{f.name}") + if (iv == 0u) { + delete ops + delete ref_temps + return + } + ops |> push(iv) } else { - let v = value_of(arg) + var v = value_of(arg) if (v == 0u) { delete ops delete ref_temps return } + if (fi < length(f.arguments) && f.arguments[fi]._type.baseType == Type.tStructure) { + let want = emit_value_type(bm, f.arguments[fi]._type, errs) + let have = id2sty?[v] ?? 0u + if (want != 0u && have != 0u && have != want) { + if (bm.version < SPV_VERSION_1_4) { + bm.version = SPV_VERSION_1_4 + } + let cv = alloc_id(bm) + emit(bm, SEC_FUNCS, SpvOp.CopyLogical, want, cv, v) + v = cv + } + } ops |> push(v) } } @@ -6970,6 +7022,12 @@ def private register_user_func(var m : SpirvModule; var ctx : EmitCtx; f : Funct uf.valid = false continue } + if (is_struct && uf.block_arg < 0 && key_exists(ctx.decode_elem_calls, intptr(f))) { + uf.block_arg = ai + uf.block_ssbo = ctx.decode_elem_calls?[intptr(f)] ?? 0ul + ptypes |> push(type_uint(m)) + continue + } let vt = emit_value_type(m, at, errors) if (vt == 0u) { uf.valid = false @@ -7425,6 +7483,45 @@ class private SpirvMethodScan : AstVisitor { ctx.invoke_fields |> insert(intptr(da0)) found_decode |> push(mfn) } + [arch(at="../ARCHITECTURE.md#direct-decode-call")] + def note_decode_elem_arg(mfn : FunctionPtr; expr : ExprInvoke const?) : void { + if (!function_has_annotation(mfn, "spirv_decode") || length(expr.arguments) < 3) return + let a0 = expr.arguments[2] + let r2v = a0 ?as ExprRef2Value + let at = (r2v != null ? r2v.subexpr : a0) ?as ExprAt + let sv = at != null ? self_member_of(at.subexpr) : null + let key = intptr(mfn) + let prev = ctx.decode_elem_calls?[key] ?? 0ul + if (sv == null || !(ctx.globals?[intptr(sv)] ?? GlobalInfo()).is_ssbo) { + if (prev != 0ul) { + errs |> push("'{mfn.name}': a [spirv_decode] method called directly takes its block the same way at every call - one call passes a plane element, this one does not") + } + return + } + if (prev != 0ul && prev != intptr(sv)) { + errs |> push("'{mfn.name}': a [spirv_decode] method called directly on plane elements takes them from ONE plane ('{sv.name}' here, another plane elsewhere)") + return + } + ctx.decode_elem_calls[key] = intptr(sv) + ctx.invoke_fields |> insert(intptr(a0)) + } + def self_member_of(e : ExpressionPtr) : VariablePtr { + var pe = e + let r2v = pe ?as ExprRef2Value + if (r2v != null) { + pe = r2v.subexpr + } + let fld = pe ?as ExprField + if (fld == null) return null + var be = fld.value + let br2v = be ?as ExprRef2Value + if (br2v != null) { + be = br2v.subexpr + } + let bv = be ?as ExprVar + if (bv == null || !bv.varFlags.argument || intptr(bv.variable) != self_arg) return null + return ctx.member_vars?["{fld.name}"] ?? null + } def override preVisitExprInvoke(expr : ExprInvoke?) : void { if (key_exists(ctx.invoke_targets, intptr(expr))) return let fld = invoke_self_method_field(self_arg, expr) @@ -7435,6 +7532,7 @@ class private SpirvMethodScan : AstVisitor { delete lerr if (mfn == null) return ctx.invoke_targets |> insert(intptr(expr), intptr(mfn)) + note_decode_elem_arg(mfn, expr) // suppress the structural operands: the method-field read + its wrapper, and the WHOLE // receiver chain (a base-class method's receiver arrives upcast: Ref2Value/Cast wrappers // down to self) — the invoke arm consumes them without producing values @@ -7659,9 +7757,26 @@ def private alloc_call_temps(var m : SpirvModule; var ctx : EmitCtx; f : Functio } } +[arch(at="../ARCHITECTURE.md#direct-decode-call")] +def private spills_struct_param(var m : SpirvModule; at : TypeDecl?) : bool { + if (at == null || at.baseType != Type.tStructure || at.structType == null + || coopmat_info(at).ok || sampler_info(at).ok || "{at.structType.name}" == "rayQueryEXT" + || tensor_object_type_of(m, at).id != 0u) { + return false + } + for (fld in at.structType.fields) { + if (fld._type != null && (fld._type.baseType == Type.tFixedArray || fld._type.baseType == Type.tStructure)) { + return true + } + } + return false +} + // Emit one user-defined shader function: OpFunction, its OpFunctionParameters (value params bind an -// SSA id; ref params bind a Function pointer), the entry block (locals + ref-arg temps), the body, -// and the trailing return. The result id / type were pre-allocated by register_user_func. +// SSA id, a struct value param additionally spills into a Function local; ref params bind a Function +// pointer), the entry block (locals + ref-arg temps + the spill stores), the body, and the trailing +// return. The result id / type were pre-allocated by register_user_func. +[arch(at="../ARCHITECTURE.md#direct-decode-call")] def private emit_user_function(var m : SpirvModule; var ctx : EmitCtx; uf : UserFunc; var errors : array) { let f = unsafe(reinterpret(uf.fn)) // read-only emission; strip pointee const emit(m, SEC_FUNCS, SpvOp.Function, uf.ret_type, uf.fn_id, 0u, uf.type_id) @@ -7671,19 +7786,31 @@ def private emit_user_function(var m : SpirvModule; var ctx : EmitCtx; uf : User if (uf.is_method && !empty(f.arguments)) { ctx.self_arg = intptr(f.arguments[0]) } + var spills : array> + var elem_arg = 0ul + var elem_pid = 0u for (arg, ai in f.arguments, count()) { if (uf.is_method && ai == 0) { continue } let at = arg._type - let vt = emit_value_type(m, at, errors) let pid = alloc_id(m) + if (ai == uf.block_arg) { + emit(m, SEC_FUNCS, SpvOp.FunctionParameter, type_uint(m), pid) + elem_arg = intptr(arg) + elem_pid = pid + continue + } + let vt = emit_value_type(m, at, errors) if (is_out_param(at)) { // written through: the pointer binds as a memory local, so a field of it access-chains // and a bare write stores -- exactly like a `var` local declared in the body. let pt = type_pointer(m, SpvStorageClass.Function, vt) emit(m, SEC_FUNCS, SpvOp.FunctionParameter, pt, pid) ctx.local_vars |> insert(intptr(arg), LocalVar(ptr_id = pid, value_type = vt)) + } elif (spills_struct_param(m, at)) { + emit(m, SEC_FUNCS, SpvOp.FunctionParameter, vt, pid) + spills |> push((arg = intptr(arg), param_id = pid, value_type = vt)) } else { // read-only: binds an SSA id, off which a field read is an OpCompositeExtract like any // other struct value. @@ -7694,8 +7821,31 @@ def private emit_user_function(var m : SpirvModule; var ctx : EmitCtx; uf : User let id_label = alloc_id(m) emit(m, SEC_FUNCS, SpvOp.Label, id_label) ctx.terminated = false + var spill_vars : array + spill_vars |> reserve(length(spills)) + for (s in spills) { + let pt = type_pointer(m, SpvStorageClass.Function, s.value_type) + let vid = alloc_id(m) + emit(m, SEC_FUNCS, SpvOp.Variable, pt, vid, uint(SpvStorageClass.Function)) + ctx.local_vars |> insert(s.arg, LocalVar(ptr_id = vid, value_type = s.value_type)) + spill_vars |> push(vid) + } collect_locals(m, ctx, f.body, errors) alloc_call_temps(m, ctx, f, errors) + for (s, si in spills, count()) { + emit(m, SEC_FUNCS, SpvOp.Store, spill_vars[si], s.param_id) + } + if (elem_arg != 0ul) { + let gi = ctx.globals?[uf.block_ssbo] ?? GlobalInfo() + let ept = type_pointer(m, gi.storage, gi.elem_type) + let chain = alloc_id(m) + var eops <- [ept, chain, gi.var_id, const_uint(m, 0u), elem_pid] + emit_n(m, SEC_FUNCS, SpvOp.AccessChain, eops) + delete eops + ctx.local_vars |> insert(elem_arg, LocalVar(ptr_id = chain, value_type = gi.elem_type, is_opaque = true, storage = gi.storage)) + } + delete spills + delete spill_vars drive_emit(m, ctx, f, errors) if (!ctx.terminated) { if (uf.is_void) { diff --git a/modules/dasUnitTest/unit_test.das.inc b/modules/dasUnitTest/unit_test.das.inc index 9db5163853..9a909d6317 100644 --- a/modules/dasUnitTest/unit_test.das.inc +++ b/modules/dasUnitTest/unit_test.das.inc @@ -1,32 +1,32 @@ -static unsigned char unit_test_das[] = { -0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, -0x67,0x65,0x6e,0x32,0x0a, -0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, -0x72,0x65,0x6d,0x6f,0x76,0x65,0x5f,0x75, -0x6e,0x75,0x73,0x65,0x64,0x5f,0x73,0x79, -0x6d,0x62,0x6f,0x6c,0x73,0x20,0x3d,0x20, -0x66,0x61,0x6c,0x73,0x65,0x0a, -0x0a, -0x72,0x65,0x71,0x75,0x69,0x72,0x65,0x20, -0x55,0x6e,0x69,0x74,0x54,0x65,0x73,0x74, -0x0a, -0x0a, -0x6c,0x65,0x74,0x20,0x4f,0x54,0x48,0x45, -0x52,0x5f,0x55,0x4e,0x49,0x54,0x5f,0x54, -0x45,0x53,0x54,0x5f,0x43,0x4f,0x4e,0x53, -0x54,0x41,0x4e,0x54,0x20,0x3d,0x20,0x55, -0x4e,0x49,0x54,0x5f,0x54,0x45,0x53,0x54, -0x5f,0x43,0x4f,0x4e,0x53,0x54,0x41,0x4e, -0x54,0x20,0x2b,0x20,0x31,0x3b,0x0a, -0x0a, -0x64,0x65,0x66,0x20,0x73,0x74,0x61,0x72, -0x74,0x5f,0x65,0x66,0x66,0x65,0x63,0x74, -0x28,0x6e,0x61,0x6d,0x65,0x3b,0x20,0x74, -0x6d,0x29,0x20,0x7b,0x0a, -0x20,0x20,0x20,0x20,0x72,0x65,0x74,0x75, -0x72,0x6e,0x20,0x73,0x74,0x61,0x72,0x74, -0x5f,0x65,0x66,0x66,0x65,0x63,0x74,0x28, -0x6e,0x61,0x6d,0x65,0x2c,0x20,0x74,0x6d, -0x2c,0x20,0x31,0x2e,0x30,0x29,0x0a, -0x7d,0x0a, -}; +static unsigned char unit_test_das[] = { +0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, +0x67,0x65,0x6e,0x32,0x0a, +0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, +0x72,0x65,0x6d,0x6f,0x76,0x65,0x5f,0x75, +0x6e,0x75,0x73,0x65,0x64,0x5f,0x73,0x79, +0x6d,0x62,0x6f,0x6c,0x73,0x20,0x3d,0x20, +0x66,0x61,0x6c,0x73,0x65,0x0a, +0x0a, +0x72,0x65,0x71,0x75,0x69,0x72,0x65,0x20, +0x55,0x6e,0x69,0x74,0x54,0x65,0x73,0x74, +0x0a, +0x0a, +0x6c,0x65,0x74,0x20,0x4f,0x54,0x48,0x45, +0x52,0x5f,0x55,0x4e,0x49,0x54,0x5f,0x54, +0x45,0x53,0x54,0x5f,0x43,0x4f,0x4e,0x53, +0x54,0x41,0x4e,0x54,0x20,0x3d,0x20,0x55, +0x4e,0x49,0x54,0x5f,0x54,0x45,0x53,0x54, +0x5f,0x43,0x4f,0x4e,0x53,0x54,0x41,0x4e, +0x54,0x20,0x2b,0x20,0x31,0x3b,0x0a, +0x0a, +0x64,0x65,0x66,0x20,0x73,0x74,0x61,0x72, +0x74,0x5f,0x65,0x66,0x66,0x65,0x63,0x74, +0x28,0x6e,0x61,0x6d,0x65,0x3b,0x20,0x74, +0x6d,0x29,0x20,0x7b,0x0a, +0x20,0x20,0x20,0x20,0x72,0x65,0x74,0x75, +0x72,0x6e,0x20,0x73,0x74,0x61,0x72,0x74, +0x5f,0x65,0x66,0x66,0x65,0x63,0x74,0x28, +0x6e,0x61,0x6d,0x65,0x2c,0x20,0x74,0x6d, +0x2c,0x20,0x31,0x2e,0x30,0x29,0x0a, +0x7d,0x0a, +}; diff --git a/modules/dasVulkan/ARCHITECTURE.md b/modules/dasVulkan/ARCHITECTURE.md index 8cc77ba2aa..43934363e6 100644 --- a/modules/dasVulkan/ARCHITECTURE.md +++ b/modules/dasVulkan/ARCHITECTURE.md @@ -216,7 +216,25 @@ either way, and a missing pair leaves the bits off. partial-subgroup guards. A pipeline it returns is valid only on a device from a creator that enabled the pair, and only for a workgroup width that is a multiple of the subgroup size. -## 12. Documentation pipeline +## 12. The OS's video memory numbers {#os-video-memory} + +`vk_os_video_memory` (`src/dasVULKAN.main.cpp`, bound by hand) answers what Vulkan cannot: how +much of the adapter's dedicated memory every process holds right now, and how much system memory +the adapter maps, beside this process's ceiling and its own usage. Its query is one of four +(`OsVideoMemoryQuery`): the process's budget, the process's usage, the adapter's dedicated usage +across every process, and the adapter's shared usage across every process. On Windows the ceiling +and the usage come from three gdi32 exports (`D3DKMTOpenAdapterFromLuid`, +`D3DKMTQueryVideoMemoryInfo`, `D3DKMTCloseAdapter`) over the LOCAL segment group, and the +adapter-wide numbers from pdh's "GPU Adapter Memory" Dedicated Usage and Shared Usage counters +summed over the instances named by the adapter's LUID (`luid_0x_0x_phys_`, one per +physical segment); every entry point is resolved by name (no import library, no WDK, the structs +spelled from `d3dkmthk.h` and `pdh.h`), and everywhere else, or whenever a call fails, the answer +is 0, which a caller reads as "no answer". The boost helper `os_video_memory` takes the LUID from +`VkPhysicalDeviceIDProperties` and returns the four. Neither `VK_EXT_memory_budget` nor the +process's own WDDM budget carries the desktop: on the NVIDIA Windows driver both read the card +minus the OS reserve whatever other processes hold. + +## 13. Documentation pipeline `utils/vulkan2rst.das` documents the ergonomic layer into the generated stdlib pages of the main Sphinx tree by RTTI introspection, modeled on the `imgui2rst` of dasImgui. The hand-filled @@ -238,7 +256,7 @@ explains the patterns and points at the spec, which stays correct as the registr Doc snippets are not compile-checked. -## 13. Tutorial units +## 14. Tutorial units A tutorial is a self-contained unit under `tutorials//`: an offscreen module, its `[compute_shader]` or `[shader]` blob, a pixel-oracle `[test]` that CI gates, and a @@ -254,7 +272,7 @@ resident single-float-pushconstant compute-to-image builder, held as two copies reason. A non-digit shared path such as `tutorials/common/` is what a third windowed compute tutorial would need. -## 14. CI gates +## 15. CI gates - `.github/workflows/vulkan_checks.yml` - the per-PR gate, paths-filtered to `modules/dasVulkan/**` so an unrelated PR pays nothing. It carries the two cheap correctness @@ -271,6 +289,6 @@ paravirtualized GPU that MoltenVK cannot render the suite on. and compute into a storage buffer. No window, no subprocess. A test body calls `volkInitialize()` itself, because nothing in the harness does it. -## 15. Exception ledger +## 16. Exception ledger Empty. No rule in `REVIEW.md` has a ruled-acceptable case here yet. diff --git a/modules/dasVulkan/REVIEW.md b/modules/dasVulkan/REVIEW.md index 4103184d36..d9db976334 100644 --- a/modules/dasVulkan/REVIEW.md +++ b/modules/dasVulkan/REVIEW.md @@ -5,26 +5,27 @@ `record_*.das` recording driver wherever the diff puts it - answers to the `tutorials/` subfolder's checklist. A generator source or a committed generator report, wherever the diff puts it, answers to the `generator/` subfolder's checklist. A `[test]` file, wherever the diff puts it, answers to -the `tests/` subfolder's checklist. +the `tests/` subfolder's checklist. A `[compute_shader]` or `[spirv_kernel]` body, wherever the +diff puts it, answers to `modules/REVIEW_SHADER_EMITTERS.md` as well. **Weakening `REVIEW.das` (beside this file) is a defect:** dropping a check, adding a name to its ignore set, or changing a finding text so it no longer names what failed. -**A diff that adds a feature bit to a `create_device_storage_8_16*` creator in -`daslib/vulkan_boost.das` also adds or updates that bit's probe, in the same change.** A bit the -creator always enables is added to the `storage_8_16_supported` probe; a bit the creator enables -only when the device reports support gets its own `_supported` probe. A bit with no -probe either fails `vkCreateDevice` or lets a kernel use a feature the device never enabled. +**A diff that adds a feature bit to a `create_device*` creator in `daslib/vulkan_boost.das` +also adds or updates that bit's probe, in the same change.** A bit the creator always enables +joins the creator family's `*_supported` probe; a bit the creator enables only when the device +reports support gets its own `_supported` probe. A bit with no probe either fails +`vkCreateDevice` or lets a kernel use a feature the device never enabled. -**A diff to `utils/vulkan2rst.das` that puts a public helper of a module it documents into a -catch-all `group_by_regex` group is a defect - give the helper a group that names what it -does.** A catch-all group hides the helper from the `Uncategorized` check in -`.github/workflows/doc.yml`. +**A diff to `utils/vulkan2rst.das` gives a public helper of a module it documents a +`group_by_regex` group whose title names what the helper does - never a group whose title does +not cover it.** A helper filed under a title that does not cover it is hidden from the +`Uncategorized` check in `.github/workflows/doc.yml` and found by nobody. -**A diff that adds or edits a call to raw `vkCreateDevice`, wherever the diff puts it, appends -`VK_KHR_portability_subset` to that call's extension list when the device advertises it - or -calls a `create_device*` boost creator instead, which appends it.** The Vulkan spec fails -`vkCreateDevice` on a device that advertises the extension without enabling it. +**A diff that adds or edits a call to raw `vkCreateDevice` appends `VK_KHR_portability_subset` +to that call's extension list when the device advertises it - or calls a `create_device*` boost +creator instead, which appends it.** The Vulkan spec fails `vkCreateDevice` on a device that +advertises the extension without enabling it. **A diff that puts the result of a `create_*` call into a local the function neither returns nor moves into a container it deletes declares that local `var inscope`.** A plain diff --git a/modules/dasVulkan/daslib/vulkan_boost.das b/modules/dasVulkan/daslib/vulkan_boost.das index 7dd587cb7f..71ed10ac4c 100644 --- a/modules/dasVulkan/daslib/vulkan_boost.das +++ b/modules/dasVulkan/daslib/vulkan_boost.das @@ -469,6 +469,35 @@ def public pipeline_exec_props_supported(phys : VkPhysicalDevice) : bool { return fpe.pipelineExecutableInfo != 0u } +let private OSVM_BUDGET = 0 +let private OSVM_USAGE = 1 +let private OSVM_ADAPTER_DEDICATED = 2 +let private OSVM_ADAPTER_SHARED = 3 + +//! The OS's video memory numbers for ``phys``, in bytes: this process's dedicated ``budget`` and ``usage``, and the +//! adapter's ``adapter_dedicated`` and ``adapter_shared`` (system-memory) usage across every process. The query ids are +//! the C++ side's OsVideoMemoryQuery. Windows only - 0 elsewhere, without a LUID, on failure. +[arch(at="../ARCHITECTURE.md#os-video-memory")] +def public os_video_memory(phys : VkPhysicalDevice) : tuple { + var idp = VkPhysicalDeviceIDProperties() + var p2 = VkPhysicalDeviceProperties2() + unsafe { + p2.pNext = addr(idp) + } + vkGetPhysicalDeviceProperties2(phys, p2) + if (idp.deviceLUIDValid == 0u) { + return (budget = 0ul, usage = 0ul, adapter_dedicated = 0ul, adapter_shared = 0ul) + } + var lo = 0u + var hi = 0u + for (i in range(4)) { + lo |= uint(idp.deviceLUID[i]) << uint(8 * i) + hi |= uint(idp.deviceLUID[4 + i]) << uint(8 * i) + } + return (budget = vk_os_video_memory(lo, hi, OSVM_BUDGET), usage = vk_os_video_memory(lo, hi, OSVM_USAGE), + adapter_dedicated = vk_os_video_memory(lo, hi, OSVM_ADAPTER_DEDICATED), adapter_shared = vk_os_video_memory(lo, hi, OSVM_ADAPTER_SHARED)) +} + //! create_device_cooperative_matrix PLUS the Vulkan 1.3 subgroup-size-control pair so pipelines can pin //! ``requiredSubgroupSize``; each bit is enabled only when the device reports it (degrades to the plain //! coopmat device). Instance must be api 1.3; opportunistically enables VK_KHR_pipeline_executable_properties. diff --git a/modules/dasVulkan/src/dasVULKAN.main.cpp b/modules/dasVulkan/src/dasVULKAN.main.cpp index cbfcb9c3af..c595cdce9d 100644 --- a/modules/dasVulkan/src/dasVULKAN.main.cpp +++ b/modules/dasVulkan/src/dasVULKAN.main.cpp @@ -1,5 +1,6 @@ #include "dasVULKAN.h" #include +#include #if defined(_WIN32) // volk forward-declares the Win32 handle TYPES (HWND/HINSTANCE) but does not pull @@ -152,6 +153,146 @@ static void * das_vk_debug_callback_ptr() { return (void *) das_vk_debug_callback; } +enum OsVideoMemoryQuery : int32_t { + OSVM_BUDGET = 0, + OSVM_USAGE = 1, + OSVM_ADAPTER_DEDICATED = 2, + OSVM_ADAPTER_SHARED = 3, +}; + +#if defined(_WIN32) +extern "C" __declspec(dllimport) void * __stdcall LoadLibraryA(const char * name); +extern "C" __declspec(dllimport) void * __stdcall GetProcAddress(void * module, const char * name); +namespace { + struct PdhFmtCounterValue { uint32_t CStatus; uint32_t pad; int64_t largeValue; }; + struct PdhFmtCounterValueItemA { char * szName; PdhFmtCounterValue FmtValue; }; + typedef int32_t (__stdcall * PfnPdhOpenQueryA)(const char *, uintptr_t, void **); + typedef int32_t (__stdcall * PfnPdhAddEnglishCounterA)(void *, const char *, uintptr_t, void **); + typedef int32_t (__stdcall * PfnPdhCollectQueryData)(void *); + typedef int32_t (__stdcall * PfnPdhGetFormattedCounterArrayA)(void *, uint32_t, uint32_t *, uint32_t *, PdhFmtCounterValueItemA *); + typedef int32_t (__stdcall * PfnPdhCloseQuery)(void *); + const uint32_t PDH_FMT_LARGE_ = 0x00000400u; + const int32_t PDH_MORE_DATA_ = (int32_t) 0x800007D2; + + uint64_t pdh_adapter_usage(uint32_t luid_lo, uint32_t luid_hi, const char * counter_path) { + static PfnPdhOpenQueryA open_q = nullptr; + static PfnPdhAddEnglishCounterA add_c = nullptr; + static PfnPdhCollectQueryData collect = nullptr; + static PfnPdhGetFormattedCounterArrayA get_arr = nullptr; + static PfnPdhCloseQuery close_q = nullptr; + static bool resolved = false; + if (!resolved) { + resolved = true; + if (void * pdh = LoadLibraryA("pdh.dll")) { + open_q = (PfnPdhOpenQueryA) GetProcAddress(pdh, "PdhOpenQueryA"); + add_c = (PfnPdhAddEnglishCounterA) GetProcAddress(pdh, "PdhAddEnglishCounterA"); + collect = (PfnPdhCollectQueryData) GetProcAddress(pdh, "PdhCollectQueryData"); + get_arr = (PfnPdhGetFormattedCounterArrayA) GetProcAddress(pdh, "PdhGetFormattedCounterArrayA"); + close_q = (PfnPdhCloseQuery) GetProcAddress(pdh, "PdhCloseQuery"); + } + } + if (!open_q || !add_c || !collect || !get_arr || !close_q) { + return 0; + } + void * query = nullptr; + if (open_q(nullptr, 0, &query) != 0 || !query) { + return 0; + } + uint64_t total = 0; + void * counter = nullptr; + if (add_c(query, counter_path, 0, &counter) == 0 && collect(query) == 0) { + uint32_t bytes = 0, count = 0; + if (get_arr(counter, PDH_FMT_LARGE_, &bytes, &count, nullptr) == PDH_MORE_DATA_ && bytes > 0) { + std::vector buf(bytes); + if (get_arr(counter, PDH_FMT_LARGE_, &bytes, &count, (PdhFmtCounterValueItemA *) buf.data()) == 0) { + char prefix[64]; + snprintf(prefix, sizeof(prefix), "luid_0x%08x_0x%08x_phys", luid_hi, luid_lo); + const PdhFmtCounterValueItemA * items = (const PdhFmtCounterValueItemA *) buf.data(); + for (uint32_t i = 0; i < count; ++i) { + const char * name = items[i].szName; + bool match = name != nullptr; + for (size_t k = 0; match && prefix[k]; ++k) { + const char a = name[k], b = prefix[k]; + match = a != 0 && ((a >= 'A' && a <= 'Z') ? a + 32 : a) == b; + } + if (match && items[i].FmtValue.CStatus == 0 && items[i].FmtValue.largeValue > 0) { + total += (uint64_t) items[i].FmtValue.largeValue; + } + } + } + } + } + close_q(query); + return total; + } + struct D3dkmtLuid { uint32_t LowPart; int32_t HighPart; }; + struct D3dkmtOpenAdapterFromLuid { D3dkmtLuid AdapterLuid; uint32_t hAdapter; }; + struct D3dkmtCloseAdapter { uint32_t hAdapter; }; + const uint32_t D3DKMT_SEGMENT_LOCAL = 0; + struct D3dkmtQueryVideoMemoryInfo { + void * hProcess; // NULL = this process + uint32_t hAdapter; + uint32_t MemorySegmentGroup; + uint64_t Budget; + uint64_t CurrentUsage; + uint64_t AvailableForReservation; + uint64_t CurrentReservation; + uint32_t PhysicalAdapterIndex; + }; + typedef int32_t (__stdcall * PfnD3dkmtOpen)(D3dkmtOpenAdapterFromLuid *); + typedef int32_t (__stdcall * PfnD3dkmtQuery)(D3dkmtQueryVideoMemoryInfo *); + typedef int32_t (__stdcall * PfnD3dkmtClose)(const D3dkmtCloseAdapter *); +} +static uint64_t das_vk_os_video_memory(uint32_t luid_lo, uint32_t luid_hi, int32_t query) { + static PfnD3dkmtOpen open_fn = nullptr; + static PfnD3dkmtQuery query_fn = nullptr; + static PfnD3dkmtClose close_fn = nullptr; + static bool resolved = false; + if (!resolved) { + resolved = true; + if (void * gdi = LoadLibraryA("gdi32.dll")) { + open_fn = (PfnD3dkmtOpen) GetProcAddress(gdi, "D3DKMTOpenAdapterFromLuid"); + query_fn = (PfnD3dkmtQuery) GetProcAddress(gdi, "D3DKMTQueryVideoMemoryInfo"); + close_fn = (PfnD3dkmtClose) GetProcAddress(gdi, "D3DKMTCloseAdapter"); + } + } + if ((luid_lo == 0 && luid_hi == 0) || query < OSVM_BUDGET || query > OSVM_ADAPTER_SHARED) { + return 0; + } + if (query == OSVM_ADAPTER_DEDICATED) { + return pdh_adapter_usage(luid_lo, luid_hi, "\\GPU Adapter Memory(*)\\Dedicated Usage"); + } + if (query == OSVM_ADAPTER_SHARED) { + return pdh_adapter_usage(luid_lo, luid_hi, "\\GPU Adapter Memory(*)\\Shared Usage"); + } + if (!open_fn || !query_fn || !close_fn) { + return 0; + } + D3dkmtOpenAdapterFromLuid oa{}; + oa.AdapterLuid.LowPart = luid_lo; + oa.AdapterLuid.HighPart = (int32_t) luid_hi; + if (open_fn(&oa) != 0) { + return 0; + } + D3dkmtQueryVideoMemoryInfo q{}; + q.hAdapter = oa.hAdapter; + q.MemorySegmentGroup = D3DKMT_SEGMENT_LOCAL; + const int32_t st = query_fn(&q); + D3dkmtCloseAdapter ca{}; + ca.hAdapter = oa.hAdapter; + close_fn(&ca); + if (st != 0) { + return 0; + } + return query == OSVM_BUDGET ? q.Budget : q.CurrentUsage; +} +#else +static uint64_t das_vk_os_video_memory(uint32_t luid_lo, uint32_t luid_hi, int32_t query) { + (void) luid_lo; (void) luid_hi; (void) query; + return 0; +} +#endif + Module_dasVULKAN::Module_dasVULKAN() : Module("vulkan") { ModuleLibrary lib(this); lib.addBuiltInModule(); @@ -172,6 +313,8 @@ Module_dasVULKAN::Module_dasVULKAN() : Module("vulkan") { SideEffects::modifyExternal, "das_vk_surface_from_native")->args({"instance", "native_window", "native_display"}); addExtern(*this, lib, "vk_debug_callback", SideEffects::accessExternal, "das_vk_debug_callback_ptr"); + addExtern(*this, lib, "vk_os_video_memory", + SideEffects::accessExternal, "das_vk_os_video_memory")->args({"luid_lo", "luid_hi", "query"}); verifyAotReady(); } diff --git a/modules/dasVulkan/tests/REVIEW.md b/modules/dasVulkan/tests/REVIEW.md index 450dabab8a..2259735265 100644 --- a/modules/dasVulkan/tests/REVIEW.md +++ b/modules/dasVulkan/tests/REVIEW.md @@ -6,3 +6,7 @@ doc: `../ARCHITECTURE.md`. **A diff that adds a `[test]` under `integration/` reaches `volkInitialize()` before its first Vulkan call - in the `[test]` function, or in a helper that function calls first.** Nothing in the harness calls it, and every Vulkan entry point is null until it runs. + +**A `[test]` under `integration/` compares the numbers a driver or OS call returned only to +each other and to the call's own no-answer value - never to a magnitude or a threshold.** They +differ per machine and moment; whether the call answered does not. diff --git a/modules/dasVulkan/tests/integration/test_os_video_memory.das b/modules/dasVulkan/tests/integration/test_os_video_memory.das new file mode 100644 index 0000000000..eef0ca9b35 --- /dev/null +++ b/modules/dasVulkan/tests/integration/test_os_video_memory.das @@ -0,0 +1,33 @@ +options gen2 +options _comment_hygiene = true +options indenting = 4 + +require dastest/testing_boost public +require vulkan +require vulkan/vulkan_boost + +[test] +def test_os_video_memory(t : T?) { + if (volkInitialize() != 0) { + t |> failure("no Vulkan loader") + return + } + var inscope instance <- create_instance("dasVulkan test", make_api_version(1u, 3u, 0u)) + volkLoadInstance(boost_value_to_vk(instance)) + let phys = select_physical_device(instance) + var idp = VkPhysicalDeviceIDProperties() + var p2 = VkPhysicalDeviceProperties2() + unsafe { + p2.pNext = addr(idp) + } + vkGetPhysicalDeviceProperties2(phys, p2) + let os = os_video_memory(phys) + let answered = os.budget > 0ul + if (idp.deviceLUIDValid == 0u) { + t |> success(!answered && os.usage == 0ul && os.adapter_dedicated == 0ul && os.adapter_shared == 0ul, "a device with no LUID reads 0, 0, 0, 0 (no answer)") + return + } + t |> success(os.usage <= os.budget, "the process's usage never passes its ceiling ({int64(os.usage)} <= {int64(os.budget)})") + t |> success(!answered || os.adapter_dedicated >= os.usage, "the adapter's usage is at least the process's ({int64(os.adapter_dedicated)} >= {int64(os.usage)})") + to_log(LOG_INFO, "os video memory: budget {int64(os.budget) / 1_000_000l} MB, usage {int64(os.usage) / 1_000_000l} MB, adapter {int64(os.adapter_dedicated) / 1_000_000l} MB dedicated + {int64(os.adapter_shared) / 1_000_000l} MB system{answered ? "" : " (no OS answer on this platform)"}\n") +} diff --git a/modules/dasVulkan/utils/vulkan2rst.das b/modules/dasVulkan/utils/vulkan2rst.das index 7c6123c4f9..f5a8ae860e 100644 --- a/modules/dasVulkan/utils/vulkan2rst.das +++ b/modules/dasVulkan/utils/vulkan2rst.das @@ -78,6 +78,7 @@ def document_module_vulkan_boost() { group_by_regex("Conveniences", mod, %regex~^(full_area|clear_color|clear_depth)$%%), group_by_regex("Compute tier & device capabilities", mod, %regex~^(storage_8_16_supported|compute_full_subgroups_supported|subgroup_properties|subgroup_compute_ops_supported|integer_dot_product_supported|cooperative_matrix_supported|cooperative_matrix2_supported|cooperative_matrix2_fa_supported|cooperative_matrix2_decode_vector_supported|cooperative_matrix2_properties|cooperative_vector_supported|memory_priority_supported|timeline_semaphore_supported|external_memory_host_supported|external_memory_host_min_alignment|select_transfer_queue_family|create_device_(storage_8_16.*|cooperative_matrix|coopmat_full_subgroups))$%%), group_by_regex("Pipeline introspection", mod, %regex~^(pipeline_exec_props_supported|dump_pipeline_executables)$%%), + group_by_regex("Video memory budget", mod, %regex~^os_video_memory$%%), hide_group(group_by_regex("Finalizers", mod, %regex~^finalize$%%)) ) document("Boost — selection, builders, block brackets, conveniences", mod, "vulkan_boost.rst", groups) diff --git a/plans/REVIEW.md b/plans/REVIEW.md new file mode 100644 index 0000000000..6e71d32c1c --- /dev/null +++ b/plans/REVIEW.md @@ -0,0 +1,7 @@ +# plans Code Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** + +**A document here that a dasLLAMA change writes - a model board, a kernel census, a ledger of +that module's work - applies `modules/dasLLAMA/REVIEW_UPSTREAM.md` too.** Its figure and +reference-build rules bind the rows and readings the document carries. diff --git a/plans/kernel_twins_census.md b/plans/kernel_twins_census.md new file mode 100644 index 0000000000..8cec9bcbdb --- /dev/null +++ b/plans/kernel_twins_census.md @@ -0,0 +1,135 @@ +# Kernel twins census - the dispatch classes that differ only by shape + +Boris (2026-09-07): "we have bunch of vulkan kernels which look like twins - different only by +shape. can we really look at what we have - and see if we can convert them to parametrized +templates. then we can take similar shapes for the new ones. also check what we do with metal." +Two read-only surveys, one per backend; every class read, suspected twins compared line by line. +The conversion work this feeds is a follow-up arc; the shape a NEW family should take is section 3. + +## 1. Vulkan - `dasllama/dasllama_vulkan_classes.das`, 137 `[vk_dispatch]` classes + +| Bucket | Count | Members | +|---|---|---| +| template stamp (`class template` + `[ \|> template_struct_instance]`) | 78 | RouterGemv (2, `RouterGemvT`); AtAttn (1, `AtAttnTileT`, overrides run); DaAttnB/BF16 (2); DaAttnBH128/F16 (2, `DaAttnBH128T`); DaAttn/F16 (2, `DaAttnT`); FaCm2 H64/H128/H256 leaves (8); 42 cm2 leaves (14 formats x l/m/s) + 13 KHR leaves on `KqCm2BatchT`; RopeKvStore/RopeKvStoreB/QknRopeKv f16/f32 (6) | +| inheritance family, base shell + per-format override | 26 | `KqBatchBase` -> 13 `KqBatch*`; `KqGemvBase` -> 13 `KqGemv*` | +| hand-written twin cluster | 20 | T1-T7 below | +| singleton | 16 | MoeCombine, DecCombine, DnBaGemm, TopK, MoeGatherF16, DnConv, DnTail, DnStepFused, DnScan, DnScanP3, AtPrep, SplitKReduce, QkRms, Q8Gemv, Q8Batch, CmI8Batch | + +Out of scope: 16 probe classes in `harness/vk_gemm_probe.das` and 4 fixtures in `tests/_vkd_toy.das`, hand-laid by design. + +### Twin clusters + +| # | Classes (kernel, lines) | What differs | Lines | Proposed axes | Saving | Risk | +|---|---|---|---|---|---|---| +| T1 | `MmBatch` q8_batch_mm_cls 4465-4671; `MmABatch` q8_batch_mm_a_cls 4676-4819; `MmMBatch` q8_batch_mm_m_cls 4824-4943 | tile edge `(pa.d + 127u) / 128u` / `wt * 128u` vs `63u`/`64u`; `warp_r * 64u` vs `32u`; staging `range(8)` vs `range(4)`; LDS `cml_a : uint[2560]` vs `[1280]`; 16 vs 4 `coopmatAcc_f16_16x16`. MmA = MmBatch with the two staging guards and the whole edge-bounce `else` deleted; MmM = MmBatch at half tile. The 84-line MMA block and the 32-line store are byte-identical modulo the constants | 207/144/120 = 471 | `@template_constant BM/BN` (128 or 64); `ALIGNED : bool` gating the guards + store arm; `NFRAG` - the 16-vs-4 accumulator set needs two `static_if` arms (coopmat handles are opaque; a constant-sized array of them must be verified first) | ~230 | low-medium: all three `parity-locked to the shipped kernel`; kernel-unit cells at tests/test_vulkan_kernels.das:1522-1650 dispatch by generated name (names survive); keep the MMA ORDER identical | +| T2 | `FaCm2H64T` 1899-1983; `FaCm2H128T` 2001-2085; `FaCm2H256T` 2099-2192 (8 leaves) | coopmat widths + K step only: `coopmatWgAcc_f32_64x64` vs `64x128` vs `64x256`; `coopmatWgB_f16_64x64` vs `128x32` vs `256x32`; `coopmatLoadTensor(kt, .., j, 64u, kvh * 64u, 64u)` vs `32u, kvh * 128u, 128u` vs `256u`; `j += 64u` vs `32u`; H256T alone carries `GATED` + the `qstride`/`qhs` reads. The 40-line online softmax is character-identical | 85/85/94 = 264 | `HS`, `BC` (64 or 32) constants; 7 coopmat typedefs (QACC/QA/SACC/KB/VB/ONE/OACC) - the cm2 leaves' `BT/ACC/ACCW/FLO` idiom; lift GATED + qstride into the one template (inert at `qstride == 0`) | ~130 | low: cells at tests/test_vulkan_kernels.das:4312-4398, 4527-4620 by name; H64 reads `pa.qd` where H256 reads `qstride` - keep H64's layout | +| T3 | `CmF16Batch` q8_batch_cmf16_cls 4294-4393; `KqQ40CmF16` kq_batch_q40_cmf16_cls 4948-5052 | the 14-line MMA block and the 12-line store are byte-identical; only the two staging loops differ: `nbb = pa.n / 32u` + `nsteps` vs `nsb = pa.n / 256u`; the A-stage scale read per 32-block vs hoisted per 256 (a MEASURED q40 optimisation); B-stage `unpack8` q8 bytes vs the q40 nibble split | 100/105 = 205 | `stage_a` / `stage_b` virtual points on a `CmF16Base` shell (the `KqBatchBase` pattern); `BLKW` (32 or 256) | ~70 | low-medium: both parity-locked; do not unify the hoisted scale read away | +| T4 | `DnRequant` cls_dn_rq 236-268 + `Q8ActRq` q8_actrq_cls 337-369; `Q8kRequant` cls_q8k_rq 272-306 + `Q8kActRq` q8k_actrq_cls 373-403 | Q8_0 pair: DnRequant's tail 255-265 is `Q8ActRq.blk_store` 341-351 verbatim; only the value source `src[eb]` vs `act_mul(gate[eb], up[eb])` and the base `pa.inbase + ..` vs `..` differ. Q8_K pair: `blk_scale` 278-284 duplicated at 374-380 character for character | 33/33 + 35/31 = 132 | `Q8RqT` / `Q8kRqT` with `ACT : bool` gating `def src_val(e)` (`static_if (ACT) act_mul(gate[e], up[e]) else src[e]`) + a `BASE0` constant; the two bind different SSBO sets, so members go behind the axis too | ~42 | medium: crosses the `rq_cls` / `actrq_cls` set-layout families; the `[arch(..#q8-requant-byte-store)]` citations move with the code | +| T5 | `ArAddRmsRq` cls_ar_rq 141-173 + `ClsArAddRmsRqB` cls_ar_rq_b 176-210; `ClsArAddRms` cls_ar 122-136 + `ClsArAddRmsF16B` cls_ar_f16_b 213-227 (all on `ArBase`) | Rq pair 25 of 29 lines identical: `accum_row(0u)` vs `accum_row(rbase)`, `outs[b]`/`outq[base]` vs `outs[qb0 + b]`/`qb0 * 32u + base`. Store pair: `yo[base + k] = ..` vs `outh[rbase + k] = float16(..)`, `base = (gl_WorkGroupID.x + pa.row0) * pa.dim` vs `rbase = gl_WorkGroupID.x * pa.dim` | 15/33/35/15 = 98 | `ROWMAJOR : bool` (the qb0/row0 rebase); `OUT` constant or a `def emit_row` with three `static_if` arms (f32, f16, Q8_0 blocks) | ~25 | low: cls_ar_rq's header pins bit-parity with the split pair - fold order and `q8_quant1` rounding untouched; three `[arch]` citations move | +| T6 | `AtAttnTileT.run` 1676-1764 vs `AtAttn.run` 1776-1852 (`def override run`) | the same flash Q-tile re-authored: `pa.npos` vs `pa.rows`; `qrow = pa.nh * qhs` vs `pa.qd`; `dpl = pa.hs / 32u` vs `pa.dpl`; K/V base `pa.kbase`/`pa.vbase` vs `0u`; gate from `qpanel` vs `outp`; `pa.gated` vs `pa.flags & 1u`; the template guards the rescale/PV with `if (dpl > 1u)` / `> 2u` where AtAttn does o0..o3 unconditionally and guards only `> 4u` | 89 + 77 = 166 | `MIRROR : bool`, `GATE_IN_Q : bool`, one `AtAttnArgs`/`DaAttnBArgs` reconciliation | ~70 | medium-high: the `dpl` guards are an emitted-code difference (4 FMAs vs 1/2/4/8); two different parity bars pinned in comments; only behind both parity cells | +| T7 | `RopeKvStoreT` 6749-6801; `RopeKvStoreBT` 6837-6898; `QknRopeKvT` 6932-7010 (6 leaves) | not full twins (decode token, prefill npos, fused qk-norm), but the `static_if (CLAMP16)` store block is written four times (6781-6791, 6877-6887, 6977-6983, 6997-7007), the `pa.hasb != 0u ? bias[pa.boff + ..] : 0.0` idiom twelve times, the rotation core three times | 53/62/79 = 194 | helper extraction on a common base: `def store_kv2(mo, e0, e1; k0, k1, v0, v1)` carrying the CLAMP16 arm, `def biased(buf; i, boff)` | ~25 | low: inlined helpers leave the emitted code unchanged | + +### Near misses - same shape, different algorithm; do not template + +| Pair | Why not | +|---|---| +| `Q8Batch` 2607-2725 vs `KqBatchBase.tile_shell` 3480-3511 | Q8_0's `nbb = n / 32` with a `nsteps` outer loop and a `b0 + bu < nbb` tail vs Q8_K's `nsb = n / 256` with no tail; no block-sum term; 4 scalar accumulators vs `float4` + `bs` + two scale planes. A 14th `KqBatch*` leaf would force a superblock alignment the q8 path deliberately lacks | +| `Q8Gemv` 7038-7081 vs `KqGemvBase.gemv_shell` 7171-7195 | two lanes per 32-block (`b = bi0 + lane / 2u`, a `subgroupShuffleXor(idot, 1u)` fold) vs one lane per block (`bb = b0 + lane`); merging changes the workhorse GEMV's access pattern | +| `AtAttnTileT` (SIMT) vs `DaAttnBH128T` (coopmat) | staged f32 tiles at stride 257 + an 8-row online softmax vs `coopmatA_f16_16x16` fragments + a 4-thread-per-row shuffle softmax | +| `MoeCombine` 512-538 vs `DecCombine` 546-566 | windowed read-modify-write with a slot-window predicate, one wg per position, vs a one-row residual add, one thread per element | +| `DnRequant` vs `Q8kRequant` (and the ActRq pair) | 8 lanes x 4 quants with a 3-shuffle amax vs 32 lanes x 8 with a 5-shuffle amax and a `float[8]` stash - different formats; the CROSS pairing is the twin (T4) | +| `DnScanP3` 1437-1468 vs `DnStepFused`'s out-norm tail 1103-1120 | a batched pass over npos rows off a work surface vs a per-head in-register epilogue | +| `EmbGather` 454-476 vs `EmbGatherF32` 479-498 | 5 differing lines, the f32 arm skips binding 1; a template costs more than it saves | +| `ActF16` 407-425 vs `F16Cvt` 429-444 | 8 shared lines, different arity | + +### Ranked order (saving x inverse risk) + +1. T1 the Mm trio (~230 lines; MmABatch is MmBatch with guards deleted; cells cover all three by name). DONE - `MmBatchT` (BT/EDGE/NP/ASZ; the L and M fragment sets stay two `static_if` arms). +2. T2 the FaCm2 head-size templates (~130; the cleanest axis set - 7 typedefs + 2 constants). DONE - `FaCm2T` (HS/BC + 8 typedefs; the h256 pair's SPIR-V byte-identical, the h64/h128 gain the inert qstride/qhs selects). + Gate used for both: the kernel suite on the committed and the converted file, a `DASLLAMA_VK_SPV_DUMP` of every kernel diffed across (only the converted kernels differ), and the harness probe rows within noise. + Batch B, the detect-dupe finds (`utils/detect-dupe` over the file: 224 records, 16 exact clusters, 38 fuzzy pairs - it sees the copied helpers and the thin shells, not a twin whose bodies differ by a guard, and its "exact" collapses LITERALS, so every clone was read before deletion): `iq3s_signed` a free function (ten copies gone), `iq4_dot` a free function (two), `KqBatchIq3Base` carrying the iq3-row scale stage for iq3s/iq3xxs/iq2xxs, `KqBatchIq2Base` carrying the unsigned-strip stage and fold for iq2s/iq2xs, k3 inheriting k6's fold through a one-line `q_off` override (the bodies differed only by the 32 vs 4 literal), `KqGemvIq4nl` on q40's base, and `MoeCmBase` as the one region rail under `Q8Batch` and `KqBatchBase` too. DONE - suite 108 of 108, the kq probe rows within noise on k3 / iq2s / iq2xs / iq3s / iq4nl, 21 kernels' SPIR-V re-emitted (the three families whose bases moved) and every other kernel byte-identical. + Batch C: T3 - `CmF16Base` carries the f16 32x32 shell (prologue, the 16-MMA step, the scratch bounce store) with `stage` as the leaf's point and `body(nsteps, nblk)` taking the step and block counts the leaf's format reads (q8's eight blocks per step, q40's one superblock); T5's requant pair - `ArRqBase.rq_row(rbase, qb0)` is the loop, the one-row leaf calls it at (0, 0), the batched leaf at its workgroup's row and block base (the arithmetic is the batched form's, the one-row form's `+ 0` folds). + Not done, by ruling here: T4 crosses the `rq_cls` / `actrq_cls` set layouts (the block store references bindings the two families number differently - a base method cannot name a leaf's binding); T6 changes AtAttn's emitted code below hs 128; T7's shared store helper needs a second base under `QknRopeKvT`, which already derives from `RmsWgBase`, and the bias idiom shortens twelve lines without removing one. The Q40 / Iq4nl batch scale stage stays twice: different bases, a shared-array write no free function can carry. +3. T3 CmF16Batch/KqQ40CmF16 (~70; produces the `stage_a`/`stage_b` shape a per-format family wants). +4. T7 the rope store/bias helpers (~25; opportunistic with any rope touch). +5. T4 / T5 (~67 together; T4 crosses two set-layout families, T5 carries a bit-parity contract). +6. T6 AtAttn vs its own template - only behind both parity cells, or leave it. + +## 2. Metal - `dasllama_metal_kernels.das` (175) + `dasllama_metal_prefill.das` (121), 296 `[metal_dispatch]` classes + +| Bucket | Count | +|---|---| +| stamps of a `class template` (68 templates) | 173 | +| derived from a plain base (`def abstract` scaffold: `MetalMmTileBase`, `MetalMoeMulMmBase`, `MetalRopeStoreQ8Base`, `MetalRopeStoreTq4Base`) | 14 | +| hand-written twin-cluster members | ~44 | +| singletons | ~65 | +| hand-bound `[metal_kernel]`-only (`dasllama_metal_gemm.das`, `options _metal_manual_dispatch`) | 2 | +| dispatch classes in the tower / ASR / MTP / TTS / whisper files | 0 (pure drivers over generated `enc_*`) | + +Three authoring generations coexist: gen-0 copy-paste per format (`MetalKqGemv*`, `MetalKqMvB8*`, `MetalMoeGemv*`); gen-1 one template with one `bool` constant per format in a nested `static_if/else` ladder (`MetalKqMulMmK45T`, 12 bools, 11 levels); gen-2 a shell with one overridable stage (`MetalKqMulMmSplitTensorBase.stage16`, `MetalMoeMulMmBase.stage_a`; `ARCHITECTURE_GPU_PREFILL.md` sec.2.2aa / 2.2g) - the shape Vulkan's `KqCm2BatchT` mirrors. Gen-0 and gen-1 are already banned by `REVIEW_GPU.md` ("kernel twins stamp one `class template`"); the clusters are standing debt. + +### Twin clusters (lines in `dasllama_metal_kernels.das` unless `[pf]`) + +| # | Cluster | Members | Lines | What differs | Axis | Save | Risk | +|---|---|---|---|---|---|---|---| +| 1 | KqMv B2/B4 per-format templates | 12 templates `MetalKqMvK2T` 8368, `K3T` 8615, `Iq3xxsT` 9385, `Iq3sT` 9555, `Iq2sT` 9727, `Iq2xsT` 9899, `Iq2xxsT` 10071, `Iq4xsT` 10237, `Iq4nlT` 10398, `K4T` 10561, `K6T` 10771, `K5T` 10890 (24 stamps) | 80-99 each, 1062 | Iq4xs vs Iq4nl: ONE functional line (`let dl = float(kdh[blk]) * float((int(kscb[blk*16u+tx]) ^ 128) - 128)` vs `float(kdh[blk*8u+tx])`); Iq2s vs Iq2xs: 8 lines; the NR/NRU/TILED shell, `sumf : float[NR]` and both TILED writeback arms byte-identical | one `MetalKqMvT` shell + `def abstract stage_w16`; per-format `def override`; NR/NRU/TILED stay | ~830 | low-med: TILED branch-duplicates the b-loop for a measured +2% (k4) / +0.5% (k6); "B4 stamps byte-identical" must be proven on the generated MSL | +| 2 | KqGemv single-column per format | 14 standalone `K4` 7985, `K5` 8066, `K5C` 8140, `K6` 8219, `K2` 8287, `K3` 8548, `Iq4xs` 8805, `Iq4nl` 8862, `Iq3s` 8917, `Iq3sF4` 8973, `Iq2s` 9040, `Iq2xs` 9108, `Iq3xxs` 9264, `Iq3xxsF4` 9319 (`MetalKqGemvIq2xxsFam` already templated) | 55-81 each, ~1030 | Iq4xs vs Iq4nl one line; the x-stage / `sumy` / simd reduce / writeback shell (~45 lines) identical in all 14 | one shell + `def abstract dot_block`; F4 as `@template_constant F4` + `@workgroup @template_gate = F4 gridf` (`MetalKqGemvIq2xxsFam` proves it) | ~630 | med: `K5C`, `Iq3sF4`, `Iq3xxsF4` are per-box crowns (`race_kq_rows`, `race_gemv_f4_twin`) - separate stamps; the F4 forks bundled a second change (`first_row * 2u` -> `* 4u`, `sumf[2]` -> `[4]`, `ib += 4u` -> `2u`) = two constants (F4 + ROWS) | +| 3 | KqMvB8 per format | 10 standalone `B8K2` 8477, `B8K3` 8727, `B8Iq3xxs` 9489, `B8Iq3s` 9660, `B8Iq2s` 9826, `B8Iq2xs` 9998, `B8Iq2xxs` 10167, `B8Iq4xs` 10339, `B8Iq4nl` 10500, `B8K6` 11102 (`MetalKqMvB8K45T` templated) | 59-79 each, ~800 | the same deltas as cluster 1; the tg-staged X-panel shell byte-identical | extend `MetalKqMvB8K45T` into the format-abstract shell; `stage_w16` shared with cluster 1 | ~470 | low | +| 4 | SqAttn single / batched / decode families | 10 templates: `SqAttnKvT` 501 / `BKvT` 2452 / `DKvT` 3110; `QuantT` 558 / `BQuantT` 2516 / `DQuantT` 3335; `PartKvT` 626 / `PartBKvT` 2588; `PartQuantT` 692 / `PartBQuantT` 2663; `RopeStoreKvT` 1420 / `RopeStoreBKvT` 3249 | ~45 each, ~450 | the batched arm adds `@ssbo @binding = 4 @off = "rtoff" rt : array`, `let rrow = rt[bb]` / `cnt = rrow.w`, `qstride` addressing - 31 of 45 lines differ, one axis | `@template_constant BATCHED` + `@template_gate = BATCHED rt` - `MetalSqAttnCombT` ships exactly this | ~250 | low-med: the D forms are a third arm | +| 5 | MoE GEMV per format | 6 standalone `Q8` 6613, `K4` 7293, `K5` 7401, `K6` 7486, `Mx4` 7566, `Q51` 7644 | 65-108, 483 | the dot loop line-identical to `MetalKqGemv*`; delta = the gather prologue (`eb = sel[st*ss+slot]*esb`, `xoff`, `y0`), `MoeGemvArgs`, the `sel` binding | cluster 2's shell + `GATHERED : bool` + `@template_gate = GATHERED sel` | ~230 | med: MoeGemvK4's `float4` x view is a measured 2.25x (142 -> 321 wGB/s) - its own axis | +| 6 | KQ tensor mul_mm K45/K6 x Db `[pf]` | `K45TensorT` 472, `K45DbT` 592, `K6TensorT` 703, `K6DbT` 782 (13 stamps) | 63-85, 296 | K45 vs K6 = the decode; Db = `twb : float16[6144]` vs `[9216]` + the decode moved verbatim into `stage_chunk` with a ping-pong offset - the k4/k5/k6 decode exists twice | derive all four from `MetalKqMulMmSplitTensorBase`; `DBUF : bool` + a `TWBN`-sized `@workgroup`; the format's `stage16` once | ~165 | med: the Db forms carry `[metal_kernel(float_a_ok=true)]` and pin `XT = float16` (the sanctioned float-A list, `ARCHITECTURE_GPU.md` sec.2.2b) | +| 7 | dense mul_mm base shells | `MetalF32MulMm` 5847, `MetalQ8MulMm` 5924, `MetalBf16MulMm` [pf] 39, `MetalKqMulMmK45T` 6001 | 63-348, ~565 | the 32x64 shell (`mBase = gl_WorkGroupID.x * 32u` .. `acc_quad(aB, bB, ma, mb, mc)` .. the 8-way `simdgroup_store`) repeated four times verbatim | `MetalMoeMulMmBase` already factors this shell into `run` + `def abstract stage_a`; the four dense classes derive from a twin base | ~145 | low | +| 8 | `MetalKqMulMmK45T`'s 12-format ladder | 1 template 6001-6347 + 12 stamps 6348-6416 | 348 | 12 independent bools (`QH`, `SIXBIT`, `IQ4XS`, `K3`, `IQ3S`, `IQ3XXS`, `IQ4NL`, `K2`, `IQ2S`, `IQ2XS`, `IQ2XXS`) in an 11-deep nest; `MetalKqMulMmIq4nl` must set `IQ4XS` AND `IQ4NL` | `def abstract stage16` (the sec.2.2aa shape); each format one ~18-line override | ~120 + removes the coupled-bool trap | med: the base kernel every box falls back to; parity-locked per format by the kernel-unit gates | +| 9 | rope-store q8/tq4 batched | `RopeStoreQ8` 1657 / `BQ8` 3414; `RopeStoreTq4` 1762 / `BTq4` 3477 | 48/59, 67/72 | `q8_rope_q(.., 0u, 0u)` vs `(.., qbase, tcb)`; the batched binds `rt : array` | `BATCHED` + `@template_gate` on the existing bases | ~100 | low | +| 10 | argmax rows | `MetalArgmax` 1090 / `MetalArgmaxRows` 1169 | 52/55 | `lg[j]` vs `lg[base + j]`, `tok[0]` vs `tok[row]` | `ROWS : bool` | ~50 | low | +| 11 | Q8 mv B2/B4 | `MetalQ8MvB2` 1957 / `MetalQ8MvB4` 2018 | 57/64 | lane maps and strides (`% 16u` vs `% 8u`, `*34u` vs `*17u`, `acc : float[2]` vs `[4]`) | the sibling `MetalGemvB24T` (NR/PANEL/WIDE) exists next door - the q8 form was never converted | ~55 | low | +| 12 | `MetalQ8GemmBSplitT` / `MetalQ8Gemm64B` | 2182 / 2364 | 99/83 | `tw : float16[1024]` vs `[2048]`, `mc[4]`/`ma[2]` vs `mc[8]`/`ma[4]` | `TILE_N` - `MetalQ8GemmTensorT` already parameterizes this | ~40 | med: the hot staging loop | +| 13 | dequant K4/K5 vs K6 `[pf]` | `MetalDequantK45T` 223 / `MetalDequantK6H` 279 | 50/40 | the kmask sc/mn decode vs the split d x int8 sub-scale | a `SIXBIT` arm, or the sec.2.2aa scaffold | ~30 | low | +| 14 | cross V f32/f16 | `MetalCrossVx` 5480 / `MetalCrossVxH` 5515 | 32/32 | ONE statement (`vx[..] = v` vs the f16 clamp) + the array type | `typedef VT` + `CLAMP16` (as `MetalQ8GemvQkvRsF16/F32`) | ~30 | none | +| 15 | q8 mul_mm tensor / double-buffered `[pf]` | `MetalQ8MulMmTensorT` 102 / `MetalQ8MulMmDbT` 148 | 26/27 | `twb : float16[6144]` vs `[9216]`; `tmm2d_q8u_f32(.., 64u)` vs `128u` | `BK` + a `TWBN`-sized `@workgroup` (`MetalGemvB24T`'s `txp : float4[PANEL]` proves template-sized tgmem) | ~22 | low | +| 16 | spectrum mag/power `[pf]` | `MetalG4aMag` 3442 / `MetalQ3aPow` 3599 | 26/30 | `sqrt(re*re + im*im)` vs `re*re + im*im` | `SQRT : bool` | ~22 | none | +| 17 | bias elementwise | `MetalBiasAddRes` 5781 / `MetalBiasGeluLut` 5800 | 16/17 | `x += y + b` vs `x = gelu_lut(x + b)`; an extra `y` binding | `def abstract bias_op` + `@template_gate = HASRES y` | ~14 | none | +| 18 | `dasllama_metal_gemm.das` pair | `MetalQ8Gemm` 39 / `MetalQ8Gemm64` 129 | 90/92 | `ta/tb : float16[1024]` vs `[2048]`, `ntileN`, `c00..c11` vs `mc[16]` | `TILE` - but hand-dispatched, no lens benefit | ~40 | med, lowest value | + +Weak, do not merge: `MetalSuppressRow` 11315 / `MetalSoftcapRow` 11333 share a 6-line guard only. + +### Near misses - do not template + +`MetalAttnQK` / `MetalAttnQKMm` / `MetalAttnQKMmTensorT` (three algorithms); `MetalTowerFlash` vs the QK/Rowstat/AV slab trio (sec.2.2w rules them distinct); `MetalG4aAttn` vs `MetalCnAttn` (causal window + two-pass softmax + softcap vs bidirectional online softmax with biases); `MetalG4aDw` vs `MetalCnDw` (radius, boundary policy and epilogue differ); `MetalRopeStoreQ8Base` vs `MetalRopeStoreTq4Base` (thread-per-block amax vs a threadgroup FWHT stage); `MetalEmbedQ8` vs `MetalEmbedK6`; `MetalSqAttnCombT` vs `MetalSqAttnCombD` (two-pass with a sink vs single-pass log-sum-exp); `MetalRmsNormT` vs `MetalLayerNorm`; `MetalCrossKx` vs `MetalCrossVx` (inverted layouts); `MetalDnConv` vs `MetalDnConvHist` (opposite data flow); `MetalQkNorm` vs `MetalDnL2Norm`; `MetalQ8Gemv` vs `MetalQ8GemvT` (reduction split across simdgroups vs one row per simdgroup); `MetalWdecAttnPart/Comb` vs `MetalSqAttnPart*/Comb*`; `MetalG4aLogMel` vs `MetalQ3aLogMelT` (a `readwrite` role cannot flip to read + write bindings); `MetalMoeMulMmQ8TensorT` vs `MetalMoeMulMmKqTensorBase` (ruled in sec.2.2g). + +### Ranked order + +| Rank | Action | Save | Risk | +|---|---|---|---| +| 1 | KqGemv + KqMvB2/B4 + KqMvB8 onto ONE format-abstract shell (clusters 1+2+3): one `def abstract` decode per format, three shells consuming it; do the 12 templates of cluster 1 first | ~1930 | low-med | +| 2 | SqAttn `BATCHED` axis (cluster 4) - a mechanical repeat of `MetalSqAttnCombT` | ~250 | low | +| 3 | `MetalKqMulMmK45T` ladder -> `stage16` scaffold (8) + tensor K45/K6 x Db (6) | ~285 | med | +| 4 | dense mul_mm shells onto a `MetalMoeMulMmBase` twin (7) | ~145 | low | +| 5 | MoE GEMV `GATHERED` axis (5), the x view its own axis | ~230 | med | +| 6 | the zero-risk singles (14, 11, 10, 9, 15, 13, 16, 17) | ~325 | none-low | + +Total addressable about 3000 lines, ~44 hand classes into ~10 shells. + +### Cross-backend notes + +Copy from Metal: `@template_gate` on `@workgroup` fields gives per-stamp threadgroup sizing (a `daslib/typemacro_boost` facility, so Vulkan's `vkd_wg_fits` footprint gate is per-stamp for free - `KqCm2BatchT` already uses it); the constant-table hoisting (`iq2s_gw` .. `iq2xxs_gw` are all-literal `fixed_array` locals lowered to program-scope constants, no binding, no tgmem - confirm the SPIR-V emitter's behaviour before another staged slab; on Vulkan the constant composite read lane-serially, hence `kq_grid_dev`); the overridable-stage granularity is chosen (sec.2.2aa: `stage16` = the base GEMV arm's own granularity, so a format's decode ports verbatim - the same idea as Vulkan's `decode` / `decode_v4`). + +Avoid: the `MetalKqMulMmK45T` shape (one bool per format in an 11-deep nest; `static_elif` exists - `src/parser/ds2_parser.ypp:193` - but a 12-way format axis wants the abstract-stage scaffold, not a flatter ladder); the gated-field ternary rule bites both backends (a `@template_gate`d field may be named only inside a `static_if` arm on its own axis - the price is byte-duplicated arms, e.g. `MetalKqMvK4T`'s TILED); Metal's `tgmem=` string couples to `__msl_tgmem`, so converting a hand class to a stamp changes it - Vulkan has no such coupling and must not grow one. + +### Lint opportunity + +`REVIEW_GPU.md` bans copy-pasted twins in prose; nothing mechanical catches them and 44 standing violations survive. A `modules/dasLLAMA/dasllama/REVIEW.das` gate flagging two dispatch classes in one module whose kernel bodies exceed a normalized-token similarity threshold (identifier-renaming-insensitive, the `detect_duplicates` canonical form scoped to kernel bodies) without sharing a base would have caught every cluster above at the diff that introduced it. + +## 3. The shape a new GEMM family takes (the integer MMQ tile) - from `KqCm2BatchT` + +| Need | Copy from | Spelling | +|---|---|---| +| 128x128x32 and a 64-token variant | `KqCm2BatchT.BN`, plus the `BM` axis the Mm trio lacks | `@template_constant BN` AND `BM`; bake only BK - `KqCm2BatchT` fixes BM = 128 and paid for it in the Mm fork | +| the partial-column path | `STILE` + `FLO` | `STILE : bool` selecting the padded layout, as `Q8Cm2SBatch` 6147-6153 does | +| aligned vs edge tiles | `MmABatch` (today a whole class) | `ALIGNED : bool` gating the staging guards and the store arm - one class | +| fragment / accumulator types | the cm2 leaf stamps 6129-6181 | `typedef BT / ACC / ACCW` per leaf; the MMQ adds `typedef DEC4` (the packed-int8 word the decode returns, where cm2 returns `half4`) | +| 13 per-format decode arms | `K4Cm2T` .. `Iq2xxsCm2T` 5509-6126 | one `class template Cm2T : ` per format: `typedef BLK` (16-bit-lane block struct), `typedef ST`, `override BLKW`, a `decode` method; the leaf carries only width typedefs | +| per-format codebook LDS that must not cost the others | 5127-5138 | `IQ3GRID : bool` + `@workgroup @template_gate = IQ3GRID iq3s_gridc : uint[512]`; stage from `kq_grid_dev()` (7097-7131), never a constant composite | +| two whole tile bodies in one class | the `KHR` axis 5205-5209 | `static_if (KHR) { khr_tile(..) } else { cm2_tile(..) }` - only the reachable body materializes | +| MoE region walk + split-k | `MoeCmBase.region_rec` 4284-4288 and 5151-5165 | inherit `MoeCmBase`; the `ksplit`/`ybase`/`ptiles` block verbatim; `SplitKReduce` unchanged | +| the four-wide escape hatch | `DECVEC` / `DECV4` 5125-5126 | an int8-lane decode wants the same per-format switch from day one | diff --git a/plans/qwen_arc_board.md b/plans/qwen_arc_board.md new file mode 100644 index 0000000000..39f0c51ec9 --- /dev/null +++ b/plans/qwen_arc_board.md @@ -0,0 +1,202 @@ +# The Qwen arc board - every downloaded Qwen text model, one at a time, smallest first + +The arc's working memory: which Qwen GGUF on the reference box works on the Vulkan tier, which +does not, and how each row stands against llama.cpp on the same box state. A model that does +not work is fixed before the next one is measured. The board is appended as the walk goes; the +final PR body of the arc quotes it. + +## The rig + +- Box: RTX 5060 Ti 16 GB (16311 MiB), Windows 11, driver 616.56, 16 compute lanes; the desktop + holds ~0.65 GB of dedicated video memory fresh (a browser and a chat client took it to 2.9 GB). +- Models: `D:/Work/llama.cpp/models/`. Reference: llama.cpp b10660, `build-vulkan-357` + (`llama-bench -p 512 -n 128 -ngl 99 -fa 1 -t 16 -o md`). +- Ours: `benchmarks/lcpp_bench.das -- -m -o md --for-debug-purposes` under + `DASLLAMA_GPU=1 DASLLAMA_GPU_MIN_CTX=2048 DASLLAMA_ALLOW_UNTUNED=1 DAS_JOBQUE_THREADS=16`, no + VRAM pin (the resident plan sizes itself against the OS's room). Debug-jit grade: the CPU tune + sidecar is stale on this box, which the GPU rows do not read. +- Arms: **cm2** = NV_cooperative_matrix2 (the default on both engines); **KHR** = cooperative + matrix without cm2 (ours `DASLLAMA_COOPMAT=mm`, theirs `GGML_VK_DISABLE_COOPMAT2=1`). The + no-coopmat arm (`DASLLAMA_COOPMAT=sdot4 DASLLAMA_VK_FA=0` / `GGML_VK_DISABLE_COOPMAT=1`) is the + end-of-arc sweep, `modules/dasLLAMA/followup_vulkan.md` item 42. +- "Works" = the resident driver arms (or declines with a named reason the per-op rails then serve), + the bench's sanity gate passes (finite logits, the argmax the cm2 arm printed), and the rows + come out. Parity = same or better; below 1.0x needs a stated reason. +- Script: the session scratchpad's `bench_model.sh ` runs the four rows and writes + `model_.md`; the numbers land here by hand. + +## The roster (text decoders; the mmproj / ASR / Omni companions are other arcs) + +| # | model | GB | arch | status | ours cm2 pp / tg | llama.cpp cm2 pp / tg | ratio | ours KHR pp / tg | llama.cpp KHR pp / tg | +|---|---|---|---|---|---|---|---|---|---| +| 1 | Qwen2.5-0.5B-Instruct-Q8_0 | 0.53 | qwen2 | works (fixed: the q/k/v bias arm) | 42355 / 411.8 | 32338 / 381.9 | 1.31 / 1.08 | 33375 / 411.6 | 25437 / 365.9 | +| 2 | Qwen3.5-0.8B-Q4_K_M (local requant, the hybrid test twin) | 0.53 | qwen35 hybrid | works | 16651 / 314.2 | 16752 / 280.0 | 0.99 / 1.12 | 8580 / 314.8 | 14760 / 280.3 | +| 3 | Qwen3-0.6B-Q8_0 | 0.64 | qwen3 | works; tg 0.95x = the dispatch count per token (see notes) | 29065 / 318.8 | 22736 / 335.9 | 1.28 / 0.95 | 21187 / 319.8 | 20919 / 330.6 | +| 4 | Qwen2.5-1.5B-Instruct-IQ3_XS | 0.73 | qwen2 (bias) + iq2s/iq3xxs planes | works; pp 0.91x = the grid formats' cm2 decode callbacks | 12517 / 264.7 | 13830 / 252.0 | 0.91 / 1.05 | 4482 / 264.5 | 10809 / 245.9 | +| 5 | Qwen3.5-0.8B-Q8_0 | 0.81 | qwen35 hybrid | works | 19222 / 272.5 | 16624 / 243.6 | 1.16 / 1.12 | 17007 / 274.0 | 14978 / 242.6 | +| 6 | Qwen2.5-1.5B-Instruct-Q8_0 | 1.65 | qwen2 | works; tg 0.93x (see notes) | 14080 / 180.1 | 14184 / 193.1 | 0.99 / 0.93 | 11309 / 181.2 | 10813 / 189.0 | +| 7 | Qwen3-4B-Instruct-2507-Q4_K_M | 2.50 | qwen3 | works; tg 0.93x (see notes); KHR re-measured after the kq tile and its slab fix (`followup_vulkan.md` item 42) | 5150 / 117.6 | 5142 / 126.3 | 1.00 / 0.93 | 3051 / 117.4 | 4221 / 125.7 | +| 8 | Qwen3-4B-Instruct-2507-Q5_K_M | 2.89 | qwen3 | works; tg 0.93x (see notes) | 4700 / 104.8 | 4965 / 113.2 | 0.95 / 0.93 | 1489 / 104.8 | 4015 / 111.3 | +| 9 | Qwen3-4B-Instruct-2507-Q6_K | 3.31 | qwen3 | works | 4566 / 95.88 | 4762 / 97.50 | 0.96 / 0.98 | 1581 / 95.79 | 3911 / 96.81 | +| 10 | Qwen3-4B-Instruct-2507-Q8_0 | 4.28 | qwen3 | works; tg 0.94x (see notes) | 5907 / 79.84 | 5064 / 85.03 | 1.17 / 0.94 | 5304 / 79.70 | 4614 / 84.14 | +| 11 | Qwen3.5-4B-Q8_0 | 4.48 | qwen35 hybrid | works (fixed: the mirror's binding cap on a hybrid) | 5087 / 74.37 | 4120 / 74.78 | 1.23 / 0.99 | 4562 / 74.74 | 3657 / 74.46 | +| 12 | Qwen3.5-9B-IQ4_XS | 5.47 | qwen35 hybrid | works (re-measured no pin, ctx 262143) | 2700 / 62.94 | 2794 / 66.82 | 0.97 / 0.94 | 738 / 63.17 | 2199 / 66.26 | +| 13 | Qwen3.5-9B-MTP-Q5_K_M | 6.64 | qwen35 hybrid | works | 2743 / 57.29 | 2752 / 59.67 | 1.00 / 0.96 | 728 / 57.64 | 2102 / 59.41 | +| 14 | Qwen3.5-9B-MTP-UD-Q5_K_XL | 6.87 | qwen35 hybrid | works (re-measured no pin; pp at 10 reps - the 5-rep run drifted, see notes) | 2698 / 56.40 | 2783 / 58.12 | 0.97 / 0.97 | 767 / 56.72 | 2098 / 57.86 | +| 15 | Qwen3.5-9B-Q8_0 | 9.53 | qwen35 hybrid | works (re-measured no pin, ctx 174167) | 3227 / 43.82 | 2799 / 43.99 | 1.15 / 1.00 | 2140 / 43.58 | 1871 / 43.95 | +| 16 | Qwen3.5-9B-MTP-Q8_0 | 9.79 | qwen35 hybrid | works | 3233 / 43.82 | 2797 / 44.03 | 1.16 / 1.00 | 2150 / 43.62 | 1873 / 44.00 | +| 17 | Qwen3.8-27B.i1-IQ3_S | 12.60 | qwen35 hybrid | works (re-measured no pin, ctx 30623; the pinned run read 752 / 24.95) | 802.9 / 27.51 | 807.9 / 24.67 | 0.99 / 1.12 | 238.9 / 27.39 | 650.6 / 24.65 | +| 18 | Qwen3.8-27B-UD-Q3_K_XL | 13.15 | qwen35 hybrid | works (re-measured no pin, ctx 24651; the 5-rep run caught the stall at 803.9 +-33.8 / 25.29 - the row carries the flat 20-rep profiled run, see notes) | 840.0 / 24.92 | 812.6 / 24.88 | 1.03 / 1.00 | 228.2 / 25.15 | 675.4 / 24.80 | +| 19 | Qwen3.8-27B-UD-IQ4_XS | 14.25 | qwen35 hybrid | works, no pin; KHR re-measured after the kq tile; the slab fix's re-measure is owed - with 1.2 GB held by other processes the plan's KV room fell under the 2048 minimum and the driver declined (`followup_vulkan.md` item 42) | 865.4 / 23.51 | 814.6 / 24.13 | 1.06 / 0.97 | 395.2 / 23.51 | 675.2 / 24.07 | +| 20 | Qwen1.5-MoE-A2.7B-Chat.Q8_0 | 15.23 | qwen2moe | works (fixed: the per-op attention chain's bias arm + a 2048-wide kv cap); LAGS on the per-op MoE prefill structure and a span without a shared-expert arm (see notes; first run 390.2 / 29.31 = 0.11 / 0.56) | 506.1 / 39.04 | 3507 / 52.15 | 0.14 / 0.75 | 525.5 / 39.94 | 2185 / 51.84 | +| 21 | Qwen3-30B-A3B-Instruct-2507-Q4_K_M | 18.56 | qwen3moe | works on the per-op tier: experts of layers [13..48) resident, [0..13) streamed; llama.cpp -ngl 99 OOMs, its cells are the same-split offload (experts 0-12 on the CPU; -ngl 36 in the notes) | 749.8 / 66.45 | 612.5 / 42.24 | 1.22 / 1.57 | 499.9 / 64.39 | 588.6 / 42.49 | +| 22 | Qwen3-Coder-30B-A3B-Instruct-Q4_K_M | 18.56 | qwen3moe | | | | | | | +| 23 | Qwen3.8-27B-Q4_K_M | 18.97 | qwen35 hybrid | does not fit 16 GB on either engine (llama.cpp OOM) | | | | | | +| 24 | Qwen3.6-35B-A3B-MTP-UD-Q4_K_M | 22.13 | qwen3moe hybrid | works on the per-op tier: deltanet triples of 30 layers, attention quads of 10, experts of [16..40) resident, [0..16) streamed; llama.cpp -ngl 99 OOMs, its cells = the same-split offload (experts 0-15 on the CPU; -ngl 24 in the notes) | 657.6 / 36.39 | 414.1 / 36.76 | 1.59 / 0.99 | 422.3 / 37.03 | 393.4 / 37.06 | +| 25 | Qwen3-Coder-30B-A3B-Instruct-Q8_0 | 32.48 | qwen3moe | works on the per-op tier: attention quads of 48 layers, experts of [29..48) resident, [0..29) streamed; llama.cpp cells = the same-split offload (experts 0-28 on the CPU; -ngl 20 read 201.0 / 17.38) | 279.1 / 30.64 | 210.5 / 22.27 | 1.33 / 1.38 | 276.8 / 29.82 | 202.7 / 22.65 | +| 26 | Qwen3.6-35B-A3B-Q8_0 | 36.90 | qwen3moe hybrid | works on the per-op tier: 30 deltanet triples, 10 attention quads, experts of [27..40) resident, [0..27) streamed; llama.cpp cells = the same-split offload (-ngl 15 read 173.5 / 14.66); the KHR arm's prefill argmax is a near-tie flip (198 at 8.333 against 220 at 8.355), decode argmax and logits match | 221.4 / 28.61 | 177.8 / 25.84 | 1.25 / 1.11 | 239.0 / 28.90 | 176.3 / 26.16 | +| 27 | Qwen3-Coder-Next-Q4_K_M | 48.53 | qwen3next | works on the per-op tier: 132 dense planes, 48 shared experts, 36 deltanet triples, 12 attention quads resident, experts of [37..48) resident, [0..37) streamed; the FIRST run died out of host heap in the image mint (see notes); llama.cpp cells = the same-split offload (-ngl 13 read 115.4 / 13.88); both arms' prefill argmax are whitespace near-ties (198 vs 197), decode argmax matches | 381.3 / 27.76 | 118.1 / 22.14 | 3.23 / 1.25 | 155.8 / 27.62 | 116.4 / 22.50 | + +The 9B rows 12/14/15 and the 27B rows 17/18 were measured 2026-09-06 with the grid-format GEMV +fix and the memory-priority flip already in; the 27B rows 17 and 18 ran under a VRAM pin below +the box's knee (the auto arm without the OS query declined them then). Row 19 is the no-pin +auto arm of 2026-09-07. Ratios read ours over llama.cpp. + +## Per-model notes + +### 1. Qwen2.5-0.5B-Instruct-Q8_0 +First run: the resident driver declined the whole qwen2 family ("no arm for: attention qkv +bias") and the per-op rails served at 1228 / 104.7 (0.04x / 0.28x). Fix: the q/k/v projection +bias folded into the rope stage (decode, prefill, the fused qk-norm twin), the bias rows +uploaded once, the seat installed separately so a tier without it still declines by name +(`ARCHITECTURE_GPU_VULKAN_DECODE.md`, the bias paragraph). Second run found the demotion +check's counter query costing a prefill's worth of time on an 18 ms prefill - gated to once a +second. Parity gate: `tests/test_gpu_resident_qwen2.das` (forced-feed logits within 6% of the +CPU chain at one and two windows; the noise sits flat at 0.30-0.50 of a 10-18 max logit). + +### 3. Qwen3-0.6B-Q8_0 +Decode profile: GPU 2870 us/token (attn 550, gate 393, q 328, down 315, tail 387, the rest +small) against a host wall of 3141 us/token - the GPU work already matches llama.cpp's 2.98 ms +token; the 8.6% gap is the per-token host path (two memcpys, the submit, the fence wait, the +608 KB logits copy). Prefill reps on a ~20 ms window swing +-25-40% between runs (the mean at +or above llama.cpp); the same rig reads +-0.1% on the 27B. + +### 6-10. The 1.5B and 4B dense rows' decode (0.93-0.98x) +Profile on Qwen3-4B-Instruct-2507-Q4_K_M, both engines, per token: ours GPU 8232 us under an +8545 us host wall; llama.cpp's per-op logger sums 8160 us against its 7.92 ms free-running token. +The GEMVs favor us - qkv 904 vs 1292, gate+up 2530 vs 2652, down 1582 vs 1606, classifier 752 vs +750 - and pay 403 us of activation requant llama.cpp does not have (net +131 us ours). Three fixed +costs carry the gap: decode attention 650 vs 306 us (18 vs 8.5 us per layer at a short context - +a per-dispatch floor, the 0.6B reads the same 19.6 us/layer), the residual add + norm 601 vs 312 +us (theirs folds the add into the wo/down GEMV: MUL_MAT_ADD), and the host path 313 us (two +memcpys, the submit, the fence wait, the logits copy). Together ~0.9 ms of an 8.5 ms token; the +same three sit near 1% each on the 27B. Q6_K reads 0.98x because llama.cpp's q6_K GEMV runs at +963 GFLOPS against 1326 for q4_K. Levers for after the walk: the attention kernel's per-layer +floor, the fused residual add, a fence poll in place of the blocking wait. + +### 12-16. The 9B block, and the prefill rep swing +Every 9B file arms without a pin (ctx 262143 on the IQ4_XS - the binding cap - down to 174167 +on the Q8_0 files, the VRAM fit). Prefill on our side swings between runs on some files and not +others: UD-Q5_K_XL read 2537 +-132 at 5 reps, then 2667 +-48 at 10 reps (five reps at 2698-2708, +then 2620, 2644, 2661, 2672, 2561 - a step down and a drift over the second second of load), then +2698 +-8 at 10 reps under a 250 ms clock sampler (SM 2760-2820 MHz, 122-165 W of the 180 W limit, +50-56 C, no throttle bit, llama.cpp's window the same clocks). Decode never moves (56.3-56.6 across +30 reps). The demotion check is not it (the whole OS memory query costs 45 us). The 27B Q3_K_XL +shows the event's shape at 20 reps: 845 844 842 845 844 | 516 | 786 785 831 828 830 816 826 828 +832 833 833 844 843 844 - ONE rep loses 0.39 s outright, then the rate climbs back over ten reps; +decode over the same 20 reps sits at 25.25-25.28. Under the per-window profile the same file ran +20 reps flat at 840.0 +-0.9 (host prep 1.0-1.5 ms, record 2.7-2.9 ms, GPU 585 ms per window, every +window alike), and the box has 256 GB of RAM with 236 GB free, so neither the host chain nor memory +pressure is the stall. Seen in 3 of ~12 rows (12, 14, 18), twice at the sixth timed prefill, never +under either sampler run. Open: a one-shot stall of ~0.4 s with a paging-shaped recovery - an +interfering GPU client, or the memory manager taking the idle gap. From row 20 on the rig samples +clocks, utilization and memory.used at 100 ms beside every run of ours, so the next occurrence is +caught. Rows 14 and 18 carry their flat repeat runs. + +### 11. Qwen3.5-4B-Q8_0 +First run: the resident driver declined at the device prepare - "KV mirror (8 x 262144 x 1024) +exceeds maxStorageBufferRange 4294967295" - and the per-op rails served with the dense FFN of 32 +layers on the CPU: 254.9 / 15.37 (0.06x / 0.21x). The context negotiation read the mirror's +per-position stride from layer 0, which is recurrent on Qwen3.5 (kv width 0), so the stride was +0 and the binding cap was skipped; the file's 262144 context then put one mirror side exactly +one byte past the 4 GiB range. The 0.8B hybrid armed at 262144 only because its side is 1536 MB; +the 9B's weights push it down the VRAM path first. Fix: `resident_binding_ctx` sums the rows +across the layers (a recurrent layer adds 0), cited in `ARCHITECTURE_GPU_VULKAN_RESIDENCY.md` +2.2n; model-free cell in `tests/test_gpu_serving_declines.das`. + +### 20. Qwen1.5-MoE-A2.7B-Chat.Q8_0 +The first MoE row, served by the per-op tier (the resident driver is dense-only by design). Two +causes for 0.11x / 0.56x, both named in the load: "the attention-quad rail declined - no arm for: +attention qkv bias" - every layer's attention runs on the CPU with a device round trip per layer +(the resident driver's bias arm from row 1 lives in the rope kernels the per-op chain also binds, +but the chain still binds the placeholder and the prefill's `AtPrep` stage has no bias arm at all; +`attn_dec_shape_ok` and the rail's `arch_ok` both decline on the flag) - and "expert stacks of +layers [2..24) resident, [0..2) streamed": 14.5 GB of Q8 experts on a 16 GB card, the per-op tier's +budget keeping the 1 GiB reserve llama.cpp does not (its 14.17 GiB image sits at the brim at 3584 / +52.09). Sanity argmax matches across arms. The fix is the per-op attention chain's bias arm: the +hook signatures gain the layer's bias rows beside rmsq/rmsk, the decode chain binds the rows and +sets boff/hasb, `AtPrep` gains the binding and the add before rope, the rail and the shape gate +drop the flag when the seat is installed; cells in test_vulkan_dec_tail (bias arm) and the AtPrep +kernel unit. The family is qwen2moe (Qwen1.5-MoE, Qwen2-57B-A14B), gpt-oss and glm4moe also carry +the flag. DONE (Boris pulled it forward): the hooks carry the layer's bias rows (the prefill hook +drops the derivable kv_mul to stay under daslang's 32-argument invoke cap), the decode block binds +the row to the rope kernels' existing arm, `AtPrep` gains the binding with a third in-place pass +over the raw v window, the span carries the rows too; model-free arms in the kernel prep cell, +the tier's attention chain and the decode tail. The next gate then fired - "geometry outside the +chain (hs 128, qd 2048, kvd 2048)": 16 heads of 128 with no GQA is a 2048-wide kv, the chain's +planes were sized for 1024 - so `AT_MAX_KV` is 2048 (~48 MB more for the k/v planes and their f16 +shadows). With both: attention quads of 24 layers resident, 506.1 / 39.04 (0.14x / 0.75x), sanity +argmax unchanged. What remains is structural: prefill rides the per-op tier's per-stage synchronous +chain (attention submit, CPU kv store + routing + gather, FFN submit, per layer) at ~1.7 TFLOPS +effective against llama.cpp's whole graph on the device (its 14.17 GiB image fits where our 1 GiB +reserve streams 3-4 layers); decode runs per-layer submits because the whole-token span declines a +shared expert (`span_model_ok`: `n_ff_shexp > 0`) - a shared-expert stage in the span is the lever +for tg, the MoE prefill chain is an arc-level item (the resident driver is dense-only). + +### 21. Qwen3-30B-A3B-Instruct-2507-Q4_K_M (and the MoE reference protocol) +An 18.5 GB file on a 16 GB card: llama.cpp at `-ngl 99` fails the load (a 1 GB device allocation +refused), so its rows are the two offload settings the 30B arc settled on: the experts of layers +0-12 on the CPU (`-ot 'blk\.([0-9]|1[0-2])\.ffn_.*_exps=CPU'`, the split that mirrors our streamed +layers) - cm2 612.5 / 42.24, KHR 588.6 / 42.49 - and `-ngl 36`, their best decode then - cm2 597.7 / +45.53, KHR 566.1 / 45.55. Ours 749.8 / 66.45 (cm2) is 1.22x / 1.57x of the same-split row and 1.25x +/ 1.46x of their best decode; when that arc merged this file read 348.4 / 35.5 against their 566.4 / +37.0. The KHR arm's 499.9 is the kq-tile gap again (0.85x of their KHR same-split row); decode 1.52x. +Every MoE row past this one that does not fit at -ngl 99 uses the same two settings. + +### 22 and 24. Qwen3-Coder-30B-A3B Q4_K_M, Qwen3.6-35B-A3B-MTP UD-Q4_K_M +The Coder twin of row 21 reads the same: 745.3 / 65.50 against 608.8 / 40.65 same-split (1.22x / +1.61x) and 582.0 / 45.14 at -ngl 36; KHR 494.3 / 63.60 against 572.6 / 41.81. The 35B hybrid (22 GB, +30 deltanet + 10 attention layers) streams 16 layers' experts: 657.6 / 36.39 against llama.cpp's +same split 414.1 / 36.76 (1.59x / 0.99x) and against -ngl 24, which is worse for them (346.3 / +21.23: the layer offload takes the recurrent layers off the device too, the expert split keeps +them on). KHR 422.3 / 37.03 against 393.4 / 37.06 (1.07x / 1.00x) - the kq-tile gap narrows here +because the streamed half of the prefill is PCIe-bound on both arms. + +### 25-27. The files past the card (32-48 GB) +All three run on the per-op tier with most layers' experts streamed from pinned host memory, and +all three stay ahead of llama.cpp at the matching split (its `-ot` puts those experts on the CPU): +the Coder-30B Q8 1.33x / 1.38x (29 of 48 layers streamed), the 35B Q8 1.25x / 1.11x (27 of 40), +and Coder-Next Q4 3.23x / 1.25x (37 of 48). The Coder-Next arithmetic says why the last one is +so far ahead: 512 experts of which 10 fire, 1.6 MB each in Q4, so a token streams ~590 MB across +37 layers - 27 ms at the slot's ~22 GB/s - and a 512-row window reads each streamed stack once, +30 GB in 1.34 s; llama.cpp's CPU pass over the same experts takes 4.3 s. None of these is an +interactive configuration on this card (the Q8 files read 28-31 tok/s, the interactive picks are +the Q4 30B/35B rows at 66 / 36 tok/s); they prove the zoo runs and where the bus binds. The KHR +arm's prefill gap widens with streaming (Coder-Next 155.8 against 381.3): the streamed stacks are +kq planes, which had no KHR tile when these rows were taken, and their GEMMs are the whole window. Both arms' prefill argmax +on the biggest files are whitespace near-ties (a space, a newline, a tab within 0.05 logits); +decode argmax and logits match across arms on every row. + +The Coder-Next's first run died: "out of heap memory, requested 91 GB, used 232 GB" on a 256 GB +box, because D: had filled with the day's 37 debug-flavor `.dlim` images (293 GB), the image save +aborted mid-plane, and the mint path then loaded the model a second time beside the first +(`followup_general.md` item 120; the rig runs `DASLLAMA_IMAGE=0` since). The re-run loaded once, +in 13 s from the page cache. + +### 19. Qwen3.8-27B-UD-IQ4_XS +Resident image 12899 MB (+1.4% over the file's blocks). Decode profile against llama.cpp's +per-op logger: GEMVs 37.7 vs 37.0 ms/token (gate+up faster than theirs, the beta/alpha pair 0.46 +ms slower - the long-rows-few-outputs GEMV shape), small ops equal; the residual add is fused +into their down/out mat-vecs. Levers left: the beta/alpha GEMV shape, the fused residual add +(~1% each). On the KHR arm prefill was 0.33x before the kq formats had a KHR-coopmat tile, 0.59x +with it, and the row above is the tile after its slab fix (`followup_vulkan.md` item 42). diff --git a/plans/shader_emitter_followups.md b/plans/shader_emitter_followups.md index c12db812de..1dda56dd49 100644 --- a/plans/shader_emitter_followups.md +++ b/plans/shader_emitter_followups.md @@ -34,3 +34,22 @@ the binding rules in `modules/REVIEW_SHADER_EMITTERS.md`. This file holds what i retires to "weakening that note is a defect". Done = the operators and `select` documented in `skills/daslang/`, the emitter heuristic gone, the census sites converted with the two measurements beside them. + +2. **`for [unroll]` unrolls at emission; a fixed-array local with constant indices becomes + registers.** Found 2026-09-07 by the integer GEMM tile prototypes + (`modules/dasLLAMA/harness/vk_gemm_probe.das -- mmqx`, `modules/dasLLAMA/followup_vulkan.md` + item 42): dasSpirv emits `for [unroll] (i in range(N))` as a loop carrying the `Unroll` loop + control and a `var acc : float[64]` local as a Function-storage `OpVariable` indexed by the + loop counter - a register block written as an array runs through local memory unless the + driver both unrolls and scalarizes, and the 5060 Ti's measured the same rate with the block + as named scalars, which says the shape, not the array, capped that kernel, but the array form + is what a 128-accumulator block (the reference exe's 4 x 32 register block) needs to be + writable at all. The plan: dasSpirv clones the body N times for a constant `range(N)` with the + counter bound to `OpConstant`, so every `arr[expr(i)]` chains a constant index (SROA-friendly in + every driver), and reports the unroll it performed in the same note channel item 1 gives the + branched operators; dasMetal needs nothing - MSL's `#pragma unroll` and the Metal compiler's + scalarization already do this, which is why the Metal GEMV twins carry `sumf : float[NR]` + arrays. Gate: a `tests/spirv` fixture pinning zero `OpLoopMerge` under an unrolled body and a + constant-index `OpAccessChain` per element, the kernel suite byte-identical elsewhere, and the + `mmqx` probe's ceiling twin re-measured with the block as an array. + Done = the fixture, the note, and the probe row. diff --git a/skills/internal/make_pr.md b/skills/internal/make_pr.md index 57660602d6..9d8bc38fc0 100644 --- a/skills/internal/make_pr.md +++ b/skills/internal/make_pr.md @@ -13,9 +13,10 @@ skips the chain; exit 2 names the red gate. This file is the authority on fix po the tool prints as STILL YOURS. Commit, run it, push once (one batched PR). **The full preflight runs ONCE per PR - never a second full run.** On failure fix everything, -validate each fix with the **targeted** gate or an isolated repro (`--only `, the -failing test slice, a scratch probe - minutes, not tens of minutes), say so in your summary, -and let CI validate the tip. Later fix commits - its own findings, Copilot/CI rounds +validate each fix with the **targeted** gate or an isolated repro (`make-pr --only ` for +its own gates, `utils/internal/preflight/main.das -- --only ` for a preflight lane - the +lanes a red fast tier skipped included, each run once - the failing test slice, a scratch +probe - minutes, not tens of minutes), say so in your summary, and let CI validate the tip. Later fix commits - its own findings, Copilot/CI rounds (`skills/internal/babysit.md`) - do NOT re-trigger it. **The tree is FROZEN while the chain runs** - no commits, no reconfigure, no `cmake diff --git a/tests/module_cache/test_default_cache_path.das b/tests/module_cache/test_default_cache_path.das index 71f4e4f265..bc2c3a5ea5 100644 --- a/tests/module_cache/test_default_cache_path.das +++ b/tests/module_cache/test_default_cache_path.das @@ -15,6 +15,15 @@ def das_exe() : string { return empty(args) ? "" : args[0] } +//! an AOT-consuming host (`test_aot --use-aot`) installs no default cache, and spawned as a child it +//! runs the AOT'd program, not the script - no cache marker ever prints; the cell skips there +def aot_host_skips(t : T?) : bool { + let hostArgs <- get_command_line_arguments() + if (find_index(hostArgs, "--use-aot") < 0) return false + t |> skip("an AOT-consuming host installs no default cache and runs no child script") + return true +} + //! the child shares dastest's cwd (argv[0] is a cwd-relative interpreter path), so the default //! cache lands in this tree's own .jitted_scripts/module_cache - the test removes what it wrote. //! Commands stay unquoted: a leading quote breaks Windows _popen (cmd /c quote stripping) @@ -93,6 +102,7 @@ def quiet(out : string) : bool { //! an explicit -module-cache says how many modules the cache served and where the cutoff fell [test] def test_fallback_verdict_names_the_cutoff(t : T?) { + if (aot_host_skips(t)) return var terr : string let tmp = create_temp_directory("das_mc_verdict", terr) if (empty(tmp)) { @@ -120,6 +130,7 @@ def test_fallback_verdict_names_the_cutoff(t : T?) { //! the same explicit cache read by a -no-lint run reparses every module in place and is rewritten [test] def test_policy_change_refuses_the_cache(t : T?) { + if (aot_host_skips(t)) return var terr : string let tmp = create_temp_directory("das_mc_policy", terr) if (empty(tmp)) { @@ -149,6 +160,7 @@ def test_policy_change_refuses_the_cache(t : T?) { //! cache, and a same-size edit is a cutoff - the stamp never reads mtime [test] def test_source_stamp_is_content(t : T?) { + if (aot_host_skips(t)) return var terr : string let tmp = create_temp_directory("das_mc_stamp", terr) if (empty(tmp)) { @@ -180,6 +192,7 @@ def test_source_stamp_is_content(t : T?) { //! same derivation, so a -v1syntax host serves a gen2 module instead of rewriting the cache every run [test] def test_source_derived_syntax_policy_serves(t : T?) { + if (aot_host_skips(t)) return var terr : string let tmp = create_temp_directory("das_mc_gen2", terr) if (empty(tmp)) { @@ -274,6 +287,7 @@ def scramble_header(path : string) { //! warm run of a module whose only defence against a lint finding is the marker must still compile [test] def test_source_lines_survive_the_cache(t : T?) { + if (aot_host_skips(t)) return var terr : string let tmp = create_temp_directory("das_mc_src", terr) if (empty(tmp)) { @@ -317,11 +331,7 @@ def check_damaged_cache(t : T?; base, a : string) { //! the AST module cache is on by default - silent, at a path keyed by script, host flags and DAS* env - and the flags that turn it off or make it loud [test] def test_default_module_cache(t : T?) { - let hostArgs <- get_command_line_arguments() - if (find_index(hostArgs, "--use-aot") >= 0) { - t |> skip("an AOT-consuming host never installs the default cache") - return - } + if (aot_host_skips(t)) return var terr : string let tmp = create_temp_directory("das_mc_default", terr) if (empty(tmp)) { @@ -416,11 +426,7 @@ def record_size_in(dir, stem : string) : uint64 { //! explicit -module-cache path is never pruned [test] def test_default_cache_evicts_to_the_limit(t : T?) { - let hostArgs <- get_command_line_arguments() - if (find_index(hostArgs, "--use-aot") >= 0) { - t |> skip("an AOT-consuming host never installs the default cache") - return - } + if (aot_host_skips(t)) return var terr : string let tmp = create_temp_directory("das_mc_evict", terr) if (empty(tmp)) { diff --git a/tests/msl/REVIEW.md b/tests/msl/REVIEW.md index e576b04203..d7f8a909e6 100644 --- a/tests/msl/REVIEW.md +++ b/tests/msl/REVIEW.md @@ -1,7 +1,8 @@ # MSL emitter fixtures Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: -`modules/dasMetal/ARCHITECTURE.md`. +`modules/dasMetal/ARCHITECTURE.md`. Shared emitter rules: `modules/REVIEW_SHADER_EMITTERS.md` - +apply that list with this one. **Weakening the census check is a defect.** The check runs both directions: every kind in `declared_msl_census` is emitted by some fixture, and every kind any fixture emits is declared diff --git a/tests/spirv/REVIEW.md b/tests/spirv/REVIEW.md index d28aea1835..5fa128e4a3 100644 --- a/tests/spirv/REVIEW.md +++ b/tests/spirv/REVIEW.md @@ -2,8 +2,7 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `modules/dasSpirv/ARCHITECTURE.md`. Shared emitter rules: `modules/REVIEW_SHADER_EMITTERS.md` -(repo root) - apply that list with this one. The emitter's own checklist, -`modules/dasSpirv/REVIEW.md`, binds a diff that touches the emitter with this one. +- apply that list with this one. **Weakening `test_census.das` is a defect** - it holds every fixture opcode declared and every declared opcode emitted. @@ -11,3 +10,8 @@ declared opcode emitted. **A diff that adds or changes a fixture asserting a claim `validate_spirv` would check also asserts that claim on the emitted words.** `validate_spirv` skips where the tool is absent or predates an extension, so a spirv-val-only assertion can check nothing. + +**A fixture cell that validates its words asserts the module version on `words[1]` in the same +cell, and passes a `target_env` - directly or through the file's `validate` wrapper - that +admits that version: `vulkan1.2` for 1.4 and 1.5, `vulkan1.3` for 1.6.** The default, +`vulkan1.1`, admits SPIR-V 1.3 at most. diff --git a/tests/spirv/_fail_closed/_fc_decode_helper_copy.das b/tests/spirv/_fail_closed/_fc_decode_helper_copy.das new file mode 100644 index 0000000000..8c05b204fc --- /dev/null +++ b/tests/spirv/_fail_closed/_fc_decode_helper_copy.das @@ -0,0 +1,35 @@ +// Fail-closed fixture: a [spirv_decode] method the kernel calls on plane elements is called on a +// copy from a helper the kernel reaches - the element form is fixed per method, so the copy is +// refused. `_`-prefix + `expect 50501` keep dastest/lint off the failing shader. +expect 50501 + +options gen2 + +require spirv/spirv_shader +require spirv/spirv_builtins +require daslib/shader_lingua_franca + +struct FcBlk3 { + qs : int16[16] +} + +class FcDecodeHelperCopy { + @ssbo @binding = 0 wq : array + @ssbo @binding = 1 y : array + + [spirv_decode] + def decode(blk : FcBlk3; bc, cib : uint2) : float16 { + return float16(float(int(blk.qs[int(cib.y & 15u)]))) + } + + def first_lane(gid : uint) : float { + let blk = wq[0] + return float(decode(blk, uint2(0u, 0u), uint2(0u, gid & 31u))) + } + + [spirv_kernel(local_size_x = 64, name = "fc_decode_helper_copy_spv")] + def run { + let gid = gl_GlobalInvocationID.x + y[gid] = float(decode(wq[gid >> 5u], uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + first_lane(gid) + } +} diff --git a/tests/spirv/_fail_closed/_fc_decode_index64.das b/tests/spirv/_fail_closed/_fc_decode_index64.das new file mode 100644 index 0000000000..953de014d0 --- /dev/null +++ b/tests/spirv/_fail_closed/_fc_decode_index64.das @@ -0,0 +1,30 @@ +// Fail-closed fixture: the index of a plane element passed to a [spirv_decode] method must be int +// or uint - the callee's entry chain takes a 32-bit index, so a 64-bit one is refused by name. +// `_`-prefix + `expect 50501` keep dastest/lint off the failing shader. +expect 50501 + +options gen2 + +require spirv/spirv_shader +require spirv/spirv_builtins +require daslib/shader_lingua_franca + +struct FcBlk4 { + qs : int16[16] +} + +class FcDecodeIndex64 { + @ssbo @binding = 0 wq : array + @ssbo @binding = 1 y : array + + [spirv_decode] + def decode(blk : FcBlk4; bc, cib : uint2) : float16 { + return float16(float(int(blk.qs[int(cib.y & 15u)]))) + } + + [spirv_kernel(local_size_x = 64, name = "fc_decode_index64_spv")] + def run { + let gid = gl_GlobalInvocationID.x + y[gid] = float(decode(wq[uint64(gid >> 5u)], uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + } +} diff --git a/tests/spirv/_fail_closed/_fc_decode_mixed.das b/tests/spirv/_fail_closed/_fc_decode_mixed.das new file mode 100644 index 0000000000..46a08a2b8f --- /dev/null +++ b/tests/spirv/_fail_closed/_fc_decode_mixed.das @@ -0,0 +1,32 @@ +// Fail-closed fixture: a [spirv_decode] method called directly takes its block the same way at +// every call - one call on the plane element and one on a copy of it is refused at discovery. +// `_`-prefix + `expect 50501` keep dastest/lint off the failing shader. +expect 50501 + +options gen2 + +require spirv/spirv_shader +require spirv/spirv_builtins +require daslib/shader_lingua_franca + +struct FcBlk { + qs : int16[16] +} + +class FcDecodeMixed { + @ssbo @binding = 0 wq : array + @ssbo @binding = 1 y : array + + [spirv_decode] + def decode(blk : FcBlk; bc, cib : uint2) : float16 { + return float16(float(int(blk.qs[int(cib.y & 15u)]))) + } + + [spirv_kernel(local_size_x = 64, name = "fc_decode_mixed_spv")] + def run { + let gid = gl_GlobalInvocationID.x + let blk = wq[gid >> 5u] + y[gid] = float(decode(wq[gid >> 5u], uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + + float(decode(blk, uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + } +} diff --git a/tests/spirv/_fail_closed/_fc_decode_twoplanes.das b/tests/spirv/_fail_closed/_fc_decode_twoplanes.das new file mode 100644 index 0000000000..ccf1358dd9 --- /dev/null +++ b/tests/spirv/_fail_closed/_fc_decode_twoplanes.das @@ -0,0 +1,32 @@ +// Fail-closed fixture: a [spirv_decode] method called directly on plane elements takes them from +// ONE plane - the callee's entry chains through that plane, so elements of two planes are refused. +// `_`-prefix + `expect 50501` keep dastest/lint off the failing shader. +expect 50501 + +options gen2 + +require spirv/spirv_shader +require spirv/spirv_builtins +require daslib/shader_lingua_franca + +struct FcBlk2 { + qs : int16[16] +} + +class FcDecodeTwoPlanes { + @ssbo @binding = 0 wq : array + @ssbo @binding = 1 wq2 : array + @ssbo @binding = 2 y : array + + [spirv_decode] + def decode(blk : FcBlk2; bc, cib : uint2) : float16 { + return float16(float(int(blk.qs[int(cib.y & 15u)]))) + } + + [spirv_kernel(local_size_x = 64, name = "fc_decode_twoplanes_spv")] + def run { + let gid = gl_GlobalInvocationID.x + y[gid] = float(decode(wq[gid >> 5u], uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + + float(decode(wq2[gid >> 5u], uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + } +} diff --git a/tests/spirv/_gen_golden.das b/tests/spirv/_gen_golden.das index f1aa03beb3..2b675ddc37 100644 --- a/tests/spirv/_gen_golden.das +++ b/tests/spirv/_gen_golden.das @@ -24,7 +24,7 @@ def emit_golden(dir, name : string; words : array) { } [export] -def main { +def main { // nolint:STYLE038 — one emit per golden fixture: a flat list, one line each let dir = "{get_das_root()}/tests/spirv/_golden" mkdir(dir) emit_golden(dir, "square", square_words()) @@ -116,4 +116,7 @@ def main { emit_golden(dir, "coopmat_i8", coopmat_i8_words()) // coopmat from @workgroup shared memory (the tiled-GEMM staging path) emit_golden(dir, "coopmat_sm", coopmat_sm_words()) + emit_golden(dir, "spdirect", spdirect_words()) + emit_golden(dir, "spelem", spelem_words()) + emit_golden(dir, "spelemi", spelemi_words()) } diff --git a/tests/spirv/_golden/spdirect.txt b/tests/spirv/_golden/spdirect.txt new file mode 100644 index 0000000000..e30265b802 --- /dev/null +++ b/tests/spirv/_golden/spdirect.txt @@ -0,0 +1,131 @@ +; SPIR-V module: 511 words, id-bound 0x5e +OpCapability 0x1 +OpCapability 0x1151 +OpCapability 0x16 +OpCapability 0x9 +OpCapability 0x27 +OpMemoryModel 0x0 0x1 +OpEntryPoint 0x5 0x21 0x6e69616d 0x0 0x19 0x9 0xe 0x13 0x16 +OpExecutionMode 0x21 0x11 0x40 0x1 0x1 +OpDecorate 0x4 0x6 0x2 +OpMemberDecorate 0x5 0x0 0x23 0x0 +OpDecorate 0x6 0x6 0x20 +OpDecorate 0x7 0x2 +OpMemberDecorate 0x7 0x0 0x23 0x0 +OpDecorate 0x9 0x22 0x0 +OpDecorate 0x9 0x21 0x0 +OpDecorate 0xb 0x6 0x2 +OpDecorate 0xc 0x2 +OpMemberDecorate 0xc 0x0 0x23 0x0 +OpDecorate 0xe 0x22 0x0 +OpDecorate 0xe 0x21 0x1 +OpDecorate 0x10 0x6 0x4 +OpDecorate 0x11 0x2 +OpMemberDecorate 0x11 0x0 0x23 0x0 +OpDecorate 0x13 0x22 0x0 +OpDecorate 0x13 0x21 0x2 +OpDecorate 0x14 0x2 +OpMemberDecorate 0x14 0x0 0x23 0x0 +OpDecorate 0x19 0xb 0x1c +OpTypeInt 0x1 0x10 0x1 +OpTypeInt 0x2 0x20 0x0 +OpConstant 0x2 0x3 0x10 +OpTypeArray 0x4 0x1 0x3 +OpTypeStruct 0x5 0x4 +OpTypeRuntimeArray 0x6 0x5 +OpTypeStruct 0x7 0x6 +OpTypePointer 0x8 0xc 0x7 +OpVariable 0x8 0x9 0xc +OpTypeFloat 0xa 0x10 +OpTypeRuntimeArray 0xb 0xa +OpTypeStruct 0xc 0xb +OpTypePointer 0xd 0xc 0xc +OpVariable 0xd 0xe 0xc +OpTypeFloat 0xf 0x20 +OpTypeRuntimeArray 0x10 0xf +OpTypeStruct 0x11 0x10 +OpTypePointer 0x12 0xc 0x11 +OpVariable 0x12 0x13 0xc +OpTypeStruct 0x14 0x2 +OpTypePointer 0x15 0x9 0x14 +OpVariable 0x15 0x16 0x9 +OpTypeVector 0x17 0x2 0x3 +OpTypePointer 0x18 0x1 0x17 +OpVariable 0x18 0x19 0x1 +OpTypeArray 0x1a 0x1 0x3 +OpTypeStruct 0x1b 0x1a +OpTypeVector 0x1c 0x2 0x2 +OpTypeFunction 0x1e 0xa 0x1b 0x1c 0x1c +OpTypeVoid 0x1f +OpTypeFunction 0x20 0x1f +OpTypePointer 0x23 0x1 0x2 +OpConstant 0x2 0x25 0x0 +OpTypePointer 0x27 0x9 0x2 +OpTypeBool 0x2a +OpConstant 0x2 0x2e 0x5 +OpTypePointer 0x31 0xc 0x5 +OpTypePointer 0x34 0xc 0xf +OpConstant 0x2 0x37 0x1f +OpTypePointer 0x41 0x7 0x1b +OpTypePointer 0x43 0x7 0x1a +OpConstant 0x2 0x46 0x1e +OpConstant 0x2 0x48 0x1 +OpTypeInt 0x4a 0x20 0x1 +OpTypePointer 0x4c 0x7 0x1 +OpTypeInt 0x4f 0x8 0x1 +OpTypeVector 0x50 0x4f 0x2 +OpTypePointer 0x58 0xc 0xa +OpFunction 0x1f 0x21 0x0 0x20 +OpLabel 0x22 +OpAccessChain 0x23 0x24 0x19 0x25 +OpLoad 0x2 0x26 0x24 +OpAccessChain 0x27 0x28 0x16 0x25 +OpLoad 0x2 0x29 0x28 +OpULessThan 0x2a 0x2b 0x26 0x29 +OpSelectionMerge 0x2c 0x0 +OpBranchConditional 0x2b 0x2d 0x2c +OpLabel 0x2d +OpShiftRightLogical 0x2 0x2f 0x26 0x2e +OpAccessChain 0x31 0x30 0x9 0x25 0x2f +OpLoad 0x5 0x32 0x30 +OpAccessChain 0x34 0x33 0x13 0x25 0x26 +OpShiftRightLogical 0x2 0x35 0x26 0x2e +OpCompositeConstruct 0x1c 0x36 0x25 0x35 +OpBitwiseAnd 0x2 0x38 0x26 0x37 +OpCompositeConstruct 0x1c 0x39 0x25 0x38 +OpCopyLogical 0x1b 0x3b 0x32 +OpFunctionCall 0xa 0x3a 0x1d 0x3b 0x36 0x39 +OpFConvert 0xf 0x3c 0x3a +OpStore 0x33 0x3c +OpBranch 0x2c +OpLabel 0x2c +OpReturn +OpFunctionEnd +OpFunction 0xa 0x1d 0x0 0x1e +OpFunctionParameter 0x1b 0x3d +OpFunctionParameter 0x1c 0x3e +OpFunctionParameter 0x1c 0x3f +OpLabel 0x40 +OpVariable 0x41 0x42 0x7 +OpStore 0x42 0x3d +OpAccessChain 0x43 0x44 0x42 0x25 +OpCompositeExtract 0x2 0x45 0x3f 0x1 +OpBitwiseAnd 0x2 0x47 0x45 0x46 +OpShiftRightLogical 0x2 0x49 0x47 0x48 +OpBitcast 0x4a 0x4b 0x49 +OpAccessChain 0x4c 0x4d 0x44 0x4b +OpLoad 0x1 0x4e 0x4d +OpBitcast 0x50 0x51 0x4e +OpCompositeExtract 0x2 0x52 0x3f 0x1 +OpBitwiseAnd 0x2 0x53 0x52 0x48 +OpBitcast 0x4a 0x54 0x53 +OpVectorExtractDynamic 0x4f 0x55 0x51 0x54 +OpCompositeExtract 0x2 0x56 0x3e 0x1 +OpAccessChain 0x58 0x57 0xe 0x25 0x56 +OpLoad 0xa 0x59 0x57 +OpSConvert 0x4a 0x5a 0x55 +OpConvertSToF 0xf 0x5b 0x5a +OpFConvert 0xa 0x5c 0x5b +OpFMul 0xa 0x5d 0x59 0x5c +OpReturnValue 0x5d +OpFunctionEnd diff --git a/tests/spirv/_golden/spelem.txt b/tests/spirv/_golden/spelem.txt new file mode 100644 index 0000000000..053e6fcdcb --- /dev/null +++ b/tests/spirv/_golden/spelem.txt @@ -0,0 +1,125 @@ +; SPIR-V module: 487 words, id-bound 0x59 +OpCapability 0x1 +OpCapability 0x1151 +OpCapability 0x16 +OpCapability 0x9 +OpCapability 0x27 +OpMemoryModel 0x0 0x1 +OpEntryPoint 0x5 0x1f 0x6e69616d 0x0 0x19 +OpExecutionMode 0x1f 0x11 0x40 0x1 0x1 +OpDecorate 0x4 0x6 0x2 +OpMemberDecorate 0x5 0x0 0x23 0x0 +OpDecorate 0x6 0x6 0x20 +OpDecorate 0x7 0x2 +OpMemberDecorate 0x7 0x0 0x23 0x0 +OpDecorate 0x9 0x22 0x0 +OpDecorate 0x9 0x21 0x0 +OpDecorate 0xb 0x6 0x2 +OpDecorate 0xc 0x2 +OpMemberDecorate 0xc 0x0 0x23 0x0 +OpDecorate 0xe 0x22 0x0 +OpDecorate 0xe 0x21 0x1 +OpDecorate 0x10 0x6 0x4 +OpDecorate 0x11 0x2 +OpMemberDecorate 0x11 0x0 0x23 0x0 +OpDecorate 0x13 0x22 0x0 +OpDecorate 0x13 0x21 0x2 +OpDecorate 0x14 0x2 +OpMemberDecorate 0x14 0x0 0x23 0x0 +OpDecorate 0x19 0xb 0x1c +OpTypeInt 0x1 0x10 0x1 +OpTypeInt 0x2 0x20 0x0 +OpConstant 0x2 0x3 0x10 +OpTypeArray 0x4 0x1 0x3 +OpTypeStruct 0x5 0x4 +OpTypeRuntimeArray 0x6 0x5 +OpTypeStruct 0x7 0x6 +OpTypePointer 0x8 0xc 0x7 +OpVariable 0x8 0x9 0xc +OpTypeFloat 0xa 0x10 +OpTypeRuntimeArray 0xb 0xa +OpTypeStruct 0xc 0xb +OpTypePointer 0xd 0xc 0xc +OpVariable 0xd 0xe 0xc +OpTypeFloat 0xf 0x20 +OpTypeRuntimeArray 0x10 0xf +OpTypeStruct 0x11 0x10 +OpTypePointer 0x12 0xc 0x11 +OpVariable 0x12 0x13 0xc +OpTypeStruct 0x14 0x2 +OpTypePointer 0x15 0x9 0x14 +OpVariable 0x15 0x16 0x9 +OpTypeVector 0x17 0x2 0x3 +OpTypePointer 0x18 0x1 0x17 +OpVariable 0x18 0x19 0x1 +OpTypeVector 0x1a 0x2 0x2 +OpTypeFunction 0x1c 0xa 0x2 0x1a 0x1a +OpTypeVoid 0x1d +OpTypeFunction 0x1e 0x1d +OpTypePointer 0x21 0x1 0x2 +OpConstant 0x2 0x23 0x0 +OpTypePointer 0x25 0x9 0x2 +OpTypeBool 0x28 +OpTypePointer 0x2d 0xc 0xf +OpConstant 0x2 0x2e 0x5 +OpTypePointer 0x31 0xc 0x5 +OpConstant 0x2 0x34 0x1f +OpTypePointer 0x3e 0xc 0x4 +OpConstant 0x2 0x41 0x1e +OpConstant 0x2 0x43 0x1 +OpTypeInt 0x45 0x20 0x1 +OpTypePointer 0x47 0xc 0x1 +OpTypeInt 0x4a 0x8 0x1 +OpTypeVector 0x4b 0x4a 0x2 +OpTypePointer 0x53 0xc 0xa +OpFunction 0x1d 0x1f 0x0 0x1e +OpLabel 0x20 +OpAccessChain 0x21 0x22 0x19 0x23 +OpLoad 0x2 0x24 0x22 +OpAccessChain 0x25 0x26 0x16 0x23 +OpLoad 0x2 0x27 0x26 +OpULessThan 0x28 0x29 0x24 0x27 +OpSelectionMerge 0x2a 0x0 +OpBranchConditional 0x29 0x2b 0x2a +OpLabel 0x2b +OpAccessChain 0x2d 0x2c 0x13 0x23 0x24 +OpShiftRightLogical 0x2 0x2f 0x24 0x2e +OpAccessChain 0x31 0x30 0x9 0x23 0x2f +OpShiftRightLogical 0x2 0x32 0x24 0x2e +OpCompositeConstruct 0x1a 0x33 0x23 0x32 +OpBitwiseAnd 0x2 0x35 0x24 0x34 +OpCompositeConstruct 0x1a 0x36 0x23 0x35 +OpFunctionCall 0xa 0x37 0x1b 0x2f 0x33 0x36 +OpFConvert 0xf 0x38 0x37 +OpStore 0x2c 0x38 +OpBranch 0x2a +OpLabel 0x2a +OpReturn +OpFunctionEnd +OpFunction 0xa 0x1b 0x0 0x1c +OpFunctionParameter 0x2 0x39 +OpFunctionParameter 0x1a 0x3a +OpFunctionParameter 0x1a 0x3b +OpLabel 0x3c +OpAccessChain 0x31 0x3d 0x9 0x23 0x39 +OpAccessChain 0x3e 0x3f 0x3d 0x23 +OpCompositeExtract 0x2 0x40 0x3b 0x1 +OpBitwiseAnd 0x2 0x42 0x40 0x41 +OpShiftRightLogical 0x2 0x44 0x42 0x43 +OpBitcast 0x45 0x46 0x44 +OpAccessChain 0x47 0x48 0x3f 0x46 +OpLoad 0x1 0x49 0x48 +OpBitcast 0x4b 0x4c 0x49 +OpCompositeExtract 0x2 0x4d 0x3b 0x1 +OpBitwiseAnd 0x2 0x4e 0x4d 0x43 +OpBitcast 0x45 0x4f 0x4e +OpVectorExtractDynamic 0x4a 0x50 0x4c 0x4f +OpCompositeExtract 0x2 0x51 0x3a 0x1 +OpAccessChain 0x53 0x52 0xe 0x23 0x51 +OpLoad 0xa 0x54 0x52 +OpSConvert 0x45 0x55 0x50 +OpConvertSToF 0xf 0x56 0x55 +OpFConvert 0xa 0x57 0x56 +OpFMul 0xa 0x58 0x54 0x57 +OpReturnValue 0x58 +OpFunctionEnd diff --git a/tests/spirv/_golden/spelemi.txt b/tests/spirv/_golden/spelemi.txt new file mode 100644 index 0000000000..8680ad2df1 --- /dev/null +++ b/tests/spirv/_golden/spelemi.txt @@ -0,0 +1,127 @@ +; SPIR-V module: 495 words, id-bound 0x5b +OpCapability 0x1 +OpCapability 0x1151 +OpCapability 0x16 +OpCapability 0x9 +OpCapability 0x27 +OpMemoryModel 0x0 0x1 +OpEntryPoint 0x5 0x1f 0x6e69616d 0x0 0x19 +OpExecutionMode 0x1f 0x11 0x40 0x1 0x1 +OpDecorate 0x4 0x6 0x2 +OpMemberDecorate 0x5 0x0 0x23 0x0 +OpDecorate 0x6 0x6 0x20 +OpDecorate 0x7 0x2 +OpMemberDecorate 0x7 0x0 0x23 0x0 +OpDecorate 0x9 0x22 0x0 +OpDecorate 0x9 0x21 0x0 +OpDecorate 0xb 0x6 0x2 +OpDecorate 0xc 0x2 +OpMemberDecorate 0xc 0x0 0x23 0x0 +OpDecorate 0xe 0x22 0x0 +OpDecorate 0xe 0x21 0x1 +OpDecorate 0x10 0x6 0x4 +OpDecorate 0x11 0x2 +OpMemberDecorate 0x11 0x0 0x23 0x0 +OpDecorate 0x13 0x22 0x0 +OpDecorate 0x13 0x21 0x2 +OpDecorate 0x14 0x2 +OpMemberDecorate 0x14 0x0 0x23 0x0 +OpDecorate 0x19 0xb 0x1c +OpTypeInt 0x1 0x10 0x1 +OpTypeInt 0x2 0x20 0x0 +OpConstant 0x2 0x3 0x10 +OpTypeArray 0x4 0x1 0x3 +OpTypeStruct 0x5 0x4 +OpTypeRuntimeArray 0x6 0x5 +OpTypeStruct 0x7 0x6 +OpTypePointer 0x8 0xc 0x7 +OpVariable 0x8 0x9 0xc +OpTypeFloat 0xa 0x10 +OpTypeRuntimeArray 0xb 0xa +OpTypeStruct 0xc 0xb +OpTypePointer 0xd 0xc 0xc +OpVariable 0xd 0xe 0xc +OpTypeFloat 0xf 0x20 +OpTypeRuntimeArray 0x10 0xf +OpTypeStruct 0x11 0x10 +OpTypePointer 0x12 0xc 0x11 +OpVariable 0x12 0x13 0xc +OpTypeStruct 0x14 0x2 +OpTypePointer 0x15 0x9 0x14 +OpVariable 0x15 0x16 0x9 +OpTypeVector 0x17 0x2 0x3 +OpTypePointer 0x18 0x1 0x17 +OpVariable 0x18 0x19 0x1 +OpTypeVector 0x1a 0x2 0x2 +OpTypeFunction 0x1c 0xa 0x2 0x1a 0x1a +OpTypeVoid 0x1d +OpTypeFunction 0x1e 0x1d +OpTypePointer 0x21 0x1 0x2 +OpConstant 0x2 0x23 0x0 +OpTypePointer 0x25 0x9 0x2 +OpTypeBool 0x28 +OpConstant 0x2 0x2c 0x5 +OpTypeInt 0x2e 0x20 0x1 +OpTypePointer 0x31 0xc 0xf +OpTypePointer 0x33 0xc 0x5 +OpConstant 0x2 0x36 0x1f +OpTypePointer 0x41 0xc 0x4 +OpConstant 0x2 0x44 0x1e +OpConstant 0x2 0x46 0x1 +OpTypePointer 0x49 0xc 0x1 +OpTypeInt 0x4c 0x8 0x1 +OpTypeVector 0x4d 0x4c 0x2 +OpTypePointer 0x55 0xc 0xa +OpFunction 0x1d 0x1f 0x0 0x1e +OpLabel 0x20 +OpAccessChain 0x21 0x22 0x19 0x23 +OpLoad 0x2 0x24 0x22 +OpAccessChain 0x25 0x26 0x16 0x23 +OpLoad 0x2 0x27 0x26 +OpULessThan 0x28 0x29 0x24 0x27 +OpSelectionMerge 0x2a 0x0 +OpBranchConditional 0x29 0x2b 0x2a +OpLabel 0x2b +OpShiftRightLogical 0x2 0x2d 0x24 0x2c +OpBitcast 0x2e 0x2f 0x2d +OpAccessChain 0x31 0x30 0x13 0x23 0x24 +OpAccessChain 0x33 0x32 0x9 0x23 0x2f +OpShiftRightLogical 0x2 0x34 0x24 0x2c +OpCompositeConstruct 0x1a 0x35 0x23 0x34 +OpBitwiseAnd 0x2 0x37 0x24 0x36 +OpCompositeConstruct 0x1a 0x38 0x23 0x37 +OpBitcast 0x2 0x3a 0x2f +OpFunctionCall 0xa 0x39 0x1b 0x3a 0x35 0x38 +OpFConvert 0xf 0x3b 0x39 +OpStore 0x30 0x3b +OpBranch 0x2a +OpLabel 0x2a +OpReturn +OpFunctionEnd +OpFunction 0xa 0x1b 0x0 0x1c +OpFunctionParameter 0x2 0x3c +OpFunctionParameter 0x1a 0x3d +OpFunctionParameter 0x1a 0x3e +OpLabel 0x3f +OpAccessChain 0x33 0x40 0x9 0x23 0x3c +OpAccessChain 0x41 0x42 0x40 0x23 +OpCompositeExtract 0x2 0x43 0x3e 0x1 +OpBitwiseAnd 0x2 0x45 0x43 0x44 +OpShiftRightLogical 0x2 0x47 0x45 0x46 +OpBitcast 0x2e 0x48 0x47 +OpAccessChain 0x49 0x4a 0x42 0x48 +OpLoad 0x1 0x4b 0x4a +OpBitcast 0x4d 0x4e 0x4b +OpCompositeExtract 0x2 0x4f 0x3e 0x1 +OpBitwiseAnd 0x2 0x50 0x4f 0x46 +OpBitcast 0x2e 0x51 0x50 +OpVectorExtractDynamic 0x4c 0x52 0x4e 0x51 +OpCompositeExtract 0x2 0x53 0x3d 0x1 +OpAccessChain 0x55 0x54 0xe 0x23 0x53 +OpLoad 0xa 0x56 0x54 +OpSConvert 0x2e 0x57 0x52 +OpConvertSToF 0xf 0x58 0x57 +OpFConvert 0xa 0x59 0x58 +OpFMul 0xa 0x5a 0x56 0x59 +OpReturnValue 0x5a +OpFunctionEnd diff --git a/tests/spirv/_spirv_common.das b/tests/spirv/_spirv_common.das index b6f7802d4e..70812b5b0c 100644 --- a/tests/spirv/_spirv_common.das +++ b/tests/spirv/_spirv_common.das @@ -1620,7 +1620,8 @@ def public phase10_6_emitter_opcodes : table { // already declared). OpFunction / OpLabel / OpFunctionEnd were declared back in Phase 1. Phase-10.6 + these: def public phase11_fn_emitter_opcodes : table { var s <- phase10_6_emitter_opcodes() - for (op in [SpvOp.FunctionCall, SpvOp.FunctionParameter, SpvOp.ReturnValue]) { + //! CopyLogical: the spdirect fixture's direct decode call + for (op in [SpvOp.FunctionCall, SpvOp.FunctionParameter, SpvOp.ReturnValue, SpvOp.CopyLogical]) { s |> insert(uint(op)) } return <- s @@ -2113,6 +2114,94 @@ def public unpack16_words : array { return clone_to_move(unpack16_spv) } +// ===== spdirect: a [spirv_decode] method called DIRECTLY over a block the kernel body copied out of its plane ===== +struct SpBlk { + qs : int16[16] // 32 int8 quants as 16-bit lanes (the q8 block shape) +} + +struct SpArgs { + n : uint +} + +class SpDirectCls { + @ssbo @binding = 0 wq : array + @ssbo @binding = 1 ws : array + @ssbo @binding = 2 y : array + @push_constant pa : SpArgs + + [spirv_decode] + def decode(blk : SpBlk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return ws[bc.y] * float16(float(int(q))) + } + + [spirv_kernel(local_size_x = 64, name = "spdirect_spv"), marker(no_coverage)] + def run { + let gid = gl_GlobalInvocationID.x + if (gid < pa.n) { + let blk = wq[gid >> 5u] + y[gid] = float(decode(blk, uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + } + } +} + +def public spdirect_words : array { + return clone_to_move(spdirect_spv) +} + +// ===== spelem: the same call on the plane element itself - the block argument is the SSBO element ===== +class SpElemCls { + @ssbo @binding = 0 wq : array + @ssbo @binding = 1 ws : array + @ssbo @binding = 2 y : array + @push_constant pa : SpArgs + + [spirv_decode] + def decode(blk : SpBlk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return ws[bc.y] * float16(float(int(q))) + } + + [spirv_kernel(local_size_x = 64, name = "spelem_spv"), marker(no_coverage)] + def run { + let gid = gl_GlobalInvocationID.x + if (gid < pa.n) { + y[gid] = float(decode(wq[gid >> 5u], uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + } + } +} + +def public spelem_words : array { + return clone_to_move(spelem_spv) +} + +// ===== spelemi: the plane-element call with an int index - the index bitcasts to uint on the way in ===== +class SpElemIntCls { + @ssbo @binding = 0 wq : array + @ssbo @binding = 1 ws : array + @ssbo @binding = 2 y : array + @push_constant pa : SpArgs + + [spirv_decode] + def decode(blk : SpBlk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return ws[bc.y] * float16(float(int(q))) + } + + [spirv_kernel(local_size_x = 64, name = "spelemi_spv"), marker(no_coverage)] + def run { + let gid = gl_GlobalInvocationID.x + if (gid < pa.n) { + let i = int(gid >> 5u) + y[gid] = float(decode(wq[i], uint2(0u, gid >> 5u), uint2(0u, gid & 31u))) + } + } +} + +def public spelemi_words : array { + return clone_to_move(spelemi_spv) +} + // ===== cooperative matrix (SPV_KHR_cooperative_matrix): an f16 x f16 -> f32 GEMM tile. Loads a 16x16 A // (row-major) and B (column-major) from f16 SSBOs into subgroup tiles, one MulAdd into a zero-initialized // (OpConstantNull) f32 accumulator, stores the 16x16 result column-major. Exercises diff --git a/tests/spirv/test_census.das b/tests/spirv/test_census.das index 4c7ecff694..e1064e2964 100644 --- a/tests/spirv/test_census.das +++ b/tests/spirv/test_census.das @@ -108,6 +108,9 @@ def test_opcode_census(t : T?) { // nolint:STYLE038 — a flat one-add_set-per-f add_set(present, coopmat2fa256_words()) add_set(present, q8store_words()) add_set(present, unpack16_words()) + add_set(present, spdirect_words()) + add_set(present, spelem_words()) + add_set(present, spelemi_words()) var declared <- coopmat2fa_emitter_opcodes() for (op in keys(present)) { t |> success(key_exists(declared, op), "emitted {op_name(op)} is in the declared set") diff --git a/tests/spirv/test_fail_closed.das b/tests/spirv/test_fail_closed.das index fa1e467945..be60ebdcea 100644 --- a/tests/spirv/test_fail_closed.das +++ b/tests/spirv/test_fail_closed.das @@ -144,5 +144,10 @@ def test_fail_closed_kernel_class(t : T?) { check_rejects(t, "_fc_kernel_no_ssbo", "has no @ssbo members") check_rejects(t, "_fc_kernel_method_rec", "method call cycle") check_rejects(t, "_fc_kernel_method_abstract", "has no resolvable body") + // a [spirv_decode] method called directly: one block form per method, one plane, a 32-bit index + check_rejects(t, "_fc_decode_mixed", "takes its block the same way at every call") + check_rejects(t, "_fc_decode_twoplanes", "takes them from ONE plane") + check_rejects(t, "_fc_decode_helper_copy", "takes the element itself") + check_rejects(t, "_fc_decode_index64", "a plane element index must be int or uint") } } diff --git a/tests/spirv/test_golden.das b/tests/spirv/test_golden.das index 996cece96a..d447b3edfb 100644 --- a/tests/spirv/test_golden.das +++ b/tests/spirv/test_golden.das @@ -175,6 +175,9 @@ def test_golden_disassembly(t : T?) { // nolint:STYLE038 - flat list of golden f check(t, "userfn", userfn_words()) check(t, "userfn_ref", userfn_ref_words()) check(t, "userfn_global", userfn_global_words()) + check(t, "spdirect", spdirect_words()) + check(t, "spelem", spelem_words()) + check(t, "spelemi", spelemi_words()) // PR-D mesh + task stage scaffolding fixtures (visitor baseline): cap + ext + execution model // + execution modes + SetMeshOutputsEXT (mesh) / EmitMeshTasksEXT (task). byte-identical // check() pins the opcode AND id-allocation order of the prologue intrinsics. diff --git a/tests/spirv/test_struct_param.das b/tests/spirv/test_struct_param.das new file mode 100644 index 0000000000..d0442b3b19 --- /dev/null +++ b/tests/spirv/test_struct_param.das @@ -0,0 +1,113 @@ +options gen2 +options indenting = 4 + +// The struct-value-parameter spill (`sparam` below), and the direct-[spirv_decode] fixtures whose calls need it (`spdirect`, `spelem`, both in _spirv_common). + +require dastest/testing_boost public +require _spirv_common +require spirv/spirv_shader +require spirv/spirv_builtins public +require spirv/spirv_dis +require spirv/spirv_grammar + +struct SpArr { + arr : float[4] + k : float +} + +var @ssbo @binding = 0 sp_data : array + +def private sp_pick(c : SpArr; i : int) : float { + return c.arr[i] * c.k +} + +[compute_shader(local_size_x=64, name="sparam_spv"), marker(no_coverage)] +def sparam { + let i = int(gl_GlobalInvocationID.x) + var s : SpArr + s.arr[0] = sp_data[i] + s.arr[1] = sp_data[i] + 1.0f + s.arr[2] = sp_data[i] + 2.0f + s.arr[3] = sp_data[i] + 3.0f + s.k = 0.5f + sp_data[i] = sp_pick(s, i & 3) +} + +def private validate(t : T?; words : array; lbl : string; target_env : string = "vulkan1.1") { + let r = validate_spirv(words, target_env) + if (r.ran) { + t |> success(r.ok, "{lbl}: spirv-val: {r.msg}") + } else { + feint("spirv-val not found locally; skipping (CI enforces)\n") + } +} + +[test] +def test_struct_param(t : T?) { + t |> run("struct value param: spilled to a Function local, its array member indexed at run time") <| @(t : T?) { + var words <- clone_to_move(sparam_spv) + //! entry + sp_pick = 2 OpFunctions, one call; sp_pick(c, i) = 2 by-value parameters + t |> equal(count_op(words, SpvOp.Function), 2) + t |> equal(count_op(words, SpvOp.FunctionCall), 1) + t |> equal(count_op(words, SpvOp.FunctionParameter), 2) + //! OpVariables: gid + sp_data + the kernel's `var s` + the spill of `c` - 3 without the spill + t |> equal(count_op(words, SpvOp.Variable), 4) + t |> success(has_op(words, SpvOp.Store), "sparam: the struct parameter is stored into its Function local") + t |> success(has_op(words, SpvOp.AccessChain), "sparam: the array member is access-chained off the spill") + //! SpArr + the SSBO Block = 2 OpTypeStruct; a spill is no interface block, so Offset stays the only member decoration + t |> equal(count_op(words, SpvOp.TypeStruct), 2) + t |> equal(count_op(words, SpvOp.MemberDecorate), 1) + t |> success(words[1] < SPV_VERSION_1_4, "sparam: a spill alone leaves the module at the 1.3 floor, got {words[1]}") + validate(t, words, "sparam") + delete words + } + t |> run("a [spirv_decode] method called directly is a plain OpFunctionCall in a module with no coopmat capability") <| @(t : T?) { + var words <- spdirect_words() + //! run + decode = 2 OpFunctions, called once; decode's (blk, bc, cib) = 3 parameters + t |> equal(count_op(words, SpvOp.Function), 2) + t |> equal(count_op(words, SpvOp.FunctionCall), 1) + t |> equal(count_op(words, SpvOp.FunctionParameter), 3) + t |> success(!op_has_operand(words, SpvOp.Capability, uint(SpvCapability.CooperativeMatrixTensorAddressingNV)), + "spdirect: no CooperativeMatrixTensorAddressingNV capability") + t |> success(!op_has_operand(words, SpvOp.Capability, uint(SpvCapability.CooperativeMatrixKHR)), + "spdirect: no CooperativeMatrixKHR capability") + t |> success(!op_has_operand(words, SpvOp.Capability, uint(SpvCapability.PhysicalStorageBufferAddresses)), + "spdirect: no PhysicalStorageBufferAddresses capability (the block travels by value, not by device address)") + t |> success(has_op(words, SpvOp.Store), "spdirect: the block parameter is stored into its Function local") + t |> success(has_op(words, SpvOp.CopyLogical), "spdirect: the laid-out block value is OpCopyLogical'd to the parameter's plain struct") + t |> success(words[1] >= SPV_VERSION_1_4, "spdirect: the module version is at least 1.4 (OpCopyLogical's floor), got {words[1]}") + validate(t, words, "spdirect", "vulkan1.2") + delete words + } + t |> run("a [spirv_decode] method called directly on the plane element takes the element's index and chains through the plane") <| @(t : T?) { + var words <- spelem_words() + //! run + decode = 2 OpFunctions, called once; decode's (blk, bc, cib) = 3 parameters, blk a uint index + t |> equal(count_op(words, SpvOp.Function), 2) + t |> equal(count_op(words, SpvOp.FunctionCall), 1) + t |> equal(count_op(words, SpvOp.FunctionParameter), 3) + //! OpVariables: gid + wq/ws/y + pa = 5, no sixth - the element form spills nothing + t |> equal(count_op(words, SpvOp.Variable), 5) + t |> success(!has_op(words, SpvOp.CopyLogical), "spelem: no OpCopyLogical - the element is never loaded as a value") + t |> success(words[1] < SPV_VERSION_1_4, "spelem: the module version stays below 1.4, got {words[1]}") + //! >= 3: the callee's entry chain into the plane plus its member reads through it + t |> success(count_op(words, SpvOp.AccessChain) >= 3, "spelem: the callee chains through the plane ({count_op(words, SpvOp.AccessChain)} access chains)") + t |> success(!op_has_operand(words, SpvOp.Capability, uint(SpvCapability.CooperativeMatrixTensorAddressingNV)), + "spelem: no CooperativeMatrixTensorAddressingNV capability") + validate(t, words, "spelem") + delete words + } + t |> run("[spirv_decode] called on a plane element with an int index: the index bitcasts to uint") <| @(t : T?) { + var words <- spelemi_words() + var uwords <- spelem_words() + //! the same module as spelem plus two OpBitcasts: the int() of the lane id, and the index back to uint at the call + t |> equal(count_op(words, SpvOp.Function), 2) + t |> equal(count_op(words, SpvOp.FunctionParameter), 3) + t |> equal(count_op(words, SpvOp.Variable), 5) + t |> equal(count_op(words, SpvOp.Bitcast), count_op(uwords, SpvOp.Bitcast) + 2) + t |> success(!has_op(words, SpvOp.CopyLogical), "spelemi: no OpCopyLogical - the element is never loaded as a value") + t |> success(words[1] < SPV_VERSION_1_4, "spelemi: the module version stays below 1.4, got {words[1]}") + validate(t, words, "spelemi") + delete words + delete uwords + } +} diff --git a/utils/dasllama-server/REVIEW.md b/utils/dasllama-server/REVIEW.md index 7d1461cb3c..a5ba436a9f 100644 --- a/utils/dasllama-server/REVIEW.md +++ b/utils/dasllama-server/REVIEW.md @@ -48,3 +48,7 @@ fails the compile of a build without dasLLVM. **A function signature in this folder never names a type from one of those modules - those types stay inside the guarded arm, and plain types cross the boundary.** A signature cannot sit inside a `static_if` arm, so no guard fixes it. + +**A `[test]` file in this folder that carries a `require dasllama/...` line of its own answers +to `modules/dasLLAMA/tests/REVIEW.md` (repo root) as well** - its out-of-folder ledger row lives +there. diff --git a/utils/internal/ast-fuzz/test_ast_fuzz.das b/utils/internal/ast-fuzz/test_ast_fuzz.das index a53e52f5f3..2676a42d96 100644 --- a/utils/internal/ast-fuzz/test_ast_fuzz.das +++ b/utils/internal/ast-fuzz/test_ast_fuzz.das @@ -4,7 +4,13 @@ require dastest/testing_boost public require daslib/fio require daslib/strings_boost -let BIN = "bin/daslang" +//! the binary running this test, whatever the build layout put it (`bin/daslang` on a single-config build, `bin/Release/daslang.exe` under MSVC) +def private das_exe() : string { + let args <- get_command_line_arguments() + return empty(args) ? "bin/daslang" : get_full_file_name(args[0]) +} + +let BIN = das_exe() def private run(argv : array; var out : string&; timeout : float) : int { return run_and_capture(argv, out, timeout)