From 655b014aa7c0bfb6a66344bc756653bd69dbd9c8 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:24:29 +0900 Subject: [PATCH 01/85] gpu: support six-slot F32 tier handoff --- crates/gpu-deepseek41/src/chain/ffn/batch.rs | 215 +++++++++++++++ crates/gpu/src/host/handoff.rs | 272 +++++++++++++++---- docs/research/flint-attach.md | 24 ++ tools/unsafe-ratchet.txt | 5 +- 4 files changed, 466 insertions(+), 50 deletions(-) diff --git a/crates/gpu-deepseek41/src/chain/ffn/batch.rs b/crates/gpu-deepseek41/src/chain/ffn/batch.rs index fd4f1f741..fe5f6ed57 100644 --- a/crates/gpu-deepseek41/src/chain/ffn/batch.rs +++ b/crates/gpu-deepseek41/src/chain/ffn/batch.rs @@ -2604,3 +2604,218 @@ fn count_of(by_count: &mut [T], m: usize) -> Result<&mut T, GpuError> { detail: format!("a chunk of {m} tokens; the batch holds scratch for 1..={n}"), }) } + +#[cfg(test)] +mod tests { + use super::{CardAccTier, FfnBatchKernels}; + use bloomery_gpu::host::handoff::{Handoff, HandoffKernels}; + use bloomery_gpu::host::page::HandoffLayout; + use bloomery_gpu::host::step::HandoffTarget; + use bloomery_gpu::host::tier::{TierImageLayout, TierTarget}; + use bloomery_gpu::hybrid::HOST; + use bloomery_gpu::{FaultSite, Gpu, GpuError, Window}; + use cuda_core::DeviceBuffer; + + /// The handoff's places feed the production tier sum, not a readback + /// alone: every routed row belongs to exactly one of host, card and + /// tier, and all other copies are poisoned. Both slot entries must + /// conserve that union, including the final routed lane and partial + /// blocks, eagerly and on replay. Invalid ids must still raise. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-ds41-lib` runs it on the box"] + fn hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes() { + const EXPERTS: usize = 16; + const ROW_OFF: usize = 3; + const LAYER: usize = 7; + let gpu = Gpu::new().expect("a GPU"); + let stream = gpu.stream(); + let handoff = HandoffKernels::load(gpu.context()).expect("handoff kernels"); + let sums = FfnBatchKernels::load(gpu.context()).expect("tier sum kernels"); + let mut map = vec![HOST - 1; ROW_OFF + EXPERTS]; + let mut tmap = map.clone(); + for id in 0..EXPERTS { + map[ROW_OFF + id] = if id % 3 == 0 { (id / 3) as u32 } else { HOST }; + tmap[ROW_OFF + id] = if id % 3 == 1 { (id / 3) as u32 } else { HOST }; + } + let map = DeviceBuffer::from_host(stream, &map).expect("card map"); + let tmap = DeviceBuffer::from_host(stream, &tmap).expect("tier map"); + for n_used in [6, 8] { + for hidden in [n_used, 256, 257] { + for bad in [None, Some(EXPERTS as u32), Some(u32::MAX)] { + let mut ids = vec![15, 0, 10, 5, 8, 1, 2, 7]; + ids.truncate(n_used); + if let Some(id) = bad { + ids[n_used - 1] = id; + } + // Neither handoff may read the router's trailing lane. + ids.push(u32::MAX); + let weights: Vec = (0..n_used) + .map(|s| (s + 1) as f32 / 16.0) + .chain([f32::NAN]) + .collect(); + let x: Vec = (0..hidden).map(|d| (d % 7 + 1) as f32 / 8.0).collect(); + let row = |s: usize, d: usize| (ids[s] + 1) as f32 * x[d]; + let mut down = vec![f32::NAN; n_used * hidden]; + let mut trows = down.clone(); + let mut host = vec![0.0f32; hidden]; + let mut want = host.clone(); + for d in 0..hidden { + let mut card_tier = 0.0f32; + for s in 0..n_used { + if ids[s] >= EXPERTS as u32 { + continue; + } + let value = row(s, d); + match ids[s] % 3 { + 0 => down[s * hidden + d] = value, + 1 => trows[s * hidden + d] = value, + _ => host[d] = value.mul_add(weights[s], host[d]), + } + if ids[s] % 3 != 2 { + card_tier = value.mul_add(weights[s], card_tier); + } + } + want[d] = card_tier + host[d]; + } + let ids_d = DeviceBuffer::from_host(stream, &ids).expect("ids"); + let mut weights_d = DeviceBuffer::from_host(stream, &weights).expect("weights"); + let x_d = DeviceBuffer::from_host(stream, &x).expect("activation"); + let seq = DeviceBuffer::from_host(stream, &[19u32]).expect("sequence"); + let down = DeviceBuffer::from_host(stream, &down).expect("card rows"); + let trows = DeviceBuffer::from_host(stream, &trows).expect("tier rows"); + let doubled_host: Vec = host.iter().map(|h| 2.0 * h).collect(); + let mut host = DeviceBuffer::from_host(stream, &host).expect("host sum"); + let lay = HandoffLayout { + seq: 0, + ids: 1, + weights: 1 + n_used, + x: 1 + 2 * n_used, + n_used, + hidden, + }; + let tl = TierImageLayout { + sel: 0, + q3: 0, + d8: 0, + x: n_used, + n_used, + q3_words: 0, + d8_len: 0, + x_len: hidden, + hidden, + cols: 1, + }; + for captured in [false, true] { + let mut image = + DeviceBuffer::zeroed(stream, lay.x + hidden).expect("image"); + let mut timage = + DeviceBuffer::zeroed(stream, tl.x + hidden).expect("tier image"); + let poison = vec![HOST - 1; n_used + 1]; + let mut sel = DeviceBuffer::from_host(stream, &poison).expect("places"); + let mut tsel = + DeviceBuffer::from_host(stream, &poison).expect("tier places"); + let mut acc = DeviceBuffer::zeroed(stream, hidden).expect("sum"); + let mut result = DeviceBuffer::zeroed(stream, hidden).expect("result"); + let mut enqueue = || -> Result<(), GpuError> { + handoff.enqueue_handoff_tier( + stream, + &Handoff { + ids: &ids_d, + weights: &weights_d, + map: &map, + row_off: ROW_OFF, + n_expert: EXPERTS, + }, + &tmap, + ( + HandoffTarget { + x: &x_d, + seq: &seq, + image: &mut image, + layout: lay, + }, + TierTarget { + image: &mut timage, + rows: &trows, + layout: tl, + }, + ), + gpu.layer_sink(LAYER)?, + (&mut sel, &mut tsel), + )?; + let routed_weights = + Window::::of(&image, 4 * lay.weights, n_used)?; + sums.enqueue_card_acc_tier( + stream, + &CardAccTier { + down: &down, + trows: &trows, + w: &routed_weights, + sel: &sel, + tsel: &tsel, + n: hidden, + m: 1, + n_card: EXPERTS.div_ceil(3), + n_tier: EXPERTS.div_ceil(3), + n_used, + }, + &mut acc, + )?; + gpu.elem() + .enqueue_add(stream, &acc, &host, hidden, &mut result) + }; + if captured { + let graph = gpu.capture(|_| enqueue()).expect("capture"); + graph.launch(stream).expect("replay"); + let doubled_weights: Vec = + weights.iter().map(|w| 2.0 * w).collect(); + weights_d + .copy_from_host(stream, &doubled_weights) + .expect("new route weights"); + host.copy_from_host(stream, &doubled_host) + .expect("new host sum"); + graph.launch(stream).expect("second replay"); + stream.synchronize().expect("replays settled"); + } else { + enqueue().expect("eager handoff and consumption"); + } + let scale = if captured { 2.0 } else { 1.0 }; + let got = result.to_host_vec(stream).expect("result"); + let places = sel.to_host_vec(stream).expect("places"); + let tier_places = tsel.to_host_vec(stream).expect("tier places"); + let image = image.to_host_vec(stream).expect("host image"); + let timage = timage.to_host_vec(stream).expect("tier image"); + stream.synchronize().expect("settled"); + let fault = gpu.take_fault().expect("fault word"); + if bad.is_some() { + let fault = fault.expect("an invalid id must raise"); + assert_eq!(fault.layer, LAYER as u32); + assert_eq!(fault.code, FaultSite::ExpertId as u32); + assert_eq!(fault.sites, 1 << FaultSite::ExpertId as u32); + } else { + assert!(fault.is_none(), "{fault:?}"); + } + assert_eq!(places[n_used], HOST - 1, "card places tail"); + assert_eq!(tier_places[n_used], HOST - 1, "tier places tail"); + assert_eq!(image[lay.seq], 19, "sequence"); + assert_eq!(&image[lay.ids..lay.ids + n_used], &ids[..n_used]); + for s in 0..n_used { + assert_eq!(image[lay.weights + s], (weights[s] * scale).to_bits()); + assert_eq!(timage[tl.sel + s], tier_places[s]); + } + for d in 0..hidden { + assert_eq!(image[lay.x + d], x[d].to_bits()); + assert_eq!(timage[tl.x + d], x[d].to_bits()); + } + assert!( + got.iter() + .zip(&want) + .all(|(g, w)| g.to_bits() == (w * scale).to_bits()), + "{n_used} slots, width {hidden}, bad {bad:?}, capture {captured}: result conservation" + ); + } + } + } + } + } +} diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index a653be6db..99f6b4efe 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,9 +18,9 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_8_tier` is -//! the eight-slot handoff of a layer an expert tier holds experts of: the -//! same writes, and the slots' tier places and the activation in f32 into +//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_6_tier` and +//! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an +//! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); //! `ds41_ffn_handoff_10_cols_tier` is `_10_cols`'s of such a layer, every //! column's tier places and activation into the row's tier image @@ -281,6 +281,88 @@ mod handoff_kernels { unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; } + /// [`ds41_ffn_handoff`] and the row's f32 tier image, with the same + /// per-thread writes and following go as [`ds41_ffn_handoff_8_tier`]. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + ids_in.len() >= 6, + w_in.len() >= 6, + map.len() >= row_off + n_expert, + tmap.len() >= row_off + n_expert, + x.len() >= n, + seq.len() >= 1, + n >= 6, + ids_at >= seq_at + 1, + wts_at >= ids_at + 6, + x_at >= wts_at + 6, + image.len() >= x_at + n, + tx_at >= tsel_at + 6, + timage.len() >= tx_at + n, + sel.len() >= 6, + tsel.len() >= 6 + ) + )] + pub fn ds41_ffn_handoff_6_tier( + ids_in: &[u32], + w_in: &[f32], + map: &[u32], + tmap: &[u32], + row_off: u32, + n_expert: u32, + x: &[f32], + seq: &[u32], + n: u32, + seq_at: u32, + ids_at: u32, + wts_at: u32, + x_at: u32, + tsel_at: u32, + tx_at: u32, + fault: FaultSink, + mut image: DisjointSlice, + mut timage: DisjointSlice, + mut sel: DisjointSlice, + mut tsel: DisjointSlice, + ) { + let d = thread::index_1d().get(); + if d >= n as usize { + return; + } + let a = HandoffIn { + ids_in, + w_in, + map, + row_off, + n_expert, + x, + seq, + seq_at, + ids_at, + wts_at, + x_at, + fault, + }; + // SAFETY: d < n, and the launch contract is `handoff_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; + let t = TierIn { + tmap, + tsel_at, + tx_at, + }; + // SAFETY: d < n, and the launch contract is `handoff_tier_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_tier_at::(&a, &t, d, &mut timage, &mut tsel) }; + } + /// [`ds41_ffn_handoff_8`] and the row's f32 tier image /// ([`handoff_tier_at`]): beside the handoff and the stage card's places, /// threads `s < 8` write `tsel[s] = tmap[row_off + id]` — the id's slot @@ -1181,8 +1263,9 @@ impl HandoffKernels { } impl HandoffKernels { - /// Enqueue the handoff of a tier layer of eight slots a token - /// (`ds41_ffn_handoff_8_tier`): [`HandoffKernels::enqueue_handoff`]'s + /// Enqueue the handoff of a tier layer of six or eight slots a token + /// (`ds41_ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and /// with the activation into the f32 tier image `ttarget`. A layout of @@ -1202,52 +1285,82 @@ impl HandoffKernels { fault: FaultSink, (sel, tsel): (&mut DeviceBuffer, &mut DeviceBuffer), ) -> Result<(), GpuError> { - const WHAT: &str = "ds41_ffn_handoff_8_tier"; let (lay, tl) = (target.layout, ttarget.layout); - if lay.n_used != SLOTS_8 - || tl.n_used != SLOTS_8 - || tl.x_len != lay.hidden - || tl.q3_words != 0 - || tl.d8_len != 0 - { - return Err(GpuError::Shape { - what: WHAT, - detail: format!( - "the boundary carries {} slots of {} values and the tier image {} slots, {} \ - f32, {} code words, {} scales; the entry hands over {SLOTS_8} slots and the \ - activation in f32", - lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len - ), - }); - } + let slots = check_tier(&lay, &tl)?; + let what = match slots { + HandoffSlots::Six => "ds41_ffn_handoff_6_tier", + _ => "ds41_ffn_handoff_8_tier", + }; let n = lay.hidden; - let grid = launch_u32(WHAT, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; + let grid = launch_u32(what, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; let cfg = LaunchConfig1D::new(grid, HANDOFF_THREADS, 0); - let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; - self.module.ds41_ffn_handoff_8_tier( - stream, - &prep, - h.ids, - h.weights, - h.map, - tmap, - launch_u32(WHAT, "row_off", h.row_off)?, - launch_u32(WHAT, "n_expert", h.n_expert)?, - target.x, - target.seq, - launch_u32(WHAT, "n", n)?, - launch_u32(WHAT, "seq_at", lay.seq)?, - launch_u32(WHAT, "ids_at", lay.ids)?, - launch_u32(WHAT, "wts_at", lay.weights)?, - launch_u32(WHAT, "x_at", lay.x)?, - launch_u32(WHAT, "tsel_at", tl.sel)?, - launch_u32(WHAT, "tx_at", tl.x)?, - fault, - target.image, - ttarget.image, - sel, - tsel, - )?; + let row_off = launch_u32(what, "row_off", h.row_off)?; + let n_expert = launch_u32(what, "n_expert", h.n_expert)?; + let at = [ + launch_u32(what, "n", n)?, + launch_u32(what, "seq_at", lay.seq)?, + launch_u32(what, "ids_at", lay.ids)?, + launch_u32(what, "wts_at", lay.weights)?, + launch_u32(what, "x_at", lay.x)?, + launch_u32(what, "tsel_at", tl.sel)?, + launch_u32(what, "tx_at", tl.x)?, + ]; + match slots { + HandoffSlots::Six => { + let prep = self.module.prepare_ds41_ffn_handoff_6_tier(cfg)?; + self.module.ds41_ffn_handoff_6_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + _ => { + let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; + self.module.ds41_ffn_handoff_8_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + } Ok(()) } @@ -1402,6 +1515,29 @@ impl HandoffKernels { } } +/// The single-token F32 tier handoff's layout contract, shared by both +/// slot entries. The generated launch checks the input and output spans. +fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result { + let slots = HandoffSlots::of(lay.n_used); + if !matches!(slots, Ok(HandoffSlots::Six | HandoffSlots::Eight)) + || tl.n_used != lay.n_used + || tl.x_len != lay.hidden + || tl.q3_words != 0 + || tl.d8_len != 0 + { + return Err(GpuError::Shape { + what: "HandoffKernels::enqueue_handoff_tier", + detail: format!( + "the boundary carries {} slots of {} values and the tier image {} slots, {} \ + f32, {} code words, {} scales; the entries hand over matching 6 or 8 slots \ + and the activation in f32", + lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len + ), + }); + } + slots.map_err(|e| GpuError::shape("HandoffKernels::enqueue_handoff_tier", e.to_string())) +} + /// [`HandoffKernels::enqueue_handoff_cols`]'s refusals, by name before the /// launch: a layout of another slot count, `m` of 0, a pitch under ten, and a /// `sel` of `sel_len` places shorter than `m` columns of ten. @@ -1488,7 +1624,7 @@ pub struct Places<'a> { #[cfg(test)] mod tests { - use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier}; + use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier, check_tier}; use crate::host::page::HandoffLayout; use crate::host::tier::TierImageLayout; @@ -1539,6 +1675,44 @@ mod tests { } } + #[test] + fn a_token_tier_handoff_requires_matching_f32_six_or_eight_slot_images() { + for n_used in 0..=10 { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + for tier_slots in 0..=10 { + let tl = TierImageLayout { + n_used: tier_slots, + x: tier_slots, + ..image(1) + }; + assert_eq!( + check_tier(&lay, &tl).is_ok(), + matches!(n_used, 6 | 8) && tier_slots == n_used, + "host {n_used}, tier {tier_slots}" + ); + } + } + for n_used in [6, 8] { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + let tl = TierImageLayout { + n_used, + x: n_used, + ..image(1) + }; + for x_len in [0, HIDDEN - 1, HIDDEN + 1, 2 * HIDDEN] { + assert!(check_tier(&lay, &TierImageLayout { x_len, ..tl }).is_err()); + } + assert!(check_tier(&lay, &TierImageLayout { q3_words: 1, ..tl }).is_err()); + assert!(check_tier(&lay, &TierImageLayout { d8_len: 1, ..tl }).is_err()); + } + } + /// A tier handoff of `m` columns over a page of 8 refuses, by name, a /// `tsel` or a `sel` shorter than `10·m`, and takes ones of exactly that: /// the launch writes every column's places. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..e3bfeb200 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -66,6 +66,30 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Six-slot F32 tier handoff implementation + +`HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry +beside the existing eight-slot entry. Both host and tier layouts must have the +same supported slot count and activation width; compressed tier images remain +refused. The entry uses the existing copy/place/fault helpers and the existing +subsequent go/publication protocol; no Flint router or Program is registered. + +The pure layout contract passed locally after an oxide build. +`hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes`, owned by +`gate-gpu-ds41-lib`, consumes the handoff's weights and emitted places through +the production tier accumulator and adds the host sum. It covers partial widths, +invalid ids, poisoned non-owning rows, trailing sentinels and changed replay +weights. Its host sum is precomputed and activation images are checked by +readback: it is not expert execution or two-card completion/publication proof. +The hardware case has not been run here. + +The new `ds41_ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +owning-gate evidence; the eight-slot and other existing entries must remain +unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. +Actual six-slot body/tier and two-card completion remain integration work, not +capabilities inferred from this primitive. + + ## 1. Sources and observed artifact diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 94a2bb78a..902fcf81c 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -528,7 +528,10 @@ gguf 7 2 0 # copy, scale read and MMA instruction), `Mxfp8Cols` (refused, fetch, land, scale_word), the activations' MXFP8 form # body (`mxfp8_cols_body`: its load and its two plane stores) and the two kernel entries' body calls and shared-memory # view. Each block has its SAFETY line. -gpu 2331 306 10 +# PIN(2026-10-24): gpu 2331 -> 2333 blocks, a new six-slot F32 tier entry, not old code relaxed: +# ds41_ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; +# both blocks state their per-thread bounds and disjoint-write SAFETY invariant. +gpu 2333 306 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. From 55df14add265f2c49f7504e8dfb540c4facddc51 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:32:42 +0900 Subject: [PATCH 02/85] gpu: support ordinary IQ4_XS shared sites --- crates/chain/src/site.rs | 4 + crates/gguf/src/card.rs | 11 +- crates/gguf/src/qtype.rs | 2 +- crates/gpu-deepseek41/src/block.rs | 65 +++--- crates/gpu-deepseek41/src/shared_routed.rs | 185 ++++++++++++++++ crates/gpu-gates/src/act_rule.rs | 45 +--- crates/gpu/src/site.rs | 237 ++++++--------------- crates/placement/src/accepts.rs | 5 +- crates/placement/src/kernels.rs | 114 ---------- docs/research/flint-attach.md | 25 +++ 10 files changed, 334 insertions(+), 359 deletions(-) diff --git a/crates/chain/src/site.rs b/crates/chain/src/site.rs index da5ccefcf..d44ac713e 100644 --- a/crates/chain/src/site.rs +++ b/crates/chain/src/site.rs @@ -38,6 +38,8 @@ pub enum SiteTy { /// entry is none; a backend whose launch reads it names it in the list /// [`file_site`] takes. MlxA4, + /// IQ4_XS in its original blocks, over q8_1 columns, narrow and wide. + Iq4Xs, } impl SiteTy { @@ -56,6 +58,7 @@ impl SiteTy { SiteTy::Mxfp8, SiteTy::Fp8B128, SiteTy::Bf16, + SiteTy::Iq4Xs, ]; /// The family of a file tensor of type `ty`, the one that launches the @@ -99,6 +102,7 @@ impl SiteTy { SiteTy::Mxfp8 => Some(SiteEntry::Mxfp8), SiteTy::Fp8B128 => Some(SiteEntry::Fp8B128), SiteTy::Bf16 => Some(SiteEntry::Bf16), + SiteTy::Iq4Xs => Some(SiteEntry::Iq4xs), SiteTy::MlxA4 => None, } } diff --git a/crates/gguf/src/card.rs b/crates/gguf/src/card.rs index 711ab5dea..48a51c8fe 100644 --- a/crates/gguf/src/card.rs +++ b/crates/gguf/src/card.rs @@ -255,6 +255,9 @@ pub enum SiteEntry { /// `Q8F32Kernels::enqueue_bf16_gemv` over BF16 words and f32 rows; /// wide, `Q8F32Kernels::enqueue_bf16_tile` over the same words and rows. Bf16, + /// `QuantSelKernels::enqueue_gemv_sel` over one matrix in the file's + /// IQ4_XS blocks and `Q8Act` columns; wide, the grouped IQ4_XS GEMM. + Iq4xs, } /// The activation a dense site's launch reads; `crates/gpu/src/site.rs` @@ -281,7 +284,11 @@ impl SiteEntry { #[must_use] pub const fn reads(self) -> SiteAct { match self { - SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128, + SiteEntry::Q3k + | SiteEntry::Q4k + | SiteEntry::Q5k + | SiteEntry::Q6k + | SiteEntry::Iq4xs => SiteAct::Q8x128, SiteEntry::Q8_0 => SiteAct::Q8x32, SiteEntry::F32 => SiteAct::F32, SiteEntry::Q5_0 => SiteAct::Q8Blocks32, @@ -300,7 +307,7 @@ impl SiteEntry { } /// One past the last [`SiteEntry`]: the span [`DENSE_TYPE`] covers. -const DENSE_SPAN: usize = SiteEntry::Bf16 as usize + 1; +const DENSE_SPAN: usize = SiteEntry::Iq4xs as usize + 1; /// Dense entry to the file type of the row that names it. The build refuses /// an entry two rows name, an entry no row names, and an entry past diff --git a/crates/gguf/src/qtype.rs b/crates/gguf/src/qtype.rs index a161da0c4..4fdb641ff 100644 --- a/crates/gguf/src/qtype.rs +++ b/crates/gguf/src/qtype.rs @@ -115,7 +115,7 @@ pub const QROWS: &[QRow] = &[ card: card!(- - Iq3s (Iq3s, Q8Act) Iq3s - - -) }, QRow { ty: T::IQ2_S, name: "iq2_s", blck: 256, bytes: 82, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::IQ4_XS, name: "iq4_xs", blck: 256, bytes: 136, dequant: true, host_act: H::Q8K, - card: card!(- - Iq4xs (Iq4xs, Q8Act) Iq4xs - - -) }, + card: card!(KQuant - Iq4xs (Iq4xs, Q8Act) Iq4xs Iq4xs - -) }, // The C scalars: ggml.c:621-649. QRow { ty: T::I8, name: "i8", blck: 1, bytes: 1, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::I16, name: "i16", blck: 1, bytes: 2, dequant: false, host_act: H::None, card: Card::NONE }, diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index e2f819456..e884d498a 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -38,7 +38,13 @@ const BLOCK: &str = "gated block"; /// The types a block's matrices read, one launch family each ([`site::file_site`]'s `allowed`); a model's /// coverage pin for the block lists the same set. -pub const BLOCK_TYPES: [SiteTy; 4] = [SiteTy::Q8_0, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K]; +pub const BLOCK_TYPES: [SiteTy; 5] = [ + SiteTy::Q8_0, + SiteTy::Q4K, + SiteTy::Q5K, + SiteTy::Q6K, + SiteTy::Iq4Xs, +]; /// The most columns [`enqueue_block_cols`] takes: the q8_0 block's multi-column launches' cap, kept for every plan. const COLS_MAX: usize = 8; @@ -48,6 +54,11 @@ fn refuse(what: &'static str, detail: String) -> GpuError { GpuError::Shape { what, detail } } +/// A site whose narrow and wide launches both read the 128-value q8_1 form. +fn q128(ty: SiteTy) -> bool { + ty.reads() == site::Form::Q8x128 && site::gemm_weight(ty).is_some() +} + /// A block's gate·up half, picked once at load from the file's types. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GateUp { @@ -105,19 +116,19 @@ impl BlockPlan { !matches!(self.gate_up, GateUp::Split { .. }) } - /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair or two K-quants for gate·up (the - /// same Q4_K or Q5_K fused, else split), any of [`BLOCK_TYPES`] for the down, which a split gate·up needs a - /// K-quant one of. Another mix is refused by name as `what`'s. + /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair or two sites over + /// 128-value q8_1 blocks (the same Q4_K or Q5_K fused, else split). A split pair needs a + /// down over that same activation form. Another mix is refused by name as `what`'s. /// /// # Errors - /// A gate and up that are not two q8_0 or two K-quants, or a split pair over a down that is not a K-quant. + /// A gate and up whose activation forms disagree, or a split pair over a down of another form. pub fn of_types( what: &'static str, [gate, up, down]: [SiteTy; 3], ) -> Result { let gate_up = match (gate, up) { (SiteTy::Q8_0, SiteTy::Q8_0) => GateUp::Ds41, - (g, u) if g.kquant() && u.kquant() => match (g == u, g) { + (g, u) if q128(g) && q128(u) => match (g == u, g) { (true, SiteTy::Q4K | SiteTy::Q5K) => GateUp::Fused(g), _ => GateUp::Split { gate: g, up: u }, }, @@ -125,12 +136,12 @@ impl BlockPlan { return Err(refuse( what, format!( - "gate is {gate} and up is {up}: a block takes two q8_0 or two K-quants" + "gate is {gate} and up is {up}: a block takes two q8_0 or two sites over 128-value q8_1" ), )); } }; - if matches!(gate_up, GateUp::Split { .. }) && !down.kquant() { + if matches!(gate_up, GateUp::Split { .. }) && !q128(down) { return Err(refuse( what, format!( @@ -167,7 +178,7 @@ impl BlockPlan { /// SwiGLU already wrote it). #[must_use] pub fn extra_launches(self) -> usize { - let down_quantize = usize::from(self.down.kquant()); + let down_quantize = usize::from(q128(self.down)); match self.gate_up { GateUp::Ds41 => down_quantize, GateUp::Fused(_) => 1 + down_quantize, @@ -280,7 +291,7 @@ impl BlockScratch { let (mut any, mut split_ff) = (false, 0); for (plan, ff) in blocks.into_iter().filter(|(p, _)| !p.is_q8()) { any = true; - if plan.down.kquant() && !ha.iter().any(|(f, _)| *f == ff) { + if q128(plan.down) && !ha.iter().any(|(f, _)| *f == ff) { ha.push((ff, Q8Act::with_k(stream, 1, ff)?)); } if matches!(plan.gate_up, GateUp::Split { .. }) { @@ -426,7 +437,7 @@ pub fn enqueue_block_act( gpu.layer_sink(blk.layer)?, h, )?; - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } } @@ -470,7 +481,7 @@ pub fn enqueue_block_act( return site::kgemv(gpu, &kk.gemv, (blk.plan.down, w, down), act, fault, y); } } - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } let (kk, sc) = kit(k, s)?; @@ -620,16 +631,6 @@ mod tests { BlockPlan::of_types(WHAT, tys) } - fn refused(r: Result, words: &[&str]) { - let msg = match r { - Err(GpuError::Shape { what, detail }) => format!("{what}: {detail}"), - other => panic!("expected a named shape refusal, got {other:?}"), - }; - for w in words { - assert!(msg.contains(w), "{msg:?} lacks {w:?}"); - } - } - /// The plan is the types' dispatch: the q8_0 triple is `BlockPlan::Q8`, one Q4_K or Q5_K for gate and up /// is fused, any other K-quant pair splits, and every plan reads back the types it came from. #[test] @@ -654,17 +655,17 @@ mod tests { } } - /// A mix no arm covers is refused by name, with the types: gate and up of two kinds, and a split pair over - /// a q8_0 down. #[test] - fn a_mix_no_arm_covers_is_refused() { - use SiteTy::{Q4K, Q6K, Q8_0}; - refused( - plan([Q8_0, Q4K, Q8_0]), - &["test", "gate is Q8_0", "up is Q4_K"], - ); - refused(plan([Q6K, Q8_0, Q6K]), &["gate is Q6_K", "up is Q8_0"]); - refused(plan([Q6K, Q6K, Q8_0]), &["split", "Q8_0 down"]); + fn incompatible_activation_forms_refuse_before_launch() { + use SiteTy::{Iq4Xs, Q4K, Q6K, Q8_0}; + for types in [ + [Q8_0, Q4K, Q8_0], + [Q6K, Q8_0, Q6K], + [Q6K, Q6K, Q8_0], + [Iq4Xs, Iq4Xs, Q8_0], + ] { + assert!(matches!(plan(types), Err(GpuError::Shape { .. }))); + } } /// The launches beyond the q8_0 block's two, by plan: 0 for the q8_0 block, and the K-quant arms' derived diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index 17d147ec9..9df96d40b 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1157,3 +1157,188 @@ fn no_batch() -> GpuError { missing: "the batch's buffers", } } + +#[cfg(test)] +mod tests { + use super::SharedWide; + use crate::block::{ + Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, + enqueue_block_cols_act, + }; + use crate::experts::ExpertKernels; + use bloomery_gpu::Gpu; + use bloomery_gpu::gemm::Gemm32Kernels; + use bloomery_gpu::kquant::Act; + use bloomery_gpu::weights::Weights; + use cuda_core::DeviceBuffer; + use gguf::Split; + use gguf::quant::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + + const N: usize = 256; + const FF: usize = 512; + const NAMES: [&str; 3] = [ + "blk.42.ffn_gate_shexp.weight", + "blk.42.ffn_up_shexp.weight", + "blk.42.ffn_down_shexp.weight", + ]; + + fn iq4_shared_file() -> (std::path::PathBuf, Split) { + let path = + std::env::temp_dir().join(format!("bloomery-iq4-shared-{}.gguf", std::process::id())); + let specs = [(N, FF, 0x1c00u16), (N, FF, 0x1800), (FF, N, 0x1800)]; + let payloads: Vec> = specs + .iter() + .enumerate() + .map(|(i, &(k, rows, scale))| { + let mut bytes = Vec::with_capacity(rows * (k / 256) * 136); + for row in 0..rows { + let d = if i == 2 && row % 2 == 1 { + scale - 0x400 + } else { + scale + }; + let mut block = [0x88u8; 136]; + block[..2].copy_from_slice(&d.to_le_bytes()); + block[2..4].copy_from_slice(&0xaaaau16.to_le_bytes()); + block[4..8].fill(0x11); + for _ in 0..k / 256 { + bytes.extend_from_slice(&block); + } + } + bytes + }) + .collect(); + let decls = NAMES + .iter() + .zip(specs) + .zip(&payloads) + .map(|((&name, (k, rows, _)), bytes)| TensorDecl { + name: name.to_owned(), + dims: vec![u64::try_from(k).unwrap(), u64::try_from(rows).unwrap()], + type_id: GgmlType::IQ4_XS.as_u32(), + nbytes: u64::try_from(bytes.len()).unwrap(), + }) + .collect(); + let layout = Layout::new(&[], decls).unwrap(); + let file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + let mut writer = Writer::new(file, layout).unwrap(); + for (name, bytes) in NAMES.iter().zip(payloads) { + writer.tensor(name, &bytes).unwrap(); + } + writer.finish().unwrap(); + let split = Split::open(&path).unwrap(); + (path, split) + } + + fn check_shared_rows(input: &[f32], rows: &[f32]) { + // Uniform positive rows quantize to code127 without integer rounding. + // The gate is at least128, so SiLU rounds to the gate itself. The + // scalar positive-dot upper bound has four input-width operations, + // two output-width operations and the scale/product roundings. + let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; + let eps = f64::from(f32::EPSILON); + let gamma = operations * eps / (1.0 - operations * eps); + for (c, row) in rows.chunks_exact(N).enumerate() { + let x = f64::from(input[c * N]); + for (r, &got) in row.iter().enumerate() { + let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; + assert!(got.is_finite()); + assert!( + (f64::from(got) - expected).abs() <= gamma * expected, + "column {c} row {r}: {got} vs {expected}, relative bound {gamma}" + ); + } + } + } + + #[test] + #[ignore = "needs a CUDA card"] + fn hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay() { + let (path, split) = iq4_shared_file(); + let gpu = Gpu::new().expect("card"); + let stream = gpu.stream(); + let weights = Weights::load_where(stream, &split, |name| NAMES.contains(&name)).unwrap(); + let plan = BlockPlan::of_file(&split, "IQ4_XS shared", NAMES, (FF, N)).unwrap(); + plan.check_resident(&weights, "IQ4_XS shared", NAMES, (FF, N)) + .unwrap(); + let block = Block { + plan, + names: NAMES, + ff: FF, + limit: 0.0, + layer: 42, + }; + let experts = ExpertKernels::load(gpu.context()).unwrap(); + let quant = KBlockKernels::load(&gpu).unwrap(); + let kernels = BlockKernels { + experts: &experts, + kquant: Some(&quant), + }; + let mut scratch = BlockScratch::new(stream, N, [(plan, FF)]).unwrap(); + let mut h = DeviceBuffer::zeroed(stream, FF).unwrap(); + let g32 = Gemm32Kernels::load(gpu.context()).unwrap(); + for cols in [1, 8, super::GATED_FROM] { + let input: Vec = (0..cols) + .flat_map(|c| std::iter::repeat_n([128.0, 256.0, 384.0][c % 3], N)) + .collect(); + let x = DeviceBuffer::from_host(stream, &input).unwrap(); + let mut y = DeviceBuffer::zeroed(stream, cols * N).unwrap(); + if cols < super::GATED_FROM { + enqueue_block_cols_act( + &gpu, + &weights, + &kernels, + &mut scratch, + (&block, Act::SiluMul), + cols, + BlockIo { + x: &x, + h: &mut h, + y: &mut y, + }, + ) + .unwrap(); + } else { + let types = plan.types(); + let mut wide = + SharedWide::new(&gpu, (&types[..2], &types[2..]), (cols, N, FF)).unwrap(); + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + .unwrap(); + let eager = y.to_host_vec(stream).unwrap(); + check_shared_rows(&input, &eager); + let graph = gpu + .capture(|_| { + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + }) + .unwrap(); + graph.launch(stream).unwrap(); + let replay = y.to_host_vec(stream).unwrap(); + assert!( + eager + .iter() + .zip(&replay) + .all(|(a, b)| a.to_bits() == b.to_bits()) + ); + } + check_shared_rows(&input, &y.to_host_vec(stream).unwrap()); + assert_eq!(gpu.take_fault().unwrap(), None); + } + stream.synchronize().unwrap(); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/gpu-gates/src/act_rule.rs b/crates/gpu-gates/src/act_rule.rs index a959cb768..33596c379 100644 --- a/crates/gpu-gates/src/act_rule.rs +++ b/crates/gpu-gates/src/act_rule.rs @@ -237,8 +237,11 @@ impl SiteRel { #[must_use] pub fn site_rel(ty: GgmlType, arm: Arm) -> Option { let ours = match (ty, arm) { - (GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K, _) - | (GgmlType::IQ3_XXS | GgmlType::IQ4_XS, Arm::Expert) => Some(Q8_1_BLOCK), + ( + GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K | GgmlType::IQ4_XS, + _, + ) + | (GgmlType::IQ3_XXS, Arm::Expert) => Some(Q8_1_BLOCK), // Bolt's IQ2_XXS: the routed `iq2_xxs_gemv_sel` and the dense `iq2_xxs_rows` read q8_1. (GgmlType::IQ2_XXS, Arm::Expert | Arm::Gemv) => Some(Q8_1_BLOCK), // Bolt's dense NVFP4 (`nvfp4_gemv`, the ssm_in/ssm_out) reads the engine's q8_1. @@ -1012,7 +1015,6 @@ mod site_tests { for ty in [ GgmlType::IQ3_S, GgmlType::IQ3_XXS, - GgmlType::IQ4_XS, GgmlType::Q5_1, GgmlType::IQ4_NL, ] { @@ -1040,43 +1042,6 @@ mod site_tests { } } - /// Ours is the form the launches read, as their sources say it: the text of the kernel - /// table's `SiteEntry::reads` (which `site.rs`'s `SiteTy::reads` returns) and of the 32-value - /// GEMM family's module doc, so a launch that moves to another form is red here until the - /// table follows it. - #[test] - fn site_rel_reads_the_forms_the_launches_name() { - let gpu = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../gpu/src"); - let read = |file: &str| { - std::fs::read_to_string(gpu.join(file)) - .unwrap_or_else(|e| panic!("crates/gpu/src/{file}: {e}")) - }; - let (site, gemm, grouped) = ( - read("../../gguf/src/card.rs"), - read("gemm/mod.rs"), - read("gemm/grouped.rs"), - ); - for line in [ - "SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128", - "SiteEntry::Q8_0 => SiteAct::Q8x32", - "SiteEntry::F32 => SiteAct::F32", - "SiteEntry::Q5_0 => SiteAct::Q8Blocks32", - "q8_1 blocks of 128 values", - ] { - assert!(site.contains(line), "kernels.rs no longer says {line:?}"); - } - assert!( - gemm.contains("(Q8_0, IQ4_NL, Q5_1), over [`GemmAct32`]"), - "gemm/mod.rs no longer names the 32-value family" - ); - for variant in ["Iq3Xxs", "Iq4Xs"] { - assert!( - grouped.contains(variant), - "the grouped GEMM no longer decodes {variant}" - ); - } - } - /// ik's side of the table is the transcription this module already holds for the K-quants /// ([`Rule::of`]), and so is ours for Q3_K and Q4_K; Q5_K's ours differs from it by design (the /// V4.1 gemv reads f32, the bodies' site reads q8_1). diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index cc455f72f..d5d581e54 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -104,14 +104,14 @@ pub enum Order { /// file type its form holds ([`SiteTy::of_ggml`]; an F32 plane is the f32 /// site whatever type it was widened from). `None` for a form no dense entry /// reads: a stack in the file's blocks of a type whose entry reads another -/// layout (Q8_0's planes, an i-quant), a derived weight and an NVFP4 stack. +/// layout (Q8_0's planes), a derived weight and an NVFP4 stack. /// The resident form is this crate's, so the map is here, beside the /// launches that read it, and not on the chain's [`SiteTy`]. #[must_use] pub fn of_resident(w: &DevWeight) -> Option { let ty = match w { DevWeight::KQuant { ty, .. } => { - return SiteTy::of_ggml(*ty).filter(|s| s.kquant()); + return SiteTy::of_ggml(*ty).filter(|s| s.kquant() || *s == SiteTy::Iq4Xs); } DevWeight::Q5_0 { .. } => GgmlType::Q5_0, DevWeight::Q5_1 { .. } => GgmlType::Q5_1, @@ -126,7 +126,7 @@ pub fn of_resident(w: &DevWeight) -> Option { SiteTy::of_ggml(ty) } -/// How [`kgemv`] lays a site of type `ty` out: Q3_K, Q4_K and Q6_K row-major, Q5_K and NVFP4 token-major. +/// How [`kgemv`] lays a site out: Q3_K, Q4_K and Q6_K row-major, Q5_K, IQ4_XS and NVFP4 token-major. /// `None` for a type [`kgemv`] does not run: Q8_0, F32, MXFP8, FP8_B128 and BF16 ([`gemv`]), Q5_0 /// ([`blocks32_gemv`], whose row-major layout `y[r · m + c]` is that function's own), and MLX_A4, /// a backend's family no card launch reads. @@ -134,7 +134,7 @@ pub fn of_resident(w: &DevWeight) -> Option { pub fn kgemv_order(ty: SiteTy) -> Option { match ty { SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q6K => Some(Order::RowMajor), - SiteTy::Q5K | SiteTy::Nvfp4 => Some(Order::TokenMajor), + SiteTy::Q5K | SiteTy::Iq4Xs | SiteTy::Nvfp4 => Some(Order::TokenMajor), SiteTy::Q5_0 | SiteTy::Q8_0 | SiteTy::F32 @@ -666,6 +666,22 @@ pub fn kgemv( y, ) } + SiteTy::Iq4Xs => { + let wt = kq(w, WHAT, name)?; + kk.kq.enqueue_gemv_sel( + gpu.stream(), + GgmlType::IQ4_XS, + &SelDown { + w: wt, + act, + sel: &kk.one, + n_slots: act.m(), + rows_per_expert: wt.rows(), + }, + fault, + y, + ) + } SiteTy::Nvfp4 => kk .nv .enqueue_gemv(gpu.stream(), w.nvfp4(WHAT, name)?, act, y), @@ -752,9 +768,11 @@ impl<'a> SiteWeight<'a> { /// name as `what`. fn check(self, what: &'static str, ty: SiteTy) -> Result<(), GpuError> { match self { - SiteWeight::Words { name, .. } if !ty.kquant() => Err(GpuError::shape( + SiteWeight::Words { name, .. } if gemm_weight(ty).is_none() => Err(GpuError::shape( what, - format!("{name} is {ty} on words a caller holds; only a K-quant's words launch"), + format!( + "{name} is {ty} on words a caller holds; its grouped GEMM reads no such words" + ), )), SiteWeight::Part { name, .. } if !BY_NAME.contains(&ty) => Err(GpuError::shape( what, @@ -1006,21 +1024,23 @@ pub fn gemm( src.check(WHAT, ty)?; match ty { SiteTy::Nvfp4 => Err(no_wide_gemm(WHAT, name, ty)), - SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K => k.gemm.enqueue_gemm( - stream, - GemmArgs { - ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, - w: match src { - SiteWeight::Words { w, .. } => w, - _ => kq(w, WHAT, name)?, + SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K | SiteTy::Iq4Xs => { + k.gemm.enqueue_gemm( + stream, + GemmArgs { + ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, + w: match src { + SiteWeight::Words { w, .. } => w, + _ => kq(w, WHAT, name)?, + }, + rows_per_expert: rows, + act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, + route, + input, + y, }, - rows_per_expert: rows, - act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, - route, - input, - y, - }, - ), + ) + } SiteTy::Q8_0 | SiteTy::F32 | SiteTy::Mxfp8 | SiteTy::Fp8B128 | SiteTy::Bf16 => { dense()?; let (site, part) = src.rows(WHAT)?; @@ -1470,95 +1490,42 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, Order, SiteTy, SwigluLaunch, WideActs, file_site, gemm_weight, kgemv_order, - swiglu_launch, + Form, SiteTy, SwigluLaunch, WideActs, file_site, gemm_weight, kgemv_order, swiglu_launch, }; - use crate::gemm::GemmWeight; use crate::kquant::Act; use gguf::Split; use gguf::quant::GgmlType; - use gguf::write::{Layout, TensorDecl, Writer}; - use model::placement::kernels; - use std::fs::File; - use std::io::BufWriter; - - /// A site's type is read from the file's header alone: a Q8_0 matrix of - /// two rows of 64 is read as Q8_0 where Q8_0 is launched, and refused by - /// name where only K-quants are; a Q3_K, a Q5_K, a Q5_0 and an MXFP8 one - /// (bloomery's id, sized by the split) are read as theirs; an IQ4_XS one (a type no site launch reads) and a matrix of - /// another shape are refused by name. + use gguf::write::{Layout, TensorDecl}; + #[test] - fn a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused() { - let dir = std::env::temp_dir().join(format!("bloomery-site-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap_or_else(|e| panic!("{e}")); - let path = dir.join("m.gguf"); - let decls = [ - ("q8", vec![64, 2], 8u32, 2 * 2 * 34), - ("iq", vec![256, 1], 23, 136), - ("q3", vec![256, 2], 11, 2 * 110), - ("q5", vec![256, 2], 13, 2 * 176), - ("q50", vec![64, 2], 6, 2 * 2 * 22), - ("mx", vec![128, 2], GgmlType::MXFP8.as_u32(), 2 * 4 * 33), - ("bf", vec![64, 2], GgmlType::BF16.as_u32(), 2 * 64 * 2), - ]; - let tensors = decls - .iter() - .map(|(name, dims, type_id, nbytes)| TensorDecl { - name: (*name).to_string(), - dims: dims.clone(), - type_id: *type_id, - nbytes: *nbytes, - }) - .collect(); - let layout = Layout::new(&[], tensors).unwrap_or_else(|e| panic!("{e}")); - let file = BufWriter::new(File::create(&path).unwrap_or_else(|e| panic!("{e}"))); - let mut w = Writer::new(file, layout).unwrap_or_else(|e| panic!("{e}")); - for (name, _, _, nbytes) in &decls { - w.tensor(name, &vec![0u8; *nbytes as usize]) - .unwrap_or_else(|e| panic!("{e}")); - } - w.finish().unwrap_or_else(|e| panic!("{e}")); - let split = Split::open(&path).unwrap_or_else(|e| panic!("{e}")); - let all = [ - SiteTy::Q3K, - SiteTy::Q4K, - SiteTy::Q5K, - SiteTy::Q6K, - SiteTy::Q5_0, - SiteTy::Q8_0, - SiteTy::F32, - SiteTy::Nvfp4, - SiteTy::Mxfp8, - SiteTy::Bf16, - ]; - for (name, k, want) in [ - ("q8", 64, SiteTy::Q8_0), - ("q3", 256, SiteTy::Q3K), - ("q5", 256, SiteTy::Q5K), - ("q50", 64, SiteTy::Q5_0), - ("mx", 128, SiteTy::Mxfp8), - ("bf", 64, SiteTy::Bf16), + fn iq4_sites_keep_shape_and_launch_admission_separate() { + let decl = |name: &str, ty: GgmlType, bytes| TensorDecl { + name: name.to_owned(), + dims: vec![256, 2], + type_id: ty.as_u32(), + nbytes: bytes, + }; + let layout = Layout::new( + &[], + vec![ + decl("shared", GgmlType::IQ4_XS, 2 * 136), + decl("unsupported", GgmlType::IQ3_S, 2 * 110), + ], + ) + .unwrap(); + let file = Split::header_only("sites.gguf".into(), &layout).unwrap(); + assert_eq!( + file_site(&file, "test", "shared", (2, 256), SiteTy::ALL).ok(), + Some(SiteTy::Iq4Xs) + ); + for (name, shape, allowed) in [ + ("shared", (1, 256), SiteTy::ALL), + ("shared", (2, 512), SiteTy::ALL), + ("shared", (2, 256), &[SiteTy::Q4K][..]), + ("unsupported", (2, 256), SiteTy::ALL), ] { - let got = file_site(&split, "test", name, (2, k), &all); - assert_eq!(got.ok(), Some(want), "{name}"); + assert!(file_site(&file, "test", name, shape, allowed).is_err()); } - let refused = [ - ( - "q8", - (2, 64), - &[SiteTy::Q4K, SiteTy::Q6K][..], - "q8 is q8_0 in the file", - ), - ("iq", (1, 256), &all[..], "iq is iq4_xs in the file"), - ("q8", (4, 64), &all[..], "it takes 4 rows of 64"), - ]; - for (name, shape, allowed, want) in refused { - match file_site(&split, "test", name, shape, allowed) { - Err(e) => assert!(e.to_string().contains(want), "{name}: {e}"), - Ok(t) => panic!("{name} {shape:?} read as {t}"), - } - } - let _ = std::fs::remove_dir_all(&dir); } /// The SwiGLU's launch is the down's form under the rule: every K-quant @@ -1616,72 +1583,6 @@ mod tests { } } - /// Every family is the kernel table's dense row of one file type: a type - /// has a family exactly where the table names a dense entry for it (F16, - /// which loads widened to f32, is the f32 site of its widened rows), the - /// family maps back to that type and is named by it in upper case, its - /// K-quant GEMM is the one the table names for the type, it is a - /// `kgemv` exactly where it reads the q8_1 rows of 128 values, and it - /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or - /// the q8 blocks of 32. - #[test] - fn every_family_is_the_tables_dense_row_of_one_type() { - // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. - let own = [ - GgmlType::F8E4M3_ROWS, - GgmlType::MXFP8, - GgmlType::NVFP4_ROWS, - GgmlType::FP8_B128, - ]; - for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { - let family = SiteTy::of_ggml(ty); - if ty == GgmlType::F16 { - assert_eq!( - family, - Some(SiteTy::F32), - "{ty}: the f32 site of its widened rows" - ); - assert_eq!(kernels::dense(ty), None, "{ty}"); - continue; - } - assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); - if let Some(s) = family { - assert_eq!(s.ggml(), ty, "{s}"); - } - } - let expected = [ - (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), - (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), - (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), - (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), - (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), - (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), - (SiteTy::F32, GgmlType::F32, "F32", None), - (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), - (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), - (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), - (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - ]; - assert_eq!(SiteTy::ALL.len(), expected.len()); - for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { - assert_eq!(s, want); - assert_eq!(s.ggml(), ty); - assert_eq!(s.to_string(), label); - assert_eq!(gemm_weight(s), grouped); - assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); - let by_name = model::placement::accepts::dense_by_name(ty); - assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); - if let Some(entry) = by_name { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - } - assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); - assert_eq!(kgemv_order(SiteTy::Q5_0), None); - assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); - assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); - assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); - } - #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 1afdc9890..245bae230 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -231,7 +231,8 @@ pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { /// is `None`. An NVFP4 site has a gemv and no wide GEMM, which the body's /// prompt units run every dense site through, and is `None`. A BF16 site /// reads the words only a load under `ModelTensors::bf16_words` keeps, which -/// the qwen3moe files do not set, and is `None`. The body's load and the +/// the qwen3moe files do not set, and is `None`. Its legacy shared-expert +/// dispatch does not feed IQ4_XS sites either. The body's load and the /// coverage census both read this, so a site the arena cannot feed is /// refused at load and listed together. #[must_use] @@ -239,7 +240,7 @@ pub const fn qwen3moe_dense(ty: GgmlType) -> Option { let Some(entry) = kernels::dense(ty) else { return None; }; - if let SiteEntry::Nvfp4 | SiteEntry::Bf16 = entry { + if let SiteEntry::Nvfp4 | SiteEntry::Bf16 | SiteEntry::Iq4xs = entry { return None; } match entry.reads() { diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 8b9d69940..5d7b4425b 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -621,32 +621,6 @@ mod tests { ); } - // PIN(2026-10-16): the dense pairs gain NVFP4 (`nvfp4_gemv` through - // `site::kgemv`); every other pair is unchanged. - // PIN(2026-10-19): and bloomery's own MXFP8 (`mxfp8_gemv` through - // `site::gemv`); every other pair is unchanged. - // PIN(2026-10-20): and bloomery's own FP8_B128 (`fp8_b128_gemv` through - // `site::gemv`); BF16 also uses `bf16_gemv` and `bf16_tile_gemm` over its words. - #[test] - fn dense_admits_exactly_the_pinned_pairs() { - assert_eq!( - rows(dense), - [ - (GgmlType::F32, SiteEntry::F32), - (GgmlType::Q5_0, SiteEntry::Q5_0), - (GgmlType::Q8_0, SiteEntry::Q8_0), - (GgmlType::Q3_K, SiteEntry::Q3k), - (GgmlType::Q4_K, SiteEntry::Q4k), - (GgmlType::Q5_K, SiteEntry::Q5k), - (GgmlType::Q6_K, SiteEntry::Q6k), - (GgmlType::BF16, SiteEntry::Bf16), - (GgmlType::NVFP4, SiteEntry::Nvfp4), - (GgmlType::MXFP8, SiteEntry::Mxfp8), - (GgmlType::FP8_B128, SiteEntry::Fp8B128), - ] - ); - } - /// A dense entry is named by one type's row, so its file type is that /// row's type. #[test] @@ -686,94 +660,6 @@ mod tests { ); } - /// The layout a file tensor of each type loads in, and the one a routed - /// stack loads in under the default expert rule: every type's but q5_K's, - /// which that rule does not admit, and mxfp8's, which has no card expert - /// entry. - #[test] - fn the_card_layouts_are_the_pinned_pairs() { - assert_eq!( - rows(CardFormat::of), - [ - (GgmlType::F32, CardFormat::F32), - (GgmlType::F16, CardFormat::Bf16AsF32), - (GgmlType::Q5_0, CardFormat::Q5_0), - (GgmlType::Q5_1, CardFormat::Q5_1), - (GgmlType::Q8_0, CardFormat::Q8_0Planes), - (GgmlType::Q3_K, CardFormat::KQuant), - (GgmlType::Q4_K, CardFormat::KQuant), - (GgmlType::Q5_K, CardFormat::KQuant), - (GgmlType::Q6_K, CardFormat::KQuant), - (GgmlType::BF16, CardFormat::Bf16AsF32), - (GgmlType::NVFP4, CardFormat::Nvfp4Planes), - (GgmlType::IQ2_XXS, CardFormat::KQuant), - (GgmlType::MXFP8, CardFormat::Mxfp8Planes), - (GgmlType::FP8_B128, CardFormat::Fp8B128Planes), - ] - ); - for ty in ALL { - let routed = - CardFormat::of(ty).filter(|_| !matches!(ty, GgmlType::Q5_K | GgmlType::MXFP8)); - assert_eq!(CardFormat::of_routed(ty), routed, "{ty:?}"); - } - } - - /// The types each layout holds: the types it is the load layout of, the - /// K-quants as decoded f32, bf16 as raw words, and in the file's blocks - /// every type a routed entry reads in them. - #[test] - fn each_layout_holds_exactly_the_pinned_types() { - let held = |f: CardFormat| -> Vec { - ALL.into_iter().filter(|&ty| f.holds(ty)).collect() - }; - assert_eq!( - held(CardFormat::KQuant), - [ - GgmlType::Q5_1, - GgmlType::Q8_0, - GgmlType::Q3_K, - GgmlType::Q4_K, - GgmlType::Q5_K, - GgmlType::Q6_K, - GgmlType::MXFP4, - GgmlType::IQ2_XXS, - GgmlType::IQ2_XS, - GgmlType::IQ3_XXS, - GgmlType::IQ4_NL, - GgmlType::IQ3_S, - GgmlType::IQ4_XS, - ] - ); - assert_eq!( - held(CardFormat::Bf16AsF32), - [ - GgmlType::F16, - GgmlType::Q3_K, - GgmlType::Q4_K, - GgmlType::Q5_K, - GgmlType::Q6_K, - GgmlType::BF16, - GgmlType::IQ2_XXS, - ] - ); - // PIN(2026-10-22): independent BF16 heads also occupy MXFP8 planes. - assert_eq!( - held(CardFormat::Mxfp8Planes), - [GgmlType::BF16, GgmlType::MXFP8] - ); - for (f, ty) in [ - (CardFormat::Q5_0, GgmlType::Q5_0), - (CardFormat::Q5_1, GgmlType::Q5_1), - (CardFormat::Q8_0Planes, GgmlType::Q8_0), - (CardFormat::Nvfp4Planes, GgmlType::NVFP4), - (CardFormat::Fp8B128Planes, GgmlType::FP8_B128), - (CardFormat::F32, GgmlType::F32), - (CardFormat::Bf16Raw, GgmlType::BF16), - ] { - assert_eq!(held(f), [ty], "{f:?}"); - } - } - /// A card entry reads a layout the card loader can build: every type a /// row names has a [`CardFormat`] of its own, or is one of the file-block /// types a program's routed stack keeps as `KQuant`. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..012ef946d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -66,6 +66,31 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Ordinary IQ4_XS shared-site implementation + +IQ4_XS now has an ordinary file-layout/dense entry in QROWS and a common site family. +The loader keeps its original block words; `site::kgemv` uses the existing IQ4_XS +entry over one matrix and `site::gemm` uses the existing grouped GEMM. The common +gated block selects a split gate/up and sizes the down's q8_1 scratch from the +actual activation form. Its column walk and the shared wide-prompt owner use +those same sites. IQ4_XS is not relabeled as a routed expert or requantized. + +A compiled public `file_site` consumer on the pinned header refused all three +layer42 shared matrices before this change and accepts their IQ4_XS/Q8x128 +bindings after it. This reads no payload and proves neither loading nor arithmetic. +`hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay`, owned by +`gate-gpu-ds41-lib`, covers real packed-weight loading, unequal matrix widths, +one/eight-column block execution and the wide-prompt transition/captured replay. +Its nonnegative constant-weight case derives a scalar floating-operation bound; +it is not a whole-model accuracy pin. The hardware case has not been run here. + +Existing program-specific admission remains unchanged: in particular the legacy +Qwen projection/shared dispatch does not gain this site automatically. Flint +still has no Program, attention/router binding, body or serve seat. Run the new +consumer gate and the affected regressions on NVIDIA before claiming device +support; reconcile full-model coverage only with the actual body integration. + + ## 1. Sources and observed artifact From a4f3919e445f7d1b3fd3c739731a75dda2adea71 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:34:14 +0900 Subject: [PATCH 03/85] gpu: size mixed expert scratch from selected resident forms --- crates/gpu-deepseek41/src/body.rs | 2 +- .../gpu-gates/src/bin/gate_glm5next_nodes.rs | 2 +- .../src/bin/gate_glm5next_twocard.rs | 2 +- crates/gpu-gates/src/bin/gate_load_v41.rs | 1 + .../gpu-gates/src/bin/gate_solar2_twocard.rs | 2 +- crates/gpu-gates/src/bin/generate_glm5next.rs | 2 +- .../src/bin/shared/glm5next_place.rs | 11 +- .../src/bin/shared/routed_legs/glm5next.rs | 3 +- .../src/bin/shared/serve_seats/glm.rs | 2 +- .../src/bin/shared/serve_seats/hybrid.rs | 15 +- .../src/bin/shared/serve_seats/solar2.rs | 9 +- .../src/bin/shared/serve_seats/tiered.rs | 12 +- .../gpu-gates/src/bin/shared/solar2_seat.rs | 2 +- crates/gpu-gates/src/generate.rs | 1 + crates/gpu-glm5next/src/tier.rs | 1 + crates/gpu-mimo2/src/card.rs | 1 + crates/gpu-minimaxm3/src/ffn.rs | 1 + crates/gpu-solar2/src/ffn.rs | 1 + crates/gpu-solar2/src/tier.rs | 1 + crates/gpu/src/arch/qwen3moe/body38.rs | 2 +- crates/gpu/src/card_experts.rs | 374 +++++++++++++++++- crates/gpu/src/card_leg.rs | 18 +- crates/gpu/src/host/batch.rs | 28 +- crates/gpu/src/host/leg_tier.rs | 43 +- crates/gpu/src/host/node.rs | 14 +- crates/model/src/arch/deepseek41/kv.rs | 1 + crates/model/src/arch/glm5next/place.rs | 134 +++---- crates/model/src/arch/qwen35moe/place.rs | 6 + crates/model/src/arch/solar2/place.rs | 157 ++++++-- crates/model/tests/placement.rs | 1 + crates/model/tests/qwen4exp_mtp_reserve.rs | 1 + crates/model/tests/solar_open2_meta.rs | 1 + crates/placement/src/accepts.rs | 37 ++ crates/placement/src/kernels.rs | 157 +++++++- crates/placement/src/placement.rs | 349 ++++++++++++++-- crates/placement/src/placement/nodes_scale.rs | 2 + .../placement/src/placement/qwen38_cards.rs | 2 + crates/placement/src/placement/workstation.rs | 125 +++++- docs/research/flint-attach.md | 19 + 39 files changed, 1321 insertions(+), 221 deletions(-) diff --git a/crates/gpu-deepseek41/src/body.rs b/crates/gpu-deepseek41/src/body.rs index 0ef84a357..051d00605 100644 --- a/crates/gpu-deepseek41/src/body.rs +++ b/crates/gpu-deepseek41/src/body.rs @@ -2563,7 +2563,7 @@ impl Body { // The tiers' batch staging is reserved on their cards by the plan, // so it is made at load, not at the first prompt. hybrid.prepare_batch(gpu.context(), prefill::T_MAX)?; - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; Some(TierPiece::new(gpu, hybrid.slots(), PAIR_ROWS)?) }; let ced = ced::Ced::new(&CedLayer::table(hp), seq.ring_rows(), cfg.ced); diff --git a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs index 8b9f34dc6..0845bd66a 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs @@ -30,7 +30,7 @@ mod gate { let file = Split::open(&path)?; let inputs = PlanInputs::describe(&file)?; let (mut machine, node) = - crate::node_history::machine(inputs.model.layers, tier_batch(&inputs.hp))?; + crate::node_history::machine(inputs.model.layers, tier_batch(&inputs)?)?; reserve_checkpoints( &mut machine, Seqs::priced(1, checkpoint_price(&inputs)?), diff --git a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs index 5451922c0..1710e416d 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs @@ -515,7 +515,7 @@ mod gate { fn bp_machine(inputs: &PlanInputs) -> Result Machine + Copy, GateError> { glm_machine( Place::Bp, - Some(place::tier_batch(&inputs.hp)), + Some(place::tier_batch(inputs)?), 1, checkpoint_price(inputs)?, u64::try_from(CTX)?, diff --git a/crates/gpu-gates/src/bin/gate_load_v41.rs b/crates/gpu-gates/src/bin/gate_load_v41.rs index ddd187806..e9304096d 100644 --- a/crates/gpu-gates/src/bin/gate_load_v41.rs +++ b/crates/gpu-gates/src/bin/gate_load_v41.rs @@ -242,6 +242,7 @@ mod gate { staging: 0, scratch: 0, host: 0, + leg: None, }, ) .map_err(|e| format!("plan (b): {e}"))?; diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index dae44759b..9d413942a 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs.hp)))?; + .machine(None, Some(place::tier_batch(&inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu-gates/src/bin/generate_glm5next.rs b/crates/gpu-gates/src/bin/generate_glm5next.rs index 27a6976b5..26a907960 100644 --- a/crates/gpu-gates/src/bin/generate_glm5next.rs +++ b/crates/gpu-gates/src/bin/generate_glm5next.rs @@ -761,7 +761,7 @@ mod cli { Some(lane_at(placement)?) }; let place = placement.name(); - let tier_batch = glm_place::tier_batch(placement, &inputs.hp); + let tier_batch = glm_place::tier_batch(placement, &inputs)?; let machine = glm_machine( placement, tier_batch, diff --git a/crates/gpu-gates/src/bin/shared/glm5next_place.rs b/crates/gpu-gates/src/bin/shared/glm5next_place.rs index ac6a3dd03..36f0e5eed 100644 --- a/crates/gpu-gates/src/bin/shared/glm5next_place.rs +++ b/crates/gpu-gates/src/bin/shared/glm5next_place.rs @@ -10,7 +10,6 @@ use bloomery_gpu_gates::beside::{self, Beside}; use bloomery_gpu_gates::generate::{Place, TierRule, choose_ctx_of}; use bloomery_gpu_gates::residency38::glm_machine; use bloomery_gpu_glm5next::checkpoint_price; -use model::arch::glm5next::hparams::Hparams; use model::arch::glm5next::place::{ KdaLanes, NextnInputs, ORACLE_POSITIONS, PlaceError, PlanInputs, }; @@ -41,10 +40,12 @@ pub fn parse(v: &str) -> Result { } /// The expert tier's prompt-batch bytes `place`'s plan reserves for the file -/// of hyperparameters `hp` (`model::arch::glm5next::place::tier_batch`), +/// described by `inputs` (`model::arch::glm5next::place::tier_batch`), /// which `Place::machine` takes; `None` under a placement with no tier card. -pub fn tier_batch(place: Place, hp: &Hparams) -> Option { - (!place.tier_cards().is_empty()).then(|| model::arch::glm5next::place::tier_batch(hp)) +pub fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| model::arch::glm5next::place::tier_batch(inputs)) + .transpose()?) } /// The experts `place`'s tier cards hold in its plan at `ctx` positions @@ -65,7 +66,7 @@ pub fn tier_experts( ) -> Result { let machine = glm_machine( place, - tier_batch(place, &inputs.hp), + tier_batch(place, inputs)?, slots, checkpoint_price(inputs)?, ctx, diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs index 42307899b..5db7eebca 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs @@ -172,7 +172,7 @@ fn machine_of( slots: usize, ) -> Result { let price = checkpoint_price(inputs)?; - let tier_batch = glmplace::tier_batch(place, &inputs.hp); + let tier_batch = glmplace::tier_batch(place, inputs)?; Ok(glm_machine(place, tier_batch, slots, price, ctx)?( inputs.model.layers, )) @@ -242,7 +242,6 @@ fn place_of(p: &PlaceError) -> Outcome { PlaceError::Placement(e) => return refused_of(e), PlaceError::PastOracle { .. } => "past-oracle", PlaceError::FrontOver { .. } => "front-over", - PlaceError::TierLeg { .. } => "tier-leg", PlaceError::Broken(_) | PlaceError::NextnCards { .. } | PlaceError::Slots { .. } => { return Outcome::Red(p.to_string()); } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs index 68b6e54f5..9e4e6f3af 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs @@ -503,7 +503,7 @@ impl Placed { (set, slots): (Option, usize), (images, side): (&Images<'_>, Option<&Side>), ) -> Result { - let tier_batch = glm_place::tier_batch(place, &inputs.hp); + let tier_batch = glm_place::tier_batch(place, inputs)?; // Every machine reserves its checkpoints at the slot context it is // laid for: the choice's probes each at their own, the load's at the // context it is granted; and the encoder's reserve, as the printed plan does. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs index 0317e82a3..8d16d9516 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs @@ -79,12 +79,16 @@ pub trait Family: Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s /// tier card ([`Tiered::tier_batch`]); a body that serves no tier card /// reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// The tags the model's description names for its think span, which the @@ -324,7 +328,10 @@ impl Tiered for Hybrid { F::think_tags(inputs).map(Some) } - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { F::tier_batch(place, inputs) } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs index dc1bd2b21..c53248f87 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs @@ -166,8 +166,13 @@ impl Family for Solar { /// The tier's prompt-batch bytes of the file's widths ([`place::tier_batch`]), reserved on /// the placement's tier card and the host. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, bloomery_gpu_gates::GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(inputs)) + .transpose()?) } } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs index 7681f03e9..e80f07737 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs @@ -101,12 +101,16 @@ pub trait Tiered: Seat + Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s tier card, for the file /// `inputs` describes; `None` under a placement with no tier card. A body that serves no tier /// card reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// The think tags the model's description names, if the family splits its reasoning on them. @@ -214,7 +218,7 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result, GateErro // prompt-batch bytes the file's widths give. let machine = match arm { Arm::A => crate::gate_card::plan_gate(inputs.model.layers), - Arm::Bp => workstation::plan_bp(inputs.model.layers, None, place::tier_batch(&inputs.hp)), + Arm::Bp => workstation::plan_bp(inputs.model.layers, None, place::tier_batch(&inputs)?), }; let place = PlanLevers::from_levers(&levers)?; let plan = inputs.plan(&machine, u64::try_from(ctx)?, &place)?; diff --git a/crates/gpu-gates/src/generate.rs b/crates/gpu-gates/src/generate.rs index 05c90a4f2..49ee7a38c 100644 --- a/crates/gpu-gates/src/generate.rs +++ b/crates/gpu-gates/src/generate.rs @@ -716,6 +716,7 @@ impl Place { staging: 0, scratch: 0, host: 0, + leg: None, }; self.machine(None, (!self.tier_cards().is_empty()).then_some(none)) } diff --git a/crates/gpu-glm5next/src/tier.rs b/crates/gpu-glm5next/src/tier.rs index e540f4ed2..aee07ed13 100644 --- a/crates/gpu-glm5next/src/tier.rs +++ b/crates/gpu-glm5next/src/tier.rs @@ -62,6 +62,7 @@ impl LegFamily for Glm { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: model::placement::accepts::Program::Glm5nextBody, } } diff --git a/crates/gpu-mimo2/src/card.rs b/crates/gpu-mimo2/src/card.rs index 728908ae7..147124f1a 100644 --- a/crates/gpu-mimo2/src/card.rs +++ b/crates/gpu-mimo2/src/card.rs @@ -26,6 +26,7 @@ fn stacks(l: usize) -> [String; 3] { /// MiMo's card experts: 256 routed experts, 8 a token. const FAMILY: CardFamily = CardFamily { what: "mimo2 CardExperts", + program: model::placement::accepts::Program::Mimo2Body, n_expert: N_EXPERT, n_used: N_USED, stacks, diff --git a/crates/gpu-minimaxm3/src/ffn.rs b/crates/gpu-minimaxm3/src/ffn.rs index 828b0897d..3850f6ba4 100644 --- a/crates/gpu-minimaxm3/src/ffn.rs +++ b/crates/gpu-minimaxm3/src/ffn.rs @@ -20,6 +20,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{ /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "minimaxm3 CardExperts", + program: model::placement::accepts::Program::MiniMaxM3Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/ffn.rs b/crates/gpu-solar2/src/ffn.rs index a0f1589d5..7be370e0a 100644 --- a/crates/gpu-solar2/src/ffn.rs +++ b/crates/gpu-solar2/src/ffn.rs @@ -17,6 +17,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{GATED_FROM, RoutedDims, back, f /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "solar2 CardExperts", + program: model::placement::accepts::Program::Solar2Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/tier.rs b/crates/gpu-solar2/src/tier.rs index 071ce8d20..e23773d93 100644 --- a/crates/gpu-solar2/src/tier.rs +++ b/crates/gpu-solar2/src/tier.rs @@ -64,6 +64,7 @@ impl LegFamily for Solar2 { n_used: experts.n_used, n_embd: inputs.hp.n_embd, ff: experts.ff, + program: model::placement::accepts::Program::Solar2Body, } } diff --git a/crates/gpu/src/arch/qwen3moe/body38.rs b/crates/gpu/src/arch/qwen3moe/body38.rs index 4c1465c29..69e666d79 100644 --- a/crates/gpu/src/arch/qwen3moe/body38.rs +++ b/crates/gpu/src/arch/qwen3moe/body38.rs @@ -1982,7 +1982,7 @@ impl Body38 { hybrid.prepare_batch(gpu.context(), host_cols)?; if tiered { let cards: Vec = tiers.iter().map(|t| t.card).collect(); - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; } // The PLE table's tier is the plan's: the host set built above holds // it on the host; on the NVMe tier its reader reads it. diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 64282ef47..4542fe4ac 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -31,14 +31,22 @@ use cuda_core::{CudaStream, DeviceBuffer}; use model::arch::models::{Ffn, LayerSpec}; +use model::placement::accepts::{self, LegArms, Program}; +use model::placement::kernels::{self, DownForms}; -use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, leg_bytes}; +use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum}; use crate::card_slots::{CardSlotKernels, Places}; use crate::hybrid::SlotMap; use crate::kquant::Act; use crate::weights::Weights; use crate::{DeviceTensor, Gpu, GpuError}; +/// The selected program, rather than a union of unrelated launches, admits stacks. +const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected card gate·up arm runs", + "a resident routed down stack the selected card down arm runs", +); + /// What a family fixes of its card experts. #[derive(Clone, Copy)] pub struct CardFamily { @@ -50,6 +58,8 @@ pub struct CardFamily { pub n_used: usize, /// Layer `l`'s gate, up and down stacks' names. pub stacks: fn(usize) -> [String; 3], + /// The program whose selected gate·up and down launches run the stacks. + pub program: Program, } /// The kernels a run launches: the card leg's (its sum stage included), and @@ -57,6 +67,7 @@ pub struct CardFamily { struct CardKernels { leg: CardLeg, places: CardSlotKernels, + arms: LegArms, } impl CardFamily { @@ -89,9 +100,10 @@ impl CardBufs { n: usize, ff: usize, cap: usize, + downs: DownForms, ) -> Result { Ok(CardBufs { - leg: LegBufs::new(stream, family.shape(n, ff, cap))?, + leg: LegBufs::for_downs(stream, family.shape(n, ff, cap), downs)?, acc: DeviceBuffer::zeroed(stream, cap * n)?, pre: DeviceBuffer::zeroed(stream, cap * n)?, }) @@ -217,6 +229,8 @@ pub struct CardExperts { ff: usize, /// The model's width. n: usize, + /// The union of the resident layers' selected down activation forms. + downs: DownForms, /// The one-token step's buffers. step: CardBufs, /// The buffers of a pass of several rows, on a load that runs one. @@ -270,8 +284,13 @@ impl CardExperts { tiers: bool, ) -> Result { let what = family.what; + let arms = accepts::leg_arms(family.program).ok_or(GpuError::state( + what, + "the selected program's card leg arms", + ))?; let mut per = Vec::with_capacity(layers.len()); let mut tier_k = Vec::with_capacity(layers.len()); + let mut downs = DownForms::NONE; for (l, spec) in layers.iter().enumerate() { // A dense block is outside the host run and has no row: no card // experts, which the resident-rows check below holds it to. A @@ -309,7 +328,13 @@ impl CardExperts { per.push(None); continue; } - LegStacks::of(w, layer.names())?.check(n_card, family.shape(n, ff, 1))?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; + stacks.check(n_card, family.shape(n, ff, 1))?; + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + what, + "the selected card down's activation form", + ))?; + downs = downs.with(act); per.push(Some(layer)); } Ok(CardExperts { @@ -317,14 +342,16 @@ impl CardExperts { k: CardKernels { leg: CardLeg::load(gpu)?, places: CardSlotKernels::load(gpu.context())?, + arms, }, layers: per, tier: tier_k, ff, n, - step: CardBufs::new(gpu.stream(), &family, n, ff, 1)?, + downs, + step: CardBufs::new(gpu.stream(), &family, n, ff, 1, downs)?, pass: (rows > 1) - .then(|| CardBufs::new(gpu.stream(), &family, n, ff, rows)) + .then(|| CardBufs::new(gpu.stream(), &family, n, ff, rows, downs)) .transpose()?, }) } @@ -410,17 +437,24 @@ impl CardExperts { pub fn batch(&self, gpu: &Gpu, cap: usize) -> Result { let stream = gpu.stream(); Ok(CardBatch { - bufs: CardBufs::new(stream, &self.family, self.n, self.ff, cap)?, + bufs: CardBufs::new(stream, &self.family, self.n, self.ff, cap, self.downs)?, sel: DeviceBuffer::zeroed(stream, cap * self.family.n_used)?, }) } /// Device bytes of [`CardExperts::batch`]'s buffers for `cap` tokens - /// ([`CardBatch::bytes`]'s formula): the card leg's ([`leg_bytes`]), the + /// ([`CardBatch::bytes`]'s formula): the card leg's, the /// card sum and the sum plus the base, and the places. #[must_use] pub fn batch_bytes(&self, cap: usize) -> usize { - leg_bytes(self.family.shape(self.n, self.ff, cap)) + let leg = kernels::leg_bytes( + cap as u64, + self.family.n_used as u64, + self.n as u64, + self.ff as u64, + self.downs, + ); + usize::try_from(leg).expect("the card leg's bytes fit a usize") + 2 * size_of::() * cap * self.n + size_of::() * cap * self.family.n_used } @@ -550,7 +584,7 @@ impl CardKernels { io: &Io<'_>, sel: &DeviceBuffer, ) -> Result<(), GpuError> { - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer: l, stacks: &stacks, @@ -573,3 +607,325 @@ impl CardKernels { .enqueue_add(stream, &bufs.acc, io.x1, io.m * n, &mut bufs.pre) } } + +#[cfg(test)] +mod tests { + use super::*; + use gguf::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + use model::arch::models::{self, Mixer, Moe, Residual, Router, RouterNorm, Score}; + + use crate::card_leg::tests::stack_bytes; + use crate::host::leg_tier::LegTier; + use crate::host::tier::{TierBlock, TierExperts, TierInput, TierIo, TierSet}; + use crate::hybrid::HOST; + + const N: usize = 256; + const FF: usize = 256; + const USED: usize = 8; + const EXPERTS: usize = 16; + const CARD: usize = 4; + const CAP: usize = 3; + const FAMILY: CardFamily = CardFamily { + what: "mixed card wrapper", + n_expert: EXPERTS, + n_used: USED, + stacks: model::arch::names::routed_stacks, + program: Program::Qwen35Body, + }; + + fn layers(n: usize) -> Vec { + vec![ + LayerSpec { + mixer: Mixer::None, + ffn: Ffn::Moe(Moe { + experts: EXPERTS as u32, + top_k: USED as u32, + expert_ff: FF as u32, + latent: None, + act: models::Act::SwiGlu { limit: None }, + router: Router { + score: Score::Sigmoid, + bias: false, + norm: RouterNorm::SelectedSum, + scale: 1.0, + hash: false, + }, + shared: None, + }), + residual: Residual::Plain, + extras: Vec::new(), + }; + n + ] + } + + fn weights(gpu: &Gpu, triples: &[[GgmlType; 3]]) -> Weights { + let mut decls = Vec::new(); + let mut data = Vec::new(); + for (l, triple) in triples.iter().enumerate() { + for (part, (name, &ty)) in (FAMILY.stacks)(l).into_iter().zip(triple).enumerate() { + let (rows, k) = if part == 2 { (N, FF) } else { (FF, N) }; + let bytes = stack_bytes(ty, CARD * rows, k, 7 + (3 * l + part) as u64); + decls.push(TensorDecl { + name, + dims: vec![k as u64, rows as u64, CARD as u64], + type_id: ty.as_u32(), + nbytes: bytes.len() as u64, + }); + data.push(bytes); + } + } + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let call = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "bloomery-mixed-wrappers-{}-{call}.gguf", + std::process::id() + )); + let layout = Layout::new(&[], decls.clone()).expect("a layout"); + let file = std::io::BufWriter::new(std::fs::File::create(&path).expect("a file")); + let mut writer = Writer::new(file, layout).expect("a writer"); + for (decl, bytes) in decls.iter().zip(data) { + writer.tensor(&decl.name, &bytes).expect("a tensor"); + } + writer.finish().expect("a finished file"); + let file = gguf::Split::open(&path).expect("the file"); + let weights = Weights::load_where(gpu.stream(), &file, |_| true).expect("resident stacks"); + std::fs::remove_file(path).expect("remove the fixture"); + weights + } + + fn assert_bits(got: &[f32], want: &[f32]) { + assert_eq!(got.len(), want.len()); + for (i, (&g, &w)) in got.iter().zip(want).enumerate() { + assert!(g.is_finite() && w.is_finite(), "lane {i}: {g} vs {w}"); + assert_eq!(g.to_bits(), w.to_bits(), "lane {i}: {g} vs {w}"); + } + } + + /// Each real wrapper uses its resident layers' union, not a model-wide + /// quant: both one-form loads and mixed loads conserve bytes and execute + /// step, pass, batch and tier chunks as the same common leg. Host slots + /// stay untouched on the tier, including a chunk's short final run. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_mixed_card_and_tier_wrappers_conserve_forms_and_rows() { + use GgmlType::{IQ3_S, IQ4_XS, Q5_K, Q6_K, Q8_0}; + let gpu = Gpu::with_device(0).expect("a Gpu on device 0"); + let stream = gpu.stream(); + let leg = CardLeg::load(&gpu).expect("the reference leg"); + for triples in [ + vec![[IQ3_S, IQ3_S, Q5_K], [IQ4_XS; 3]], + vec![[Q5_K, Q5_K, Q8_0]], + vec![[Q5_K, Q5_K, Q6_K]], + vec![ + [IQ3_S, IQ3_S, Q5_K], + [Q5_K, Q5_K, Q8_0], + [Q5_K, Q5_K, Q6_K], + [IQ4_XS; 3], + ], + ] { + let w = weights(&gpu, &triples); + let specs = layers(triples.len()); + let map = SlotMap::prefix(0..triples.len(), EXPERTS, CARD).expect("a map"); + let mut card = CardExperts::new(&gpu, &w, FAMILY, &specs, &map, (N, FF), CAP) + .expect("the card wrapper"); + let mut batch = card.batch(&gpu, CAP).expect("the batch"); + assert_eq!(card.batch_bytes(CAP), batch.bytes()); + let forms = triples.iter().fold(DownForms::NONE, |forms, tys| { + forms.with(kernels::down(tys[2]).expect("a down").1) + }); + let counted = |cap| { + kernels::leg_bytes(cap as u64, USED as u64, N as u64, FF as u64, forms) as usize + }; + assert_eq!( + card.bytes(), + counted(1) + counted(CAP) + 2 * size_of::() * (1 + CAP) * N, + ); + assert_eq!( + batch.bytes(), + counted(CAP) + 2 * size_of::() * CAP * N + size_of::() * CAP * USED, + ); + let shape = FAMILY.shape(N, FF, 2); + let set = TierSet::new( + 0..triples.len(), + EXPERTS, + vec![(0..CARD as u32).collect(); triples.len()], + ) + .expect("the tier set"); + let mut tier = LegTier::new(&gpu, &specs, &set, &w, shape, FAMILY.program) + .expect("the tier wrapper"); + assert_eq!(tier.block_bytes(), counted(shape.cap)); + let host_x: Vec = (0..CAP * N).map(|i| (i % 71) as f32 / 71.0 - 0.5).collect(); + let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); + let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); + let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); + let route = + DeviceBuffer::from_host(stream, &vec![0.125f32; CAP * USED]).expect("weights"); + let host_sel: Vec = (0..CAP * USED) + .map(|i| { + if i % USED == USED - 1 { + HOST + } else { + (i % CARD) as u32 + } + }) + .collect(); + let sel = DeviceBuffer::from_host(stream, &host_sel).expect("the places"); + let ids = DeviceBuffer::from_host( + stream, + &host_sel + .iter() + .map(|&s| if s == HOST { (EXPERTS - 1) as u32 } else { s }) + .collect::>(), + ) + .expect("the ids"); + let places = DeviceTensor::upload(stream, &map.stage_view(), triples.len(), EXPERTS) + .expect("the map's card view"); + let act = crate::Q8Act::with_tier_cols(stream, CAP, N).expect("the tier block's form"); + for (l, tys) in triples.iter().enumerate() { + let names = (FAMILY.stacks)(l); + let stacks = LegStacks::of_arms( + &w, + names.each_ref().map(String::as_str), + accepts::leg_arms(FAMILY.program).expect("the selected arms"), + STACK_ROLES, + ) + .expect("the reference stacks"); + let mut reference = LegBufs::for_down( + stream, + FAMILY.shape(N, FF, CAP), + kernels::down(tys[2]).expect("a down").1, + ) + .expect("the reference buffers"); + let mut acc = DeviceBuffer::zeroed(stream, CAP * N).expect("the reference sum"); + for m in [1, CAP] { + let io = Io { + normed: &x, + weights: &route, + x1: &x1, + m, + }; + let rule = Act::SwigluClamp { limit: 0.0 }; + leg.enqueue( + &gpu, + &mut reference, + &LegRun { + layer: l, + stacks: &stacks, + normed: &x, + m, + sel: &sel, + n_card: CARD, + rule, + }, + Some(LegSum { + w: &route, + acc: &mut acc, + }), + ) + .expect("the reference"); + let sums = acc.to_host_vec(stream).expect("the reference sums"); + let want: Vec = sums[..m * N] + .iter() + .zip(&base) + .map(|(&a, &b)| a + b) + .collect(); + let got = if m == 1 { + card.enqueue_step(&gpu, &w, l, rule, &io, &sel) + .expect("step"); + card.pre() + } else { + card.enqueue_pass(&gpu, &w, l, rule, &io, &sel) + .expect("pass"); + card.pass_pre().expect("pass output") + }; + assert_bits(&got.to_host_vec(stream).expect("output")[..m * N], &want); + card.enqueue_rows( + &gpu, + &w, + &mut batch, + l, + rule, + &io, + (&ids, &places, l * EXPERTS), + ) + .expect("batch"); + assert_bits( + &batch.pre().to_host_vec(stream).expect("batch output")[..m * N], + &want, + ); + let mut down = + DeviceBuffer::from_host(stream, &vec![1.0e30f32; CAP * USED * N]) + .expect("the tier output"); + if m == 1 { + tier.enqueue_layer( + &gpu, + &w, + l, + TierIo { + act: TierInput::F32(&x), + sel: &sel, + rows: &mut down, + cols: m, + }, + ) + .expect("tier step"); + } else { + tier.enqueue_block( + &gpu, + &w, + l, + TierBlock { + x: &x, + act: &act, + sel: &sel, + cols: m, + down: &mut down, + }, + ) + .expect("tier chunks"); + } + let got = down.to_host_vec(stream).expect("tier output"); + let want = reference + .down() + .to_host_vec(stream) + .expect("reference rows"); + for (s, &place) in host_sel[..m * USED].iter().enumerate() { + if place == HOST { + assert!(got[s * N..(s + 1) * N].iter().all(|&v| v == 1.0e30)); + } else { + assert_bits(&got[s * N..(s + 1) * N], &want[s * N..(s + 1) * N]); + } + } + } + } + assert!(gpu.take_fault().expect("the fault word").is_none()); + } + for (triple, program) in [ + ([IQ3_S, Q5_K, Q8_0], Program::Qwen35Body), + ([Q5_K, Q5_K, Q6_K], Program::Solar2Body), + ] { + let w = weights(&gpu, &[triple]); + let specs = layers(1); + let map = SlotMap::prefix(0..1, EXPERTS, CARD).expect("a map"); + let family = CardFamily { program, ..FAMILY }; + assert!( + matches!( + CardExperts::new(&gpu, &w, family, &specs, &map, (N, FF), CAP), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + let set = + TierSet::new(0..1, EXPERTS, vec![(0..CARD as u32).collect()]).expect("a tier set"); + assert!( + matches!( + LegTier::new(&gpu, &specs, &set, &w, family.shape(N, FF, 2), program), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + } + } +} diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 54242a724..91c0f270a 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -555,8 +555,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: kernels::card_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: accepts::planes_leg_down, }, (GATE_UP, DOWN), ) @@ -571,8 +571,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: kernels::rank_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: |ty| kernels::rank_down_runs(ty) || kernels::card_planes_runs(ty), }, (GATE_UP, DOWN), ) @@ -589,8 +589,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: accepts::resident_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: |ty| accepts::resident_down_runs(ty) || kernels::card_planes_runs(ty), }, (GATE_UP, DOWN), ) @@ -610,9 +610,9 @@ impl<'a> LegStacks<'a> { ) -> Result, GpuError> { let [gate, up, down] = names; let (gate, up, down) = ( - LegStack::of(w, gate, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, up, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, down, arms.down, kernels::card_planes_runs, roles.1)?, + LegStack::of(w, gate, arms.gate_up, arms.gate_up, roles.0)?, + LegStack::of(w, up, arms.gate_up, arms.gate_up, roles.0)?, + LegStack::of(w, down, arms.down, arms.down, roles.1)?, ); if up.ty != gate.ty { return Err(GpuError::Tensor { diff --git a/crates/gpu/src/host/batch.rs b/crates/gpu/src/host/batch.rs index 5a69a4707..29dc6c687 100644 --- a/crates/gpu/src/host/batch.rs +++ b/crates/gpu/src/host/batch.rs @@ -29,7 +29,7 @@ use crate::graph::{MappedHost, cu}; use crate::tensor::WindowMut; use crate::{Gpu, GpuError, Q8Act}; use cuda_core::{CudaContext, CudaEvent, CudaStream, DeviceBuffer, PinnedHostBuffer, sys}; -use model::placement::Machine; +use model::placement::Plan; use model::placement::workstation::{TIER_BATCH_HOST_RESERVE, TIER_BATCH_RESERVE}; use model::{Tensor2, Tensor2View}; use std::mem::size_of; @@ -1079,18 +1079,19 @@ impl HostTier { Ok(()) } - /// The expert tiers' prompt-batch bytes against the reserves `machine`'s + /// The expert tiers' prompt-batch bytes against the selected reserves the /// plan carries for them: on each tier's card — tier `t`'s is index - /// `cards[t]` of [`Machine::all_cards`] — its batch staging and the + /// `cards[t]` of [`model::placement::Machine::all_cards`] — its batch staging and the /// architecture's block scratch ([`TierCard::block_bytes`]) against that - /// card's [`TIER_BATCH_RESERVE`] row; on the host, the tier legs' rows and + /// card's selected [`TIER_BATCH_RESERVE`] row in `plan`; on the host, the tier legs' rows and /// places against the host's [`TIER_BATCH_HOST_RESERVE`] row. A card list /// that is not one card a tier, a missing row, a row named twice, or a /// difference is refused by name, naming the reserve. Load-time only, once /// [`HostTier::prepare_batch`] has made the legs. Step-only tiers make /// no batch leg ([`HostTier::prepare_tier_batch`]) and none is checked. - pub fn check_tier_reserves(&self, machine: &Machine, cards: &[usize]) -> Result<(), GpuError> { + pub fn check_tier_reserves(&self, plan: &Plan<'_>, cards: &[usize]) -> Result<(), GpuError> { const WHAT: &str = "HostTier::check_tier_reserves"; + let machine = plan.machine; if self.tiers.iter().any(|t| !t.supports_blocks()) { return Ok(()); } @@ -1124,6 +1125,7 @@ impl HostTier { &card.reserves, TIER_BATCH_RESERVE, port.tier_device_bytes(i) + t.block_bytes()?, + Some(index), )); } rows.push(( @@ -1131,15 +1133,27 @@ impl HostTier { &machine.host().reserves, TIER_BATCH_HOST_RESERVE, port.tier_host_bytes(), + None, )); - for (on, reserves, name, got) in rows { + for (on, reserves, name, got, card) in rows { let want = match reserves .iter() .filter(|(n, _)| n == name) .collect::>() .as_slice() { - [(_, b)] => *b, + [(_, b)] => match card { + Some(c) => plan + .card_reserve(c, name) + .map_err(|e| GpuError::plan(WHAT, e))? + .ok_or_else(|| { + GpuError::shape( + WHAT, + format!("the plan's selected reserve {name:?} on {on} is invalid"), + ) + })?, + None => *b, + }, named => { return Err(GpuError::shape( WHAT, diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 5620547a4..a77459fa0 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -8,8 +8,8 @@ //! //! The tier's layer is the card leg ([`CardLeg::enqueue_rows`]) over the //! tier's stacks: the q8_1 of the activation, the gate·up `_sel` with the -//! routed clamped SwiGLU, the q8_1 of the tier slots' columns and the down -//! `_sel` into the tier's rows; over a prompt batch's block, the same run +//! routed clamped SwiGLU, the selected down form of the tier slots' columns +//! and the down `_sel` into the tier's rows; over a prompt batch's block, the same run //! chunk by chunk of `shape.cap` tokens. Each slot's down output is the one //! the stage card would write for it, so the card sum over both cards' slots //! in slot order is the one-card sum over their union. The row's tier image @@ -26,6 +26,8 @@ use model::ModelError; use model::arch::models::{Ffn, LayerSpec}; use model::arch::names::routed_stacks; use model::moe::HostLayer; +use model::placement::accepts::{self, LegArms, Program}; +use model::placement::kernels::{self, DownForms}; use model::placement::{ModelTensors, Plan}; use model::r8file::R8Source; @@ -46,6 +48,12 @@ const WHAT: &str = "LegTier"; /// What the stage card's side of the tier layers names in its errors. const SIDE: &str = "TierSide"; +/// The selected program's launches admit the same stacks on both cards. +const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected tier gate·up arm runs", + "a resident routed down stack the selected tier down arm runs", +); + /// The tier the stage card's tier entries serve: they hand one tier its /// image and join one tier's rows, so a load holds one tier card /// ([`crate::host::refuse_tier_count`]). @@ -173,6 +181,7 @@ pub struct LegTier { /// Per layer of `layers`, `None` off the tier. cfg: Vec>, shape: LegShape, + arms: LegArms, } impl LegTier { @@ -188,7 +197,13 @@ impl LegTier { set: &TierSet, w: &Weights, shape: LegShape, + program: Program, ) -> Result { + let arms = accepts::leg_arms(program).ok_or(GpuError::state( + WHAT, + "the selected program's card leg arms", + ))?; + let mut downs = DownForms::NONE; let run = set.layers(); let mut cfg = Vec::with_capacity(run.len()); for l in run.clone() { @@ -216,15 +231,22 @@ impl LegTier { limit: m.act.swiglu_limit(), names, }; - LegStacks::of(w, layer.names())?.check(k, shape)?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; + stacks.check(k, shape)?; + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + WHAT, + "the selected tier down's activation form", + ))?; + downs = downs.with(act); cfg.push(Some(layer)); } Ok(LegTier { leg: CardLeg::load(gpu)?, - bufs: LegBufs::new(gpu.stream(), shape)?, + bufs: LegBufs::for_downs(gpu.stream(), shape, downs)?, layers: run, cfg, shape, + arms, }) } } @@ -270,6 +292,7 @@ impl TierExperts for LegTier { layers, cfg, shape, + arms, } = self; let LegShape { cap, @@ -278,7 +301,7 @@ impl TierExperts for LegTier { .. } = *shape; let tl = layer_of(layers, cfg, layer, W)?; - let stacks = LegStacks::of(weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), *arms, STACK_ROLES)?; for c0 in (0..io.cols).step_by(cap) { let cn = cap.min(io.cols - c0); let slots = cn * slots_per_col; @@ -321,7 +344,7 @@ impl TierExperts for LegTier { .to_string(), }); }; - let stacks = LegStacks::of(weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer, stacks: &stacks, @@ -432,6 +455,8 @@ pub struct LegWidths { pub n_embd: usize, /// A routed expert's width. pub ff: usize, + /// The program whose card leg runs these experts. + pub program: Program, } impl LegWidths { @@ -523,13 +548,15 @@ fn node_experts( open: &NodeOpen, ) -> Result, GpuError> { let inputs = node_inputs::(file, open)?; - let shape = F::widths(&inputs).shape(F::CHUNK); + let widths = F::widths(&inputs); + let shape = widths.shape(F::CHUNK); Ok(Box::new(LegTier::new( gpu, F::layers(&inputs), &open.set, w, shape, + widths.program, )?)) } @@ -603,5 +630,5 @@ pub fn hang_tiers( let planned: Vec = hang.tiers.iter().map(|t| t.card).collect(); hybrid.attach_tiers(cards, gpu)?; hybrid.prepare_batch(gpu.context(), hang.cols)?; - hybrid.check_tier_reserves(plan.machine, &planned) + hybrid.check_tier_reserves(plan, &planned) } diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index ed5fb0bc4..b15500b2f 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -320,13 +320,13 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let reserve = plan - .machine - .card(self.card) - .ok_or_else(|| { - GpuError::shape(WHAT, "the projected tier card is absent from the plan") - })? - .set_aside_bytes() + let totals = plan.cards.get(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + let reserve = totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, diff --git a/crates/model/src/arch/deepseek41/kv.rs b/crates/model/src/arch/deepseek41/kv.rs index 97506d933..c6eb5d744 100644 --- a/crates/model/src/arch/deepseek41/kv.rs +++ b/crates/model/src/arch/deepseek41/kv.rs @@ -262,6 +262,7 @@ mod tests { let machine = Machine { cards: vec![Card { name: "pool".into(), + tier_leg: None, device: None, usable_bytes: pool, context_bytes: 0, diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 477567fbd..20ef61907 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -137,15 +137,6 @@ fn card_leg_forms(model: &ModelTensors) -> DownForms { .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)) } -/// The first routed layer of `model` the card experts run that the expert -/// tier card's leg does not, with its stacks' types: that leg reads the -/// common leg's types in every part ([`kernels::card_runs`]: the file's blocks -/// over the 128-value form, the one its buffers and the plans' reserve hold), -/// so a plan that hangs a tier card is refused it ([`PlaceError::TierLeg`]). -fn off_tier_layer(model: &ModelTensors) -> Option<(usize, [GgmlType; 3])> { - carded_layers(model).find(|(_, tys)| !tys.iter().all(|&ty| kernels::card_runs(ty))) -} - /// Tokens a chunk of the tier's prompt-batch service runs at most: the /// stage card's chunk of the card experts' launches, which the body binds /// to its own constant. @@ -156,14 +147,15 @@ pub const TIER_CHUNK: u64 = 8; /// path's scratch ([`workstation::tier_chunk_scratch_bytes`]) and the host's /// rows and places, which the plan reserves on the tier card and the host /// and the load checks its allocations against. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + crate::placement::accepts::leg_down_forms(Program::Glm5nextBody, &inputs.model), ) } @@ -419,16 +411,6 @@ pub enum PlaceError { /// A plan of no resident sequence: a load serves at least one. #[error("a plan of {slots} resident sequences: a load serves at least one")] Slots { slots: usize }, - /// A machine that hangs an expert tier card beside a file with a card - /// layer the tier's leg does not run (NVFP4's planes, a down over the - /// 32-value form): the tier card's buffers and reserve hold the file's - /// blocks over the 128-value form alone. - #[error( - "layer {layer}: routed stacks {tys:?} run on the stage card's leg, and the expert tier \ - card's leg reads the file's blocks over 128-value q8_1 columns alone; plan it without \ - the tier card" - )] - TierLeg { layer: usize, tys: [GgmlType; 3] }, } impl PlanRefusal for PlaceError { @@ -654,7 +636,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(lanes); let terms = self.seq_terms(&kv, None); let n = slots as u64; @@ -700,18 +681,6 @@ impl PlanInputs { } Ok(plan) } - - /// Refused by name ([`PlaceError::TierLeg`]) when `machine` hangs an - /// expert tier card and a card layer of the file is one the tier's leg - /// does not run. - fn refuse_off_tier(&self, machine: &Machine) -> Result<(), PlaceError> { - match off_tier_layer(&self.model) { - Some((layer, tys)) if !machine.tiers.is_empty() => { - Err(PlaceError::TierLeg { layer, tys }) - } - _ => Ok(()), - } - } } /// Card bytes the NextN walk's arena may hold beside the plan's tensors @@ -832,6 +801,7 @@ fn nextn_machine() -> Machine { Machine { cards: vec![Card { name: "nextn".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -1015,7 +985,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(KdaLanes::Two); let terms = self.seq_terms(&kv, Some(nextn)); let n = slots as u64; @@ -1591,6 +1560,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 1 << 20, @@ -2155,44 +2125,74 @@ mod tests { } } - /// A machine that hangs an expert tier card is refused, by layer - /// and types, a file with a card layer the tier's leg does not run - /// (an NVFP4 layer: its planes and 32-value form); without the tier - /// card the same file plans, the NVFP4 layer's experts on the stage - /// card, and a K-quant file is not refused with it - /// ([`node_ranges_use_the_common_loaded_assignment`]). Mutants: the - /// refusal dropped, which leaves the tier's load to refuse the - /// layer's stacks; the body's plan rule left on the common leg's, which - /// keeps the NVFP4 layer on the host no kernel there reads. + /// The tier reserves precisely the selected layers' down forms, + /// including a mixed 128/32-value load; a gate/up mismatch remains + /// host-only and cannot add a form. #[test] - fn a_tier_card_is_refused_a_layer_its_leg_does_not_run() { + fn tier_scratch_follows_selected_mixed_layers() { + use super::super::TIER_CHUNK; + use crate::placement::kernels::{self, DownAct, DownForms}; + use GgmlType::{IQ3_S, NVFP4, Q5_K, Q8_0}; let mut inputs = routed(64 << 30); - for t in inputs + inputs .model .tensors - .iter_mut() - .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(1)) - { - t.ty = GgmlType::NVFP4; - t.file_bytes = t.dims.iter().product::() / 64 * 36; - } - let plain = unified(64 << 30); - let mut machine = plain.clone(); - let mut tier = machine.cards[0].clone(); - tier.name = "remote".into(); - tier.head = false; - tier.layers = 0..0; - machine.tiers.push(tier); - match inputs.plan(&machine, CTX, &PlanLevers::default()) { - Err(PlaceError::TierLeg { layer, tys }) => { - assert_eq!((layer, tys), (1, [GgmlType::NVFP4; 3])); + .retain(|t| t.role != Role::RoutedExperts || t.layer.is_some_and(|l| l < 2)); + let set = |inputs: &mut PlanInputs, l, tys: [GgmlType; 3]| { + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match crate::placement::ExpertPart::of_name(&t.name) { + Some(crate::placement::ExpertPart::Gate) => 0, + Some(crate::placement::ExpertPart::Up) => 1, + Some(crate::placement::ExpertPart::Down) => 2, + None => unreachable!(), + }; + t.ty = tys[part]; + } + }; + for (layers, forms) in [ + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q5_K]], + DownForms::of(DownAct::Q8Act), + ), + ( + [[Q5_K, Q5_K, Q8_0], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [NVFP4; 3]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [IQ3_S, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act), + ), + ] { + for (l, tys) in layers.into_iter().enumerate() { + set(&mut inputs, l, tys); } - other => panic!("{:?}", other.map(|p| p.n_l.clone())), + let bytes = super::super::tier_batch(&inputs).expect("tier bytes"); + let hp = &inputs.hp; + assert_eq!( + bytes.scratch, + kernels::leg_bytes( + TIER_CHUNK, + hp.n_used as u64, + hp.n_embd as u64, + hp.expert_ff as u64, + forms, + ), + "{layers:?}", + ); } - let p = inputs - .plan(&plain, CTX, &PlanLevers::default()) - .expect("the plan without the tier card"); - assert!(p.n_l[1] > 0, "the NVFP4 layer on the card: {:?}", p.n_l); } #[test] diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index 2358730f1..6bd64a8f1 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1507,6 +1507,7 @@ const fn tier_batch_of(n_embd: u64, n_used: u64, ubatch: u64) -> TierBatchBytes staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_route_scratch_bytes(cols), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -2332,6 +2333,7 @@ fn draft_machine() -> Machine { Machine { cards: vec![Card { name: "mtp draft".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -2802,6 +2804,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, @@ -3567,6 +3570,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 0, @@ -4416,6 +4420,7 @@ mod tests { staging: 496_730_112, scratch: 348_980_248, host: 839_188_480, + leg: None, } ); assert_eq!( @@ -4424,6 +4429,7 @@ mod tests { staging: 62_091_264, scratch: 87_233_816, host: 104_898_560, + leg: None, } ); } diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 2b186aeb2..c345ea2f8 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -13,9 +13,10 @@ use super::hparams::Hparams; use super::{roles, spec}; use crate::arch::chat_of; use crate::arch::place_hybrid; +use crate::placement::accepts::{Program, leg_down_forms}; use crate::placement::kernels::q8_act_bytes; use crate::placement::workstation::TierBatchBytes; -use crate::placement::{PlacementError, workstation}; +use crate::placement::{ModelTensors, PlacementError, workstation}; pub use place_hybrid::{KvLayout, PlaceError}; @@ -43,18 +44,19 @@ pub const GEMM32_STEP: usize = 64; /// the stage card's card experts, which the body binds to its own constant. pub const TIER_CHUNK: u64 = 8; -/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a file of hyperparameters `hp`: +/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a described file: /// the tier card's staging, the leg's chunk scratch and the host's rows and places, for blocks of /// the host union's columns. The plan reserves them on the tier card and the host, and the load /// checks its allocations against them. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + leg_down_forms(Program::Solar2Body, &inputs.model), ) } @@ -128,8 +130,12 @@ const fn acts_bytes(k: usize) -> usize { /// The wide forms are a bound: both forms at each width that allows them, where the load makes /// the ones its sites' types read. The units a prefill group past one adds are made out of the /// card's margin. -#[must_use] -pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 { +pub fn prompt_reserve_bytes( + hp: &Hparams, + spec: &ModelSpec, + model: &ModelTensors, + rows: usize, +) -> Result { let (embd, r) = (hp.n_embd, rows); let (mut heads, mut kv_heads, mut head) = (hp.n_head, 0, hp.head_dim); let (mut k_heads, mut v_heads, mut d) = (0, 0, 0); @@ -163,13 +169,21 @@ pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 let wide: usize = others.iter().map(|&k| wide_bytes(r, k)).sum::() + 2 * wide_bytes(r, embd) + wide_bytes(r, shared_ff); - let card_leg = workstation::tier_chunk_scratch_bytes( - embd as u64, - experts.ff as u64, - experts.n_used as u64, - r as u64, - ) as usize; - (4 * f32s + acts + wide + 2 * route_bytes(r) + card_leg) as u64 + let downs = leg_down_forms(Program::Solar2Body, model); + let card_leg = workstation::TierLegScratch { + n_embd: embd as u64, + ff: experts.ff as u64, + n_used: experts.n_used as u64, + chunk: r as u64, + downs, + } + .bytes(downs)?; + ((4 * f32s + acts + wide + 2 * route_bytes(r)) as u64) + .checked_add(card_leg) + .ok_or_else(|| PlacementError::Metadata { + key: "solar2 prompt reserve".into(), + detail: "bytes overflow u64".into(), + }) } impl PlanInputs { @@ -186,7 +200,7 @@ impl PlanInputs { let model = roles::classify(split, &hp)?; // No parser is declared: see `spec`. let spec = spec::spec_of(&hp, chat_of(split, None, None)?)?; - let reserve = prompt_reserve_bytes(&hp, &spec, PROMPT_ROWS); + let reserve = prompt_reserve_bytes(&hp, &spec, &model, PROMPT_ROWS)?; let tier = tier_places_bytes(&hp, PROMPT_ROWS); Ok(PlanInputs::new(ENGINE, hp, model, spec)? .with_prompt_reserve(reserve) @@ -288,7 +302,7 @@ mod tests { #[test] fn a_tier_card_is_planned_with_the_batch_places_beside_the_reserve() { let inputs = described("so2-place-tier"); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs.hp)); + let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs).expect("tier bytes")); let plain = workstation::plan_gate(LAYERS); let places = 12 * 512 * inputs.hp.experts.n_used as u64; assert_eq!(tier_places_bytes(&inputs.hp, PROMPT_ROWS), places); @@ -326,11 +340,108 @@ mod tests { /// expert. Mutant: the reserve left out of the plan's call (the card plans, and its load stops /// at the allocation). #[test] - fn a_card_short_of_the_prompt_reserve_is_refused_by_name() { - let inputs = described("so2-place-reserve"); - let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, PROMPT_ROWS); - assert!(reserve > 0, "the synthetic header's batch has bytes"); - assert_eq!(inputs.prompt_reserve, reserve, "describe keeps the figure"); + fn a_host_only_mixture_keeps_the_exact_prompt_reserve_boundary() { + use crate::placement::{ExpertPart, Role}; + use gguf::GgmlType::{Q5_K, Q8_0}; + + let mut kv = keys(); + for (key, value) in &mut kv { + if matches!(*key, "embedding_length" | "expert_feed_forward_length") { + *value = V::U32(256); + } + } + let mut ts = tensors(); + for (name, dims) in &mut ts { + let tail = name.rsplit('.').nth(1).unwrap_or(""); + if matches!( + tail, + "ffn_gate_exps" + | "ffn_up_exps" + | "ffn_down_exps" + | "ffn_gate_shexp" + | "ffn_up_shexp" + | "ffn_down_shexp" + ) { + dims[0] = 256; + dims[1] = 256; + } else if tail == "attn_output" { + dims[1] = 256; + } else if matches!( + tail, + "token_embd" + | "output" + | "output_norm" + | "attn_norm" + | "ffn_norm" + | "attn_q" + | "attn_k" + | "attn_v" + | "attn_gate" + | "ssm_f_a" + | "ssm_g_a" + | "ssm_beta" + | "ffn_gate_inp" + ) { + dims[0] = 256; + } + } + let global = [ + ("tokenizer.ggml.pre", V::Str("solar-open")), + ("tokenizer.chat_template", V::Str("{{ messages }}")), + ]; + let path = header_shaped("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); + let split = Split::open(&path).expect("the mixed fixture opens"); + let mut inputs = PlanInputs::describe(&split).expect("valid routed geometry"); + std::fs::remove_file(path).expect("remove the header fixture"); + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.ty = Q5_K; + t.file_bytes = t.dims.iter().product::() / t.ty.blck_size().unwrap() + * t.ty.type_size().unwrap(); + } + let admitted = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("the admitted prompt reserve"); + let admitted_tier = tier_batch(&inputs).expect("the admitted tier reserve"); + let down = inputs + .model + .tensors + .iter_mut() + .find(|t| { + t.role == Role::RoutedExperts + && t.layer == Some(0) + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + }) + .expect("layer zero's down"); + down.ty = Q8_0; + down.file_bytes = down.dims.iter().product::() / down.ty.blck_size().unwrap() + * down.ty.type_size().unwrap(); + let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("prompt reserve"); + assert_eq!( + reserve, admitted, + "the host-only Q8_0 down adds no card form" + ); + assert_eq!( + tier_batch(&inputs).expect("mixed tier reserve"), + admitted_tier + ); + inputs.prompt_reserve = reserve; + let ample = workstation::plan_gate(LAYERS); + let placed = inputs + .plan(&le, 64, &PlanLevers::default()) + .expect("the mixed plan"); + assert_eq!( + placed.n_l[0], 0, + "Q8_0 stays on the host under Solar's rule" + ); + assert!( + placed.n_l[1..].iter().all(|&n| n > 0), + "other layers remain card candidates" + ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); machine.cards[0].usable_bytes = machine.cards[0].margin_bytes; @@ -367,10 +478,6 @@ mod tests { let plan = inputs .plan(&machine, 64, &levers) .expect("the card holds the terms and the reserve"); - assert_eq!( - plan.n_l.iter().sum::(), - 0, - "no room is left for an expert" - ); + assert!(plan.violations().is_empty()); } } diff --git a/crates/model/tests/placement.rs b/crates/model/tests/placement.rs index f3314b3c1..2f5d27b64 100644 --- a/crates/model/tests/placement.rs +++ b/crates/model/tests/placement.rs @@ -1368,6 +1368,7 @@ fn layered_model(layers: usize) -> ModelTensors { fn byte_card(name: &str, usable: u64, layers: Range) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, diff --git a/crates/model/tests/qwen4exp_mtp_reserve.rs b/crates/model/tests/qwen4exp_mtp_reserve.rs index 8b976bbb0..8da38ef7a 100644 --- a/crates/model/tests/qwen4exp_mtp_reserve.rs +++ b/crates/model/tests/qwen4exp_mtp_reserve.rs @@ -295,6 +295,7 @@ fn machine(usable: u64) -> Machine { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: CONTEXT, diff --git a/crates/model/tests/solar_open2_meta.rs b/crates/model/tests/solar_open2_meta.rs index de58c80de..65465b0d7 100644 --- a/crates/model/tests/solar_open2_meta.rs +++ b/crates/model/tests/solar_open2_meta.rs @@ -350,6 +350,7 @@ fn hw_solar_open2_plan_a() { staging: 0, scratch: 0, host: 0, + leg: None, }; let bp = workstation::plan_bp(layers, None, tier_batch); let tiered = inputs diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 1afdc9890..20c293dbc 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -416,6 +416,43 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { leg_arms(p).map(|a| gate == up && (a.gate_up)(gate) && (a.down)(down)) } +/// The union of down activation forms of the complete routed layers that +/// `p`'s placement rule admits, resolved from the file's descriptor at load. +#[must_use] +pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> kernels::DownForms { + use crate::placement::{ExpertPart, Role}; + + let RoutedRule::Leg { gate_up, down } = routed_rule(p) else { + return kernels::DownForms::NONE; + }; + let mut forms = kernels::DownForms::NONE; + for l in 0..model.layers { + let mut parts = [None; 3]; + for t in model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match ExpertPart::of_name(&t.name) { + Some(ExpertPart::Gate) => 0, + Some(ExpertPart::Up) => 1, + Some(ExpertPart::Down) => 2, + None => continue, + }; + parts[part] = Some(t.ty); + } + if let [Some(g), Some(u), Some(d)] = parts + && g == u + && gate_up(g).is_some() + && down(d).is_some() + && let Some((_, act)) = kernels::down(d) + { + forms = forms.with(act); + } + } + forms +} + /// Program `p`'s plan rule for its routed stacks ([`RoutedRule`]): on a card /// leg, each part by the arm its load reads ([`leg_runs`]), so a layer the /// plan gives card experts is one the load runs. The common leg's plan, over diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 8b9d69940..7bd7941c4 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -100,12 +100,30 @@ pub const fn embed(ty: GgmlType) -> Option { /// it holds; a form whose last super-block is padded passes the padded width. #[must_use] pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { - let n_sb = k / 256; - 8 * cols * 64 * n_sb.div_ceil(2) - + 4 * cols * 256 * n_sb.div_ceil(4) - + 4 * cols * 128 * n_sb.div_ceil(2) - + 4 * cols * 8 * n_sb - + 4 * cols * 2 * n_sb + match try_q8_act_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_act_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_act_bytes`] with overflow returned to the placement boundary. +#[must_use] +pub(crate) const fn try_q8_act_bytes(cols: u64, k: u64) -> Option { + let n_sb = (k / 256) as u128; + // A u64 width's per-column count fits u128; its column product may not. + let per_col = 512 * n_sb.div_ceil(2) + + 1024 * n_sb.div_ceil(4) + + 512 * n_sb.div_ceil(2) + + 32 * n_sb + + 8 * n_sb; + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The card bytes of the 32-value q8_1 activation of `cols` columns of `k` @@ -114,8 +132,25 @@ pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { /// and scales, four bytes each, `k_blocks = k / 32`. #[must_use] pub const fn q8_blocks32_bytes(cols: u64, k: u64) -> u64 { - let k_blocks = k / 32; - cols * 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks) + match try_q8_blocks32_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_blocks32_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_blocks32_bytes`] with overflow returned to the placement boundary. +#[must_use] +const fn try_q8_blocks32_bytes(cols: u64, k: u64) -> Option { + let k_blocks = (k / 32) as u128; + let per_col = 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks); + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The down activation forms ([`DownAct`]) a card leg's buffers hold: the @@ -174,30 +209,68 @@ impl DownForms { /// them read it. #[must_use] pub const fn leg_bytes(cap: u64, slots_per_col: u64, n: u64, ff: u64, downs: DownForms) -> u64 { - let slots = cap * slots_per_col; + match try_leg_bytes(cap, slots_per_col, n, ff, downs) { + Some(bytes) => bytes, + None => panic!("leg_bytes: scratch bytes overflow u64"), + } +} + +/// [`leg_bytes`] with overflow returned to the placement boundary. This +/// counts the same whole blocks and padded planes as the infallible sizing. +#[must_use] +pub(crate) const fn try_leg_bytes( + cap: u64, + slots_per_col: u64, + n: u64, + ff: u64, + downs: DownForms, +) -> Option { + let Some(slots) = cap.checked_mul(slots_per_col) else { + return None; + }; + let Some(row_bytes) = slots.checked_mul(4) else { + return None; + }; + let Some(h) = row_bytes.checked_mul(ff) else { + return None; + }; + let Some(down) = row_bytes.checked_mul(n) else { + return None; + }; + let Some(input) = try_q8_act_bytes(cap, n) else { + return None; + }; let q8 = if downs.q8_act { - q8_act_bytes(slots, ff) + let Some(bytes) = try_q8_act_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; let b32 = if downs.blocks32 { - q8_blocks32_bytes(slots, ff) + let Some(bytes) = try_q8_blocks32_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; - q8_act_bytes(cap, n) + 4 * slots * ff + q8 + b32 + 4 * slots * n + let bytes = input as u128 + h as u128 + q8 as u128 + b32 as u128 + down as u128; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// Whether the common card leg (`crates/gpu/src/card_leg.rs`) launches the /// gate·up entry of a routed gate or up stack of type `ty` in the file's /// blocks ([`crate::placement::CardFormat::KQuant`]): the table's [`gate_up`] /// row, for the entries whose launchers read a `Q8Act` of more columns than a -/// run uses. The Q8_0 row has a launcher of that shape and an arm in the leg -/// (`CardLeg::launch`), which this set leaves out: the families that plan by -/// it (GLM-5.3, MiMo, Solar-Open2, MiniMax-M3, Raon) hold `Q8Act` down buffers -/// only, so a Q8_0 layer they planned on the card would fail at its first -/// launch where it runs on the host today; a program whose buffers follow the -/// down's form reads the arm through its own row ([`crate::accepts::prompt_leg_gate_up`]). +/// run uses. The Q8_0 and Q6_K rows have launchers of that shape and arms in the +/// leg, but only programs whose selected gate·up policy admits them read those +/// arms ([`crate::accepts::prompt_leg_gate_up`]). /// A stack in its planes is [`card_planes_runs`]'. #[must_use] pub const fn card_gate_up_runs(ty: GgmlType) -> bool { @@ -323,6 +396,54 @@ mod tests { use super::*; use crate::placement::CardFormat; + /// Plane-padding transitions retain their byte counts, and both sizing + /// contracts share the last representable column and aggregate boundary. + #[test] + fn activation_and_leg_bytes_keep_padding_and_overflow_boundaries() { + for (k, bytes) in [(1024, 3232), (1280, 5320)] { + assert_eq!(try_q8_act_bytes(1, k), Some(bytes)); + assert_eq!(q8_act_bytes(1, k), bytes); + } + for (k, bytes) in [(1024, 1280), (1056, 2312)] { + assert_eq!(try_q8_blocks32_bytes(1, k), Some(bytes)); + assert_eq!(q8_blocks32_bytes(1, k), bytes); + } + let q8 = DownForms::of(DownAct::Q8Act); + let b32 = DownForms::of(DownAct::Q8Blocks32); + for (forms, bytes) in [ + (DownForms::NONE, 4136), + (q8, 6224), + (b32, 5224), + (q8.with(DownAct::Q8Blocks32), 7312), + ] { + assert_eq!(try_leg_bytes(1, 1, 256, 256, forms), Some(bytes)); + assert_eq!(leg_bytes(1, 1, 256, 256, forms), bytes); + } + assert_eq!(try_leg_bytes(1, 1, 64, 24, DownForms::NONE), Some(352)); + let act_cols = u64::MAX / 2088; + assert_eq!(try_q8_act_bytes(act_cols, 256), Some(act_cols * 2088)); + assert_eq!(q8_act_bytes(act_cols, 256), act_cols * 2088); + assert_eq!(try_q8_act_bytes(act_cols + 1, 256), None); + let block_cols = u64::MAX / 1088; + assert_eq!( + try_q8_blocks32_bytes(block_cols, 256), + Some(block_cols * 1088) + ); + assert_eq!(q8_blocks32_bytes(block_cols, 256), block_cols * 1088); + assert_eq!(try_q8_blocks32_bytes(block_cols + 1, 256), None); + let cap = u64::MAX / 6224; + assert_eq!(try_leg_bytes(cap, 1, 256, 256, q8), Some(cap * 6224)); + assert_eq!(leg_bytes(cap, 1, 256, 256, q8), cap * 6224); + assert_eq!(try_leg_bytes(cap + 1, 1, 256, 256, q8), None); + assert_eq!(try_leg_bytes(1 << 61, 8, 256, 256, q8), None); + } + + #[test] + #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] + fn infallible_leg_sizing_names_an_overflow() { + let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); + } + /// Every `GgmlType` variant in declaration order, then one id the enum /// does not name. const ALL: [GgmlType; 42] = [ diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 5a739de5b..ee6acf9ac 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -227,6 +227,8 @@ pub struct Card { /// draft model's resident bytes, say): taken from the budget with the /// context and the scratch ([`Card::set_aside_bytes`]). pub reserves: Vec<(String, u64)>, + /// A tier leg's scratch, jointly selected with its resident experts. + pub tier_leg: Option, } impl Card { @@ -248,10 +250,55 @@ impl Card { .try_fold(0u64, |sum, (_, bytes)| sum.checked_add(*bytes)) } - /// What the card sets aside before any tensor or cache: context, - /// scratch and reserves. The one owner of that sum: the expert rule's - /// budget, the budget floor, the headroom and [`Plan::violations`] all - /// take it from here. The margin is not in it. `None` past u64 bytes. + /// The tier leg's bound and selected scratch; absent on fixed-reserve paths. + fn selected_scratch( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result, PlacementError> { + let Some(leg) = self.tier_leg else { + return Ok(None); + }; + let forms = model + .tensors + .iter() + .filter(|t| { + t.role == Role::RoutedExperts + && t.layer + .is_some_and(|l| counts.get(l).is_some_and(|&n| n > 0)) + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + }) + .filter_map(|t| crate::kernels::down(t.ty)) + .fold(crate::kernels::DownForms::NONE, |forms, (_, act)| { + forms.with(act) + }); + Ok(Some((leg.bytes(leg.downs)?, leg.bytes(forms)?))) + } + + fn selected_reserve_bytes( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result { + let fixed = self.reserve_bytes().ok_or_else(|| self.bytes_past_u64())?; + match self.selected_scratch(model, counts)? { + Some((bound, selected)) => fixed + .checked_sub(bound) + .and_then(|n| n.checked_add(selected)) + .ok_or_else(|| self.bytes_past_u64()), + None => Ok(fixed), + } + } + + fn floor_with_reserves(&self, dense: u64, kv: u64, past: u64, reserves: u64) -> Option { + [kv, self.context_bytes, self.scratch_bytes, reserves, past] + .into_iter() + .try_fold(dense, u64::checked_add) + } + + /// The machine's context, scratch and reserve bound before uploads. + /// Placement replaces a tier leg's bound with its candidate's actual + /// scratch. The margin is not in it. `None` past u64 bytes. #[must_use] pub fn set_aside_bytes(&self) -> Option { self.context_bytes @@ -259,18 +306,13 @@ impl Card { .checked_add(self.reserve_bytes()?) } - /// The card's floor under `dense` bytes of uploads and `kv` bytes of - /// cache: those, its set-aside, and `past` — what the load keeps beside - /// them: a plan's margin and what its caller holds beside the plan - /// (`plan_floor` sums those), or a whole load's own arena and reserve - /// ([`WholeLoad::past_bytes`]). The one owner of that sum: the budget - /// floor, the free floor and the whole fit ([`whole_need`]) all take it - /// from here. `None` past u64 bytes. + /// The fixed/bounded floor under `dense` upload bytes, `kv` cache bytes + /// and `past` bytes held beside them ([`WholeLoad::past_bytes`]). + /// Selected plans use the same sum with their actual reserves. + /// `None` past u64 bytes. #[must_use] pub fn floor_bytes(&self, dense: u64, kv: u64, past: u64) -> Option { - [kv, self.set_aside_bytes()?, past] - .into_iter() - .try_fold(dense, u64::checked_add) + self.floor_with_reserves(dense, kv, past, self.reserve_bytes()?) } fn bytes_past_u64(&self) -> PlacementError { @@ -897,7 +939,8 @@ pub struct CardTotals { pub stage_rows_bytes: u64, pub scratch_bytes: u64, pub context_bytes: u64, - /// The card's reserves ([`Card::reserves`]). + /// The selected reserves; a tier leg's descriptor bound is replaced by + /// its actual resident layers' scratch. pub reserve_bytes: u64, /// usable − dense − experts − rounding − KV − stage rows − scratch − /// context − reserves, usable capped by the plan's card budget; the @@ -2572,6 +2615,40 @@ impl Fill<'_, '_> { .routed .unwrap_or(RoutedRule::PerType(CardFormat::of_routed)); let uploads = card_uploads(self.model, self.c, self.rows, self.eligible, format)?; + let mut down_acts = if card.tier_leg.is_some() { + vec![None; self.model.layers] + } else { + Vec::new() + }; + if card.tier_leg.is_some() { + for t in &self.model.tensors { + if t.role == Role::RoutedExperts + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + && let Some(l) = t.layer + && let Some((_, act)) = crate::kernels::down(t.ty) + { + down_acts[l] = Some(act); + } + } + } + let scratch = |counts: &[u64]| { + card.tier_leg.map_or(Ok(0), |leg| { + let forms = counts + .iter() + .zip(&down_acts) + .filter_map(|(&n, &act)| (n > 0).then_some(act).flatten()) + .fold(crate::kernels::DownForms::NONE, |forms, act| { + forms.with(act) + }); + leg.bytes(forms) + }) + }; + let bound = card + .tier_leg + .map(|leg| leg.bytes(leg.downs)) + .transpose()? + .unwrap_or(0); + let reserves = card.selected_reserve_bytes(self.model, n_l)?; let budget = i128::from(capped(card, self.card_budget)) - i128::from(self.kv) - i128::from( @@ -2579,20 +2656,23 @@ impl Fill<'_, '_> { .ok_or_else(|| card.bytes_past_u64())?, ) - i128::from(card.margin_bytes) - - i128::from(self.reserve); + - i128::from(self.reserve) + + i128::from(bound); if let Some(b) = self.card_budget { let dense = footprint(card.granule_bytes, &uploads, n_l)?; - check_floor(card, b, dense, self.kv, self.reserve)?; + check_floor(card, b, dense, self.kv, self.reserve, reserves)?; } if let Some(free) = card.free_bytes.filter(|&f| f < card.usable_bytes) { let dense = footprint(card.granule_bytes, &uploads, n_l)?; - check_free_floor(card, free, dense, self.kv, self.reserve)?; + check_free_floor(card, free, dense, self.kv, self.reserve, reserves)?; } let caps: Vec = (0..n_l.len()) .map(|l| self.model.layer_experts(l)) .collect(); spread(n_l, held, self.eligible, &caps, budget, |n| { - footprint(card.granule_bytes, &uploads, n) + footprint(card.granule_bytes, &uploads, n)? + .checked_add(scratch(n)?) + .ok_or_else(|| card.bytes_past_u64()) }) } } @@ -2604,10 +2684,10 @@ impl Fill<'_, '_> { /// least budget at which the expert rule's own budget (usable less KV, /// set-aside, margin and `beside`, in [`Fill::run`]) reaches the dense /// uploads. `None` past u64 bytes. -fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64) -> Option { +fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64, reserves: u64) -> Option { card.margin_bytes .checked_add(beside) - .and_then(|past| card.floor_bytes(dense, kv, past)) + .and_then(|past| card.floor_with_reserves(dense, kv, past, reserves)) } /// Refuse `card` under the card budget `budget` when its floor — the @@ -2622,8 +2702,9 @@ fn check_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { - let floor = plan_floor(card, dense, kv, beside); + let floor = plan_floor(card, dense, kv, beside, reserves); match floor { Some(floor) if floor <= budget => Ok(()), _ => Err(PlacementError::CardBudgetFloor { @@ -2634,7 +2715,7 @@ fn check_floor( kv, context: card.context_bytes, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, }), @@ -2653,9 +2734,10 @@ fn check_free_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { let context = card.context_bytes - card.context_in_free(); - let floor = plan_floor(card, dense, kv, beside).map(|f| f - card.context_in_free()); + let floor = plan_floor(card, dense, kv, beside, reserves).map(|f| f - card.context_in_free()); match floor { Some(floor) if floor <= free => Ok(()), floor => Err(PlacementError::CardFreeFloor(Box::new(CardFreeFloor { @@ -2668,7 +2750,7 @@ fn check_free_floor( kv, context, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, held: card @@ -3751,8 +3833,12 @@ fn totals<'a>( .map(|(c, (card, heap))| { let rounding = heap.taken - (card_dense[c] + card_experts[c]); let (kv, _, stage_rows) = kv_bytes[c]; + let counts = c + .checked_sub(machine.cards.len()) + .map_or(n_l.as_slice(), |t| tier_n_l[t].as_slice()); + let reserve_bytes = card.selected_reserve_bytes(model, counts)?; let used = card - .floor_bytes(heap.taken, kv, stage_rows) + .floor_with_reserves(heap.taken, kv, stage_rows, reserve_bytes) .ok_or_else(|| card.bytes_past_u64())?; if machine.unified().is_some() && machine.on_this_node(c) @@ -3772,9 +3858,7 @@ fn totals<'a>( stage_rows_bytes: stage_rows, scratch_bytes: card.scratch_bytes, context_bytes: card.context_bytes, - reserve_bytes: card - .reserve_bytes() - .expect("the card floor checked its reserves"), + reserve_bytes, headroom_bytes: i128::from(capped(card, card_budget)) - i128::from(used), }) }) @@ -3843,11 +3927,50 @@ pub struct PoolNeed { /// `card`'s total against usable − margin: its uploads' granules, its KV, /// its stage rows and its set-aside. fn card_total(card: &Card, totals: &CardTotals, heap: &Heap) -> u64 { - card.floor_bytes(heap.taken, totals.kv_bytes, totals.stage_rows_bytes) - .expect("the plan's card totals fit u64") + card.floor_with_reserves( + heap.taken, + totals.kv_bytes, + totals.stage_rows_bytes, + totals.reserve_bytes, + ) + .expect("the plan's card totals fit u64") } impl Plan<'_> { + /// One named reserve at its selected amount, not the machine's scratch + /// bound. Missing or duplicate rows and an invalid card return `None`; + /// invalid scratch descriptors propagate their named error. + pub fn card_reserve(&self, c: usize, name: &str) -> Result, PlacementError> { + let Some(card) = self.machine.card(c) else { + return Ok(None); + }; + let mut rows = card.reserves.iter().filter(|(n, _)| n == name); + let Some((_, bytes)) = rows.next() else { + return Ok(None); + }; + if rows.next().is_some() { + return Ok(None); + } + if name != workstation::TIER_BATCH_RESERVE { + return Ok(Some(*bytes)); + } + let counts = match c.checked_sub(self.machine.cards.len()) { + None => self.n_l.as_slice(), + Some(t) => match self.tier_n_l.get(t) { + Some(counts) => counts.as_slice(), + None => return Ok(None), + }, + }; + match card.selected_scratch(self.model, counts)? { + Some((bound, selected)) => bytes + .checked_sub(bound) + .and_then(|bytes| bytes.checked_add(selected)) + .map(Some) + .ok_or_else(|| card.bytes_past_u64()), + None => Ok(Some(*bytes)), + } + } + fn reserve_beside(&mut self, reserve: u64) -> Result<(), PlacementError> { if self.machine.unified().is_some() { let here = (0..self.machine.all_cards().count()) @@ -4469,6 +4592,7 @@ mod tests { fn card(name: &str, layers: Range, head: bool, token_embedding: bool) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: 0, context_bytes: 0, @@ -5574,6 +5698,169 @@ mod tests { assert_eq!(plan.n_l, vec![8]); } + /// Scratch enters each expert candidate's footprint monotonically. + /// A boundary that admits one down form cannot pay for an unused second + /// form, and adding the first expert of that form pays its exact bytes. + #[test] + fn a_tier_jointly_selects_expert_rows_and_down_forms() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use GgmlType::{Q5_K, Q8_0}; + use workstation::{TIER_BATCH_RESERVE, tier_leg_batch_bytes}; + + let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + let batch = tier_leg_batch_bytes(256, 256, 1, 2, 3, both).expect("tier bytes"); + let leg = batch.leg.expect("a tier leg descriptor"); + let rule = routed_rule(Program::Qwen35Body); + for (downs, first) in [ + ([Q5_K, Q8_0], DownAct::Q8Act), + ([Q8_0, Q5_K], DownAct::Q8Blocks32), + ] { + let mut model = legged(&[[Q5_K, Q5_K, downs[0]], [Q5_K, Q5_K, downs[1]]]); + model.experts = 1; + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.dims[2] = 1; + t.file_bytes *= 8; + } + let expert = |l| { + model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + .map(|t| t.file_bytes / model.experts) + .sum::() + }; + let one = + batch.staging + leg.bytes(DownForms::of(first)).expect("first scratch") + expert(0); + let two = + batch.staging + leg.bytes(both).expect("mixed scratch") + expert(0) + expert(1); + for (room, counts, forms) in [ + (one - 1, [0, 0], DownForms::NONE), + (one, [1, 0], DownForms::of(first)), + (two - 1, [1, 0], DownForms::of(first)), + (two, [1, 1], both), + ] { + let mut tier = bytes_card("tier", room, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = batch.leg; + let machine = Machine { + cards: vec![bytes_card("stage", HEAD, 0..2)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + }; + let plan = plan_routed(&model, &machine, 1, &NoKv, &PlanLevers::default(), rule) + .expect("the joint plan"); + assert_eq!(plan.n_l, [0, 0]); + assert_eq!(plan.tier_n_l[0], counts, "{downs:?}, {room}"); + let selected = batch.staging + leg.bytes(forms).expect("selected scratch"); + assert_eq!( + plan.card_reserve(1, TIER_BATCH_RESERVE) + .expect("selected reserve"), + Some(selected) + ); + assert_eq!(plan.cards[1].reserve_bytes, selected); + let rows = expert(0) * counts[0] + expert(1) * counts[1]; + assert_eq!(plan.cards[1].bytes(), selected + rows); + assert!(plan.violations().is_empty(), "{downs:?}, {room}"); + } + } + } + + /// A malformed descriptor must refuse a plan and a later selected-reserve + /// read, not wrap to a small scratch or masquerade as a fixed-reserve card. + #[test] + fn a_tier_refuses_invalid_scratch_descriptors_before_placement() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use workstation::{TIER_BATCH_RESERVE, TierLegScratch, tier_leg_batch_bytes}; + + let forms = DownForms::of(DownAct::Q8Act); + let batch = tier_leg_batch_bytes(256, 256, 1, 1, 1, forms).expect("valid batch"); + let good = batch.leg.expect("a leg descriptor"); + let mut model = legged(&[[GgmlType::Q5_K; 3]]); + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.file_bytes *= 64; + } + let machine = |leg| { + let mut tier = bytes_card("tier", 1 << 30, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = Some(leg); + Machine { + cards: vec![bytes_card("stage", HEAD, 0..1)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + } + }; + let valid = machine(good); + let rule = routed_rule(Program::Qwen35Body); + let plan = plan_routed(&model, &valid, 1, &NoKv, &PlanLevers::default(), rule) + .expect("valid descriptor plans"); + for leg in [ + TierLegScratch { + chunk: 1 << 61, + n_used: 8, + ..good + }, + TierLegScratch { chunk: 0, ..good }, + TierLegScratch { ff: 0, ..good }, + TierLegScratch { n_embd: 0, ..good }, + ] { + let bad = machine(leg); + assert!( + matches!( + plan_routed(&model, &bad, 1, &NoKv, &PlanLevers::default(), rule), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + let mut stale = plan.clone(); + stale.machine = &bad; + assert!( + matches!( + stale.card_reserve(1, TIER_BATCH_RESERVE), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + assert!( + matches!( + tier_leg_batch_bytes(leg.n_embd, leg.ff, leg.n_used, leg.chunk, 1, leg.downs), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + } + assert!(tier_leg_batch_bytes(256, 256, 8, 1, 1 << 61, forms).is_err()); + let inconsistent = workstation::TierBatchBytes { + scratch: 0, + ..batch + }; + assert!(matches!( + inconsistent.card(), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg batch" + )); + } + // PIN(2026-10-22): the Qwen3.6 leg runs a Q8_0 down and (once the leg has the // arm) a Q6_K gate and up through the common leg, so the layers that held those // types hold types it still has no GEMM for (an MXFP4 down, an IQ2_XS gate and diff --git a/crates/placement/src/placement/nodes_scale.rs b/crates/placement/src/placement/nodes_scale.rs index 67dfcbcb7..20d9c1bbf 100644 --- a/crates/placement/src/placement/nodes_scale.rs +++ b/crates/placement/src/placement/nodes_scale.rs @@ -283,6 +283,7 @@ fn census_budgets_keep_each_node_and_stage_terms() { staging: 123, scratch: 456, host: 789, + leg: None, }, ) .expect("three independent nodes"); @@ -339,6 +340,7 @@ fn cases() -> &'static [Case] { staging: 0, scratch: 0, host: 0, + leg: None, }, ) .expect("two nodes with no batch bytes"); diff --git a/crates/placement/src/placement/qwen38_cards.rs b/crates/placement/src/placement/qwen38_cards.rs index cd219e349..47bb5b9cb 100644 --- a/crates/placement/src/placement/qwen38_cards.rs +++ b/crates/placement/src/placement/qwen38_cards.rs @@ -249,6 +249,7 @@ const CHECKPOINT_38: u64 = 36 * 3_596_288 + 17 * 4 * 2560 * 4; fn stage(spec: CardSpec) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -293,6 +294,7 @@ fn machine_bp(stage_spec: CardSpec, tier: CardSpec) -> Machine { cards: vec![s], tiers: vec![Card { name: tier.name.to_string(), + tier_leg: None, device: tier.device, usable_bytes: tier.usable_bytes(), context_bytes: CONTEXT, diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index ffcae22b4..40419caa4 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -17,7 +17,7 @@ pub use super::devices::{ #[cfg(test)] use super::nodes::{Fabric, Nodes}; use super::{Card, Host, HostTotals, Machine, Node, PlacementError, Plan, UnifiedPool}; -use crate::kernels::{DownAct, DownForms, leg_bytes, q8_act_bytes}; +use crate::kernels::{DownForms, leg_bytes, q8_act_bytes, try_leg_bytes, try_q8_act_bytes}; pub const MIB: u64 = 1 << 20; @@ -164,6 +164,7 @@ pub fn model_v41() -> String { fn card(spec: CardSpec, layers: Range, head: bool) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -192,6 +193,7 @@ pub fn tier_card(spec: CardSpec, draft_bytes: Option, batch: TierBatchBytes .card() .unwrap_or_else(|error| panic!("tier batch card reserve: {error}")), )); + tier.tier_leg = batch.leg; tier } @@ -307,6 +309,7 @@ pub fn plan_a_draft(layers: usize, draft_bytes: Option) -> Result, } impl TierBatchBytes { - /// The tier card's part; refuses a sum past u64. + /// The tier card's part; refuses an invalid leg bound or a sum past u64. pub fn card(&self) -> Result { + if let Some(leg) = self.leg + && leg.bytes(leg.downs)? != self.scratch + { + return Err(PlacementError::Metadata { + key: "tier leg batch".into(), + detail: "scratch reserve differs from the descriptor's bound".into(), + }); + } self.staging .checked_add(self.scratch) .ok_or(PlacementError::TierBatchCard { @@ -371,6 +384,42 @@ impl TierBatchBytes { } } +/// The common leg's scratch geometry and the descriptor's admitted forms. +/// Placement prices its candidate's selected forms jointly with expert rows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TierLegScratch { + pub n_embd: u64, + pub ff: u64, + pub n_used: u64, + pub chunk: u64, + pub downs: DownForms, +} + +impl TierLegScratch { + /// Scratch for exactly these selected forms, refusing invalid geometry + /// and arithmetic overflow before an allocation or placement candidate. + pub fn bytes(self, downs: DownForms) -> Result { + let invalid = || PlacementError::Metadata { + key: "tier leg scratch".into(), + detail: format!("invalid or overflowing geometry: {self:?}, selected {downs:?}"), + }; + if self.chunk == 0 + || self.n_used == 0 + || self.ff == 0 + || self.n_embd == 0 + || [ + crate::kernels::DownAct::Q8Act, + crate::kernels::DownAct::Q8Blocks32, + ] + .into_iter() + .any(|act| downs.holds(act) && !self.downs.holds(act)) + { + return Err(invalid()); + } + try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs).ok_or_else(invalid) + } +} + /// Experts the tile path's bucket table takes a layer at most, and the /// columns a tile holds: the tile path's shapes, which size its scratch. const TILE_BUCKET_EXPERTS: u64 = 1024; @@ -430,12 +479,18 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// The tier's chunk scratch for blocks of rows of `n_embd`, `n_used` slots a /// token, through experts of `ff`, cut in chunks of `chunk` tokens: one run of -/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`, the GLM-5.3-Flash -/// tier) with its slots' columns in the 128-value q8_1 form the tier's downs -/// read ([`leg_bytes`]), in place of the tile path's scratch. +/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`) with its slots' +/// columns in exactly the forms the selected downs read ([`leg_bytes`]), +/// in place of the tile path's scratch. #[must_use] -pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { - leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) +pub const fn tier_chunk_scratch_bytes( + n_embd: u64, + ff: u64, + n_used: u64, + chunk: u64, + downs: DownForms, +) -> u64 { + leg_bytes(chunk, n_used, n_embd, ff, downs) } /// The host's part of the tier's prompt-batch service: per exchange set, the @@ -454,6 +509,7 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_block_scratch_bytes(n_embd, ff, cols * n_used), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -461,19 +517,48 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T /// scratch is one run of the leg's buffers for `chunk` tokens ([`tier_chunk_scratch_bytes`]), in /// place of the tile path's. The one formula of those bytes for every family whose tier is the card /// leg; a family names its own `chunk` and widths. -#[must_use] -pub const fn tier_leg_batch_bytes( +pub fn tier_leg_batch_bytes( n_embd: u64, ff: u64, n_used: u64, chunk: u64, cols: u64, -) -> TierBatchBytes { - TierBatchBytes { - staging: tier_batch_staging_bytes(n_embd, n_used, cols), - scratch: tier_chunk_scratch_bytes(n_embd, ff, n_used, chunk), - host: tier_batch_host_bytes(n_embd, n_used, cols), - } + downs: DownForms, +) -> Result { + let leg = TierLegScratch { + n_embd, + ff, + n_used, + chunk, + downs, + }; + let scratch = leg.bytes(downs)?; + let counted = || -> Option<(u64, u64)> { + if cols == 0 { + return None; + } + let slots = cols.checked_mul(n_used)?; + let places = 4u64.checked_mul(slots)?; + let down = places.checked_mul(n_embd)?; + let staging = 4u64 + .checked_mul(cols)? + .checked_mul(n_embd)? + .checked_add(places)? + .checked_add(try_q8_act_bytes(cols, n_embd)?)? + .checked_add(down)?; + let host = 2u64.checked_mul(down.checked_add(places)?)?; + Some((staging, host)) + }; + let (staging, host) = counted().ok_or_else(|| PlacementError::Metadata { + key: "tier leg batch".into(), + detail: format!("invalid or overflowing staging: {leg:?}, columns {cols}"), + })?; + Ok(TierBatchBytes { + staging, + scratch, + host, + leg: Some(leg), + }) } /// Plan (b′): the A6000 runs all `layers` and the head as in [`plan_a`]; @@ -551,6 +636,7 @@ pub fn plan_node_tiers( remote.cards[0] .reserves .extend(tier_card(spec, None, batch).reserves); + remote.cards[0].tier_leg = batch.leg; machines.push(remote); } Ok(machines) @@ -701,6 +787,7 @@ pub(crate) fn spark_nodes(n: usize, layers: usize, fabric: Fabric) -> Nodes { staging: 0, scratch: 0, host: 0, + leg: None, }; let mut machines = plan_node_tiers(layers, spec, &vec![spec; n - 1], batch) .expect("synthetic nodes with no batch bytes"); @@ -1578,6 +1665,7 @@ mod tests { staging: 1, scratch: 2, host: 3, + leg: None, }, ]; for layers in [0, 1, 20, 43, 47, 48, 61, 78] { @@ -1669,6 +1757,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }; for result in [ plan_tiers(4, A6000, &[RTX_3090], None, huge).map(|_| ()), @@ -1719,6 +1808,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let machines = plan_node_tiers(4, specs[0], &specs[1..], batch) .expect("four nodes of the same card name and ordinal"); @@ -1763,6 +1853,7 @@ mod tests { staging: 0, scratch: 0, host: u64::MAX / 2, + leg: None, }; let remotes = [gb10(80 << 30), gb10(60 << 30)]; let machines = @@ -1798,6 +1889,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let m = plan_tiers(4, A6000, &[t1, t2], None, batch).expect("two tiers"); let rows: Vec<&(String, u64)> = m @@ -1878,6 +1970,7 @@ mod tests { assert_eq!(counted_arena_bytes(&m.cards[0]), 7, "{}", spec.name); let want = Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -1925,6 +2018,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }, ); } @@ -1944,6 +2038,7 @@ mod tests { staging: u64::MAX - 1, scratch: 1, host: 0, + leg: None, }; assert_eq!(edge.card().expect("the sum fits u64"), u64::MAX); assert!(matches!( diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..15def8b18 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -66,6 +66,25 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Selected expert arms and tier scratch + +Common card and tier wrappers resolve `LegArms` from their selected `Program`, rather than assuming +every family has one gate/up type or one down activation form. Their resident layers determine the +union of `Q8Act` and `Q8Blocks32` buffers. Plane layouts remain restricted to the selected program's +arms; Solar's host-only triples do not enlarge its card scratch. + +Placement charges a tier's selected down forms jointly with its expert count. The same kernels-owned, +checked sizing arithmetic is used by allocation and planning; malformed or overflowing public scratch +descriptors are refused by name. The plan records the selected reserve, and loaders check that amount +instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver +is used. + +A host-only public planner consumer exercised the one-byte boundaries before the first resident layer +and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, +not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an +exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. + + ## 1. Sources and observed artifact From 79a2959c302b95e2eb5b67076178e8e31b3c5fad Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:59:23 +0900 Subject: [PATCH 04/85] gpu: name the new six-slot tier primitive generically --- crates/gpu/src/host/handoff.rs | 12 ++++++------ docs/research/flint-attach.md | 6 +++++- tools/unsafe-ratchet.txt | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 99f6b4efe..f2a362b91 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,7 +18,7 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_6_tier` and +//! `crate::card_slots`' `ds41_ffn_places`. `ffn_handoff_6_tier` and //! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an //! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); @@ -310,7 +310,7 @@ mod handoff_kernels { tsel.len() >= 6 ) )] - pub fn ds41_ffn_handoff_6_tier( + pub fn ffn_handoff_6_tier( ids_in: &[u32], w_in: &[f32], map: &[u32], @@ -1264,7 +1264,7 @@ impl HandoffKernels { impl HandoffKernels { /// Enqueue the handoff of a tier layer of six or eight slots a token - /// (`ds41_ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// (`ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and @@ -1288,7 +1288,7 @@ impl HandoffKernels { let (lay, tl) = (target.layout, ttarget.layout); let slots = check_tier(&lay, &tl)?; let what = match slots { - HandoffSlots::Six => "ds41_ffn_handoff_6_tier", + HandoffSlots::Six => "ffn_handoff_6_tier", _ => "ds41_ffn_handoff_8_tier", }; let n = lay.hidden; @@ -1307,8 +1307,8 @@ impl HandoffKernels { ]; match slots { HandoffSlots::Six => { - let prep = self.module.prepare_ds41_ffn_handoff_6_tier(cfg)?; - self.module.ds41_ffn_handoff_6_tier( + let prep = self.module.prepare_ffn_handoff_6_tier(cfg)?; + self.module.ffn_handoff_6_tier( stream, &prep, h.ids, diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index e3bfeb200..a42a0fc4f 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -83,11 +83,15 @@ weights. Its host sum is precomputed and activation images are checked by readback: it is not expert execution or two-card completion/publication proof. The hardware case has not been run here. -The new `ds41_ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +The new `ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and owning-gate evidence; the eight-slot and other existing entries must remain unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. Actual six-slot body/tier and two-card completion remain integration work, not capabilities inferred from this primitive. +`check-ptx-names` remains red until observed spill/JIT-local measurements can pin +the new entry in the four existing architecture/capability tables. Offline +no-JIT scans cannot supply that missing column, so no placeholder row is added. + diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 902fcf81c..cb8465ee7 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -529,7 +529,7 @@ gguf 7 2 0 # body (`mxfp8_cols_body`: its load and its two plane stores) and the two kernel entries' body calls and shared-memory # view. Each block has its SAFETY line. # PIN(2026-10-24): gpu 2331 -> 2333 blocks, a new six-slot F32 tier entry, not old code relaxed: -# ds41_ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; +# ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; # both blocks state their per-thread bounds and disjoint-write SAFETY invariant. gpu 2333 306 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: From a6beeace9c35aaad494ccb5669045c0eebf6de5c Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:12:33 +0900 Subject: [PATCH 05/85] placement: test overflow refusal without pinning panic wording --- crates/placement/src/kernels.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 7bd7941c4..a7140ba70 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -439,8 +439,8 @@ mod tests { } #[test] - #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] - fn infallible_leg_sizing_names_an_overflow() { + #[should_panic] + fn infallible_leg_sizing_refuses_overflow() { let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); } From 2824f4fea029fa7c08f9c03b7747ac013e5ac86f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:24:24 +0900 Subject: [PATCH 06/85] gpu: preserve fixed reserves for host-only node projections --- crates/gpu/src/host/node.rs | 21 +++++++++++++++------ docs/research/flint-attach.md | 3 +++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index b15500b2f..a29006c95 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -320,14 +320,23 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let totals = plan.cards.get(self.card).ok_or_else(|| { + let card = plan.machine.card(self.card).ok_or_else(|| { GpuError::shape(WHAT, "the projected tier card is absent from the plan") })?; - let reserve = totals - .context_bytes - .checked_add(totals.scratch_bytes) - .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) - .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; + // Host-only projections have no card totals; their fixed reserves + // still belong to the machine. Dynamic scratch needs selected totals. + let reserve = if card.tier_leg.is_some() { + let totals = plan.cards.get(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) + } else { + card.set_aside_bytes() + } + .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, &plan.rows, diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 15def8b18..40ac1d498 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -78,6 +78,9 @@ checked sizing arithmetic is used by allocation and planning; malformed or overf descriptors are refused by name. The plan records the selected reserve, and loaders check that amount instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver is used. +Host-only projections intentionally have no card totals: fixed reserves remain owned by their machine. +Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging +budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, From 2f2a8875161ccfed3f33680f794d736445099fc1 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:37:10 +0900 Subject: [PATCH 07/85] gpu: use fixed-size rows in the IQ4 consumer gate --- crates/gpu-deepseek41/src/shared_routed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index 9df96d40b..0c2b6063f 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1243,7 +1243,7 @@ mod tests { let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; let eps = f64::from(f32::EPSILON); let gamma = operations * eps / (1.0 - operations * eps); - for (c, row) in rows.chunks_exact(N).enumerate() { + for (c, row) in rows.as_chunks::().0.iter().enumerate() { let x = f64::from(input[c * N]); for (r, &got) in row.iter().enumerate() { let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; From b3a238017ba9c7578e8a8d41dbe0dd582aee880c Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:37:10 +0900 Subject: [PATCH 08/85] gpu: remove redundant wrapper test allocation and borrow --- crates/gpu-gates/src/bin/gate_solar2_twocard.rs | 2 +- crates/gpu/src/card_experts.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index 9d413942a..24c1daab0 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs)?))?; + .machine(None, Some(place::tier_batch(inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 4542fe4ac..8ed9d38f1 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -760,8 +760,7 @@ mod tests { let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); - let route = - DeviceBuffer::from_host(stream, &vec![0.125f32; CAP * USED]).expect("weights"); + let route = DeviceBuffer::from_host(stream, &[0.125f32; CAP * USED]).expect("weights"); let host_sel: Vec = (0..CAP * USED) .map(|i| { if i % USED == USED - 1 { From d19c5682837dbf7bfa883a2f5de9cda405a490a1 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:32:42 +0900 Subject: [PATCH 09/85] gpu: support ordinary IQ4_XS shared sites --- crates/chain/src/site.rs | 4 + crates/gguf/src/card.rs | 11 +- crates/gguf/src/qtype.rs | 2 +- crates/gpu-deepseek41/src/block.rs | 69 +++--- crates/gpu-deepseek41/src/shared_routed.rs | 185 ++++++++++++++++ crates/gpu-gates/src/act_rule.rs | 45 +--- crates/gpu/src/site.rs | 234 ++++++--------------- crates/placement/src/accepts.rs | 5 +- crates/placement/src/kernels.rs | 121 ----------- docs/research/flint-attach.md | 25 +++ 10 files changed, 333 insertions(+), 368 deletions(-) diff --git a/crates/chain/src/site.rs b/crates/chain/src/site.rs index da5ccefcf..d44ac713e 100644 --- a/crates/chain/src/site.rs +++ b/crates/chain/src/site.rs @@ -38,6 +38,8 @@ pub enum SiteTy { /// entry is none; a backend whose launch reads it names it in the list /// [`file_site`] takes. MlxA4, + /// IQ4_XS in its original blocks, over q8_1 columns, narrow and wide. + Iq4Xs, } impl SiteTy { @@ -56,6 +58,7 @@ impl SiteTy { SiteTy::Mxfp8, SiteTy::Fp8B128, SiteTy::Bf16, + SiteTy::Iq4Xs, ]; /// The family of a file tensor of type `ty`, the one that launches the @@ -99,6 +102,7 @@ impl SiteTy { SiteTy::Mxfp8 => Some(SiteEntry::Mxfp8), SiteTy::Fp8B128 => Some(SiteEntry::Fp8B128), SiteTy::Bf16 => Some(SiteEntry::Bf16), + SiteTy::Iq4Xs => Some(SiteEntry::Iq4xs), SiteTy::MlxA4 => None, } } diff --git a/crates/gguf/src/card.rs b/crates/gguf/src/card.rs index c727e067f..d1575ca56 100644 --- a/crates/gguf/src/card.rs +++ b/crates/gguf/src/card.rs @@ -263,6 +263,9 @@ pub enum SiteEntry { /// `Q8F32Kernels::enqueue_bf16_gemv` over BF16 words and f32 rows; /// wide, `Q8F32Kernels::enqueue_bf16_tile` over the same words and rows. Bf16, + /// `QuantSelKernels::enqueue_gemv_sel` over one matrix in the file's + /// IQ4_XS blocks and `Q8Act` columns; wide, the grouped IQ4_XS GEMM. + Iq4xs, } /// The activation a dense site's launch reads; `crates/gpu/src/site.rs` @@ -289,7 +292,11 @@ impl SiteEntry { #[must_use] pub const fn reads(self) -> SiteAct { match self { - SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128, + SiteEntry::Q3k + | SiteEntry::Q4k + | SiteEntry::Q5k + | SiteEntry::Q6k + | SiteEntry::Iq4xs => SiteAct::Q8x128, SiteEntry::Q8_0 => SiteAct::Q8x32, SiteEntry::F32 => SiteAct::F32, SiteEntry::Q5_0 => SiteAct::Q8Blocks32, @@ -308,7 +315,7 @@ impl SiteEntry { } /// One past the last [`SiteEntry`]: the span [`DENSE_TYPE`] covers. -const DENSE_SPAN: usize = SiteEntry::Bf16 as usize + 1; +const DENSE_SPAN: usize = SiteEntry::Iq4xs as usize + 1; /// Dense entry to the file type of the row that names it. The build refuses /// an entry two rows name, an entry no row names, and an entry past diff --git a/crates/gguf/src/qtype.rs b/crates/gguf/src/qtype.rs index dad5103ed..f579601d2 100644 --- a/crates/gguf/src/qtype.rs +++ b/crates/gguf/src/qtype.rs @@ -115,7 +115,7 @@ pub const QROWS: &[QRow] = &[ card: card!(- - Iq3s (Iq3s, Q8Act) Iq3s - - -) }, QRow { ty: T::IQ2_S, name: "iq2_s", blck: 256, bytes: 82, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::IQ4_XS, name: "iq4_xs", blck: 256, bytes: 136, dequant: true, host_act: H::Q8K, - card: card!(- - Iq4xs (Iq4xs, Q8Act) Iq4xs - - -) }, + card: card!(KQuant - Iq4xs (Iq4xs, Q8Act) Iq4xs Iq4xs - -) }, // The C scalars: ggml.c:621-649. QRow { ty: T::I8, name: "i8", blck: 1, bytes: 1, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::I16, name: "i16", blck: 1, bytes: 2, dequant: false, host_act: H::None, card: Card::NONE }, diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index 5bd972e7c..0a040e739 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -40,12 +40,13 @@ const BLOCK: &str = "gated block"; /// The types a block's matrices read, one launch family each ([`site::file_site`]'s `allowed`); a model's /// coverage pin for the block lists the same set. -pub const BLOCK_TYPES: [SiteTy; 5] = [ +pub const BLOCK_TYPES: [SiteTy; 6] = [ SiteTy::Q8_0, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K, SiteTy::Mxfp8, + SiteTy::Iq4Xs, ]; /// The most columns [`enqueue_block_cols`] takes: the q8_0 block's multi-column launches' cap, kept for every plan. @@ -56,6 +57,11 @@ fn refuse(what: &'static str, detail: String) -> GpuError { GpuError::Shape { what, detail } } +/// A site whose narrow and wide launches both read the 128-value q8_1 form. +fn q128(ty: SiteTy) -> bool { + ty.reads() == site::Form::Q8x128 && site::gemm_weight(ty).is_some() +} + /// A block's gate·up half, picked once at load from the file's types. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GateUp { @@ -115,20 +121,21 @@ impl BlockPlan { !matches!(self.gate_up, GateUp::Split { .. }) } - /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair, two K-quants (the same Q4_K or Q5_K - /// fused, else split) or two types that read the f32 row for gate·up, any of [`BLOCK_TYPES`] for the down, - /// which a split gate·up needs a K-quant one of. Another mix is refused by name as `what`'s. + /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair, + /// two sites over 128-value q8_1 blocks (the same Q4_K or Q5_K fused, + /// else split), or two f32-form sites. A split pair needs a down over + /// the same q8_1 form. /// /// # Errors - /// A gate and up that are not two q8_0, two K-quants or two f32-form types, or a split pair over a down that - /// is not a K-quant. + /// A gate/up pair with incompatible activation forms, or a split pair + /// over a down of another form. pub fn of_types( what: &'static str, [gate, up, down]: [SiteTy; 3], ) -> Result { let gate_up = match (gate, up) { (SiteTy::Q8_0, SiteTy::Q8_0) => GateUp::Ds41, - (g, u) if g.kquant() && u.kquant() => match (g == u, g) { + (g, u) if q128(g) && q128(u) => match (g == u, g) { (true, SiteTy::Q4K | SiteTy::Q5K) => GateUp::Fused(g), _ => GateUp::Split { gate: g, up: u }, }, @@ -139,13 +146,13 @@ impl BlockPlan { return Err(refuse( what, format!( - "gate is {gate} and up is {up}: a block takes two q8_0, two K-quants or two \ - types that read the f32 row" + "gate is {gate} and up is {up}: a block takes two q8_0, two sites over \ + 128-value q8_1, or two types that read the f32 row" ), )); } }; - if matches!(gate_up, GateUp::Split { .. }) && !down.kquant() { + if matches!(gate_up, GateUp::Split { .. }) && !q128(down) { return Err(refuse( what, format!( @@ -182,7 +189,7 @@ impl BlockPlan { /// an f32 `h` quantizes it first (+1; a split block's SwiGLU already wrote it). #[must_use] pub fn extra_launches(self) -> usize { - let down_quantize = usize::from(self.down.kquant()); + let down_quantize = usize::from(q128(self.down)); match self.gate_up { GateUp::Ds41 => down_quantize, GateUp::Fused(_) => 1 + down_quantize, @@ -296,7 +303,7 @@ impl BlockScratch { let (mut any, mut split_ff) = (false, 0); for (plan, ff) in blocks.into_iter().filter(|(p, _)| !p.is_q8()) { any = true; - if plan.down.kquant() && !ha.iter().any(|(f, _)| *f == ff) { + if q128(plan.down) && !ha.iter().any(|(f, _)| *f == ff) { ha.push((ff, Q8Act::with_k(stream, 1, ff)?)); } if matches!(plan.gate_up, GateUp::Split { .. } | GateUp::Site { .. }) { @@ -455,7 +462,7 @@ pub fn enqueue_block_act( gpu.layer_sink(blk.layer)?, h, )?; - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } } @@ -506,7 +513,7 @@ pub fn enqueue_block_act( .enqueue_swiglu_rule(stream, rule, (&*g, &*u), blk.ff, (fault, &mut *h))?; } } - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } let (kk, sc) = kit(k, s)?; @@ -656,16 +663,6 @@ mod tests { BlockPlan::of_types(WHAT, tys) } - fn refused(r: Result, words: &[&str]) { - let msg = match r { - Err(GpuError::Shape { what, detail }) => format!("{what}: {detail}"), - other => panic!("expected a named shape refusal, got {other:?}"), - }; - for w in words { - assert!(msg.contains(w), "{msg:?} lacks {w:?}"); - } - } - /// The plan is the types' dispatch: the q8_0 triple is `BlockPlan::Q8`, one Q4_K or Q5_K for gate and up /// is fused, any other K-quant pair splits, an MXFP8 pair runs the f32-form arm, and every plan reads back /// the types it came from. Mutant: the f32-form arm dropped — the MXFP8 triple is refused. @@ -700,19 +697,19 @@ mod tests { } } - /// A mix no arm covers is refused by name, with the types: gate and up of two kinds, and a split pair over - /// a q8_0 down. #[test] - fn a_mix_no_arm_covers_is_refused() { - use SiteTy::{Mxfp8, Q4K, Q6K, Q8_0}; - refused( - plan([Q8_0, Q4K, Q8_0]), - &["test", "gate is Q8_0", "up is Q4_K"], - ); - refused(plan([Q6K, Q8_0, Q6K]), &["gate is Q6_K", "up is Q8_0"]); - refused(plan([Q6K, Q6K, Q8_0]), &["split", "Q8_0 down"]); - refused(plan([Mxfp8, Q8_0, Mxfp8]), &["gate is MXFP8", "up is Q8_0"]); - refused(plan([Q4K, Mxfp8, Q4K]), &["gate is Q4_K", "up is MXFP8"]); + fn incompatible_activation_forms_refuse_before_launch() { + use SiteTy::{Iq4Xs, Mxfp8, Q4K, Q6K, Q8_0}; + for types in [ + [Q8_0, Q4K, Q8_0], + [Q6K, Q8_0, Q6K], + [Q6K, Q6K, Q8_0], + [Iq4Xs, Iq4Xs, Q8_0], + [Mxfp8, Q8_0, Mxfp8], + [Q4K, Mxfp8, Q4K], + ] { + assert!(matches!(plan(types), Err(GpuError::Shape { .. }))); + } } /// The launches beyond the q8_0 block's two, by plan: 0 for the q8_0 block, and the other arms' derived diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index e71e92788..c5be10e29 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1184,3 +1184,188 @@ fn no_batch() -> GpuError { missing: "the batch's buffers", } } + +#[cfg(test)] +mod tests { + use super::SharedWide; + use crate::block::{ + Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, + enqueue_block_cols_act, + }; + use crate::experts::ExpertKernels; + use bloomery_gpu::Gpu; + use bloomery_gpu::gemm::Gemm32Kernels; + use bloomery_gpu::kquant::Act; + use bloomery_gpu::weights::Weights; + use cuda_core::DeviceBuffer; + use gguf::Split; + use gguf::quant::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + + const N: usize = 256; + const FF: usize = 512; + const NAMES: [&str; 3] = [ + "blk.42.ffn_gate_shexp.weight", + "blk.42.ffn_up_shexp.weight", + "blk.42.ffn_down_shexp.weight", + ]; + + fn iq4_shared_file() -> (std::path::PathBuf, Split) { + let path = + std::env::temp_dir().join(format!("bloomery-iq4-shared-{}.gguf", std::process::id())); + let specs = [(N, FF, 0x1c00u16), (N, FF, 0x1800), (FF, N, 0x1800)]; + let payloads: Vec> = specs + .iter() + .enumerate() + .map(|(i, &(k, rows, scale))| { + let mut bytes = Vec::with_capacity(rows * (k / 256) * 136); + for row in 0..rows { + let d = if i == 2 && row % 2 == 1 { + scale - 0x400 + } else { + scale + }; + let mut block = [0x88u8; 136]; + block[..2].copy_from_slice(&d.to_le_bytes()); + block[2..4].copy_from_slice(&0xaaaau16.to_le_bytes()); + block[4..8].fill(0x11); + for _ in 0..k / 256 { + bytes.extend_from_slice(&block); + } + } + bytes + }) + .collect(); + let decls = NAMES + .iter() + .zip(specs) + .zip(&payloads) + .map(|((&name, (k, rows, _)), bytes)| TensorDecl { + name: name.to_owned(), + dims: vec![u64::try_from(k).unwrap(), u64::try_from(rows).unwrap()], + type_id: GgmlType::IQ4_XS.as_u32(), + nbytes: u64::try_from(bytes.len()).unwrap(), + }) + .collect(); + let layout = Layout::new(&[], decls).unwrap(); + let file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + let mut writer = Writer::new(file, layout).unwrap(); + for (name, bytes) in NAMES.iter().zip(payloads) { + writer.tensor(name, &bytes).unwrap(); + } + writer.finish().unwrap(); + let split = Split::open(&path).unwrap(); + (path, split) + } + + fn check_shared_rows(input: &[f32], rows: &[f32]) { + // Uniform positive rows quantize to code127 without integer rounding. + // The gate is at least128, so SiLU rounds to the gate itself. The + // scalar positive-dot upper bound has four input-width operations, + // two output-width operations and the scale/product roundings. + let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; + let eps = f64::from(f32::EPSILON); + let gamma = operations * eps / (1.0 - operations * eps); + for (c, row) in rows.chunks_exact(N).enumerate() { + let x = f64::from(input[c * N]); + for (r, &got) in row.iter().enumerate() { + let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; + assert!(got.is_finite()); + assert!( + (f64::from(got) - expected).abs() <= gamma * expected, + "column {c} row {r}: {got} vs {expected}, relative bound {gamma}" + ); + } + } + } + + #[test] + #[ignore = "needs a CUDA card"] + fn hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay() { + let (path, split) = iq4_shared_file(); + let gpu = Gpu::new().expect("card"); + let stream = gpu.stream(); + let weights = Weights::load_where(stream, &split, |name| NAMES.contains(&name)).unwrap(); + let plan = BlockPlan::of_file(&split, "IQ4_XS shared", NAMES, (FF, N)).unwrap(); + plan.check_resident(&weights, "IQ4_XS shared", NAMES, (FF, N)) + .unwrap(); + let block = Block { + plan, + names: NAMES, + ff: FF, + limit: 0.0, + layer: 42, + }; + let experts = ExpertKernels::load(gpu.context()).unwrap(); + let quant = KBlockKernels::load(&gpu).unwrap(); + let kernels = BlockKernels { + experts: &experts, + kquant: Some(&quant), + }; + let mut scratch = BlockScratch::new(stream, N, [(plan, FF)]).unwrap(); + let mut h = DeviceBuffer::zeroed(stream, FF).unwrap(); + let g32 = Gemm32Kernels::load(gpu.context()).unwrap(); + for cols in [1, 8, super::GATED_FROM] { + let input: Vec = (0..cols) + .flat_map(|c| std::iter::repeat_n([128.0, 256.0, 384.0][c % 3], N)) + .collect(); + let x = DeviceBuffer::from_host(stream, &input).unwrap(); + let mut y = DeviceBuffer::zeroed(stream, cols * N).unwrap(); + if cols < super::GATED_FROM { + enqueue_block_cols_act( + &gpu, + &weights, + &kernels, + &mut scratch, + (&block, Act::SiluMul), + cols, + BlockIo { + x: &x, + h: &mut h, + y: &mut y, + }, + ) + .unwrap(); + } else { + let types = plan.types(); + let mut wide = + SharedWide::new(&gpu, (&types[..2], &types[2..]), (cols, N, FF)).unwrap(); + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + .unwrap(); + let eager = y.to_host_vec(stream).unwrap(); + check_shared_rows(&input, &eager); + let graph = gpu + .capture(|_| { + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + }) + .unwrap(); + graph.launch(stream).unwrap(); + let replay = y.to_host_vec(stream).unwrap(); + assert!( + eager + .iter() + .zip(&replay) + .all(|(a, b)| a.to_bits() == b.to_bits()) + ); + } + check_shared_rows(&input, &y.to_host_vec(stream).unwrap()); + assert_eq!(gpu.take_fault().unwrap(), None); + } + stream.synchronize().unwrap(); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/gpu-gates/src/act_rule.rs b/crates/gpu-gates/src/act_rule.rs index a959cb768..33596c379 100644 --- a/crates/gpu-gates/src/act_rule.rs +++ b/crates/gpu-gates/src/act_rule.rs @@ -237,8 +237,11 @@ impl SiteRel { #[must_use] pub fn site_rel(ty: GgmlType, arm: Arm) -> Option { let ours = match (ty, arm) { - (GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K, _) - | (GgmlType::IQ3_XXS | GgmlType::IQ4_XS, Arm::Expert) => Some(Q8_1_BLOCK), + ( + GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K | GgmlType::IQ4_XS, + _, + ) + | (GgmlType::IQ3_XXS, Arm::Expert) => Some(Q8_1_BLOCK), // Bolt's IQ2_XXS: the routed `iq2_xxs_gemv_sel` and the dense `iq2_xxs_rows` read q8_1. (GgmlType::IQ2_XXS, Arm::Expert | Arm::Gemv) => Some(Q8_1_BLOCK), // Bolt's dense NVFP4 (`nvfp4_gemv`, the ssm_in/ssm_out) reads the engine's q8_1. @@ -1012,7 +1015,6 @@ mod site_tests { for ty in [ GgmlType::IQ3_S, GgmlType::IQ3_XXS, - GgmlType::IQ4_XS, GgmlType::Q5_1, GgmlType::IQ4_NL, ] { @@ -1040,43 +1042,6 @@ mod site_tests { } } - /// Ours is the form the launches read, as their sources say it: the text of the kernel - /// table's `SiteEntry::reads` (which `site.rs`'s `SiteTy::reads` returns) and of the 32-value - /// GEMM family's module doc, so a launch that moves to another form is red here until the - /// table follows it. - #[test] - fn site_rel_reads_the_forms_the_launches_name() { - let gpu = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../gpu/src"); - let read = |file: &str| { - std::fs::read_to_string(gpu.join(file)) - .unwrap_or_else(|e| panic!("crates/gpu/src/{file}: {e}")) - }; - let (site, gemm, grouped) = ( - read("../../gguf/src/card.rs"), - read("gemm/mod.rs"), - read("gemm/grouped.rs"), - ); - for line in [ - "SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128", - "SiteEntry::Q8_0 => SiteAct::Q8x32", - "SiteEntry::F32 => SiteAct::F32", - "SiteEntry::Q5_0 => SiteAct::Q8Blocks32", - "q8_1 blocks of 128 values", - ] { - assert!(site.contains(line), "kernels.rs no longer says {line:?}"); - } - assert!( - gemm.contains("(Q8_0, IQ4_NL, Q5_1), over [`GemmAct32`]"), - "gemm/mod.rs no longer names the 32-value family" - ); - for variant in ["Iq3Xxs", "Iq4Xs"] { - assert!( - grouped.contains(variant), - "the grouped GEMM no longer decodes {variant}" - ); - } - } - /// ik's side of the table is the transcription this module already holds for the K-quants /// ([`Rule::of`]), and so is ours for Q3_K and Q4_K; Q5_K's ours differs from it by design (the /// V4.1 gemv reads f32, the bodies' site reads q8_1). diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 5024ddd24..7c5e5d320 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -105,14 +105,14 @@ pub enum Order { /// file type its form holds ([`SiteTy::of_ggml`]; an F32 plane is the f32 /// site whatever type it was widened from). `None` for a form no dense entry /// reads: a stack in the file's blocks of a type whose entry reads another -/// layout (Q8_0's planes, an i-quant), a derived weight and an NVFP4 stack. +/// layout (Q8_0's planes), a derived weight and an NVFP4 stack. /// The resident form is this crate's, so the map is here, beside the /// launches that read it, and not on the chain's [`SiteTy`]. #[must_use] pub fn of_resident(w: &DevWeight) -> Option { let ty = match w { DevWeight::KQuant { ty, .. } => { - return SiteTy::of_ggml(*ty).filter(|s| s.kquant()); + return SiteTy::of_ggml(*ty).filter(|s| s.kquant() || *s == SiteTy::Iq4Xs); } DevWeight::Q5_0 { .. } => GgmlType::Q5_0, DevWeight::Q5_1 { .. } => GgmlType::Q5_1, @@ -127,7 +127,7 @@ pub fn of_resident(w: &DevWeight) -> Option { SiteTy::of_ggml(ty) } -/// How [`kgemv`] lays a site of type `ty` out: Q3_K, Q4_K and Q6_K row-major, Q5_K and NVFP4 token-major. +/// How [`kgemv`] lays a site out: Q3_K, Q4_K and Q6_K row-major, Q5_K, IQ4_XS and NVFP4 token-major. /// `None` for a type [`kgemv`] does not run: Q8_0, F32, MXFP8, FP8_B128 and BF16 ([`gemv`]), Q5_0 /// ([`blocks32_gemv`], whose row-major layout `y[r · m + c]` is that function's own), and MLX_A4, /// a backend's family no card launch reads. @@ -135,7 +135,7 @@ pub fn of_resident(w: &DevWeight) -> Option { pub fn kgemv_order(ty: SiteTy) -> Option { match ty { SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q6K => Some(Order::RowMajor), - SiteTy::Q5K | SiteTy::Nvfp4 => Some(Order::TokenMajor), + SiteTy::Q5K | SiteTy::Iq4Xs | SiteTy::Nvfp4 => Some(Order::TokenMajor), SiteTy::Q5_0 | SiteTy::Q8_0 | SiteTy::F32 @@ -667,6 +667,22 @@ pub fn kgemv( y, ) } + SiteTy::Iq4Xs => { + let wt = kq(w, WHAT, name)?; + kk.kq.enqueue_gemv_sel( + gpu.stream(), + GgmlType::IQ4_XS, + &SelDown { + w: wt, + act, + sel: &kk.one, + n_slots: act.m(), + rows_per_expert: wt.rows(), + }, + fault, + y, + ) + } SiteTy::Nvfp4 => kk .nv .enqueue_gemv(gpu.stream(), w.nvfp4(WHAT, name)?, act, y), @@ -753,9 +769,11 @@ impl<'a> SiteWeight<'a> { /// name as `what`. fn check(self, what: &'static str, ty: SiteTy) -> Result<(), GpuError> { match self { - SiteWeight::Words { name, .. } if !ty.kquant() => Err(GpuError::shape( + SiteWeight::Words { name, .. } if gemm_weight(ty).is_none() => Err(GpuError::shape( what, - format!("{name} is {ty} on words a caller holds; only a K-quant's words launch"), + format!( + "{name} is {ty} on words a caller holds; its grouped GEMM reads no such words" + ), )), SiteWeight::Part { name, .. } if !BY_NAME.contains(&ty) => Err(GpuError::shape( what, @@ -1071,21 +1089,23 @@ pub fn gemm( src.check(WHAT, ty)?; match ty { SiteTy::Nvfp4 => Err(no_wide_gemm(WHAT, name, ty)), - SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K => k.gemm.enqueue_gemm( - stream, - GemmArgs { - ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, - w: match src { - SiteWeight::Words { w, .. } => w, - _ => kq(w, WHAT, name)?, + SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K | SiteTy::Iq4Xs => { + k.gemm.enqueue_gemm( + stream, + GemmArgs { + ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, + w: match src { + SiteWeight::Words { w, .. } => w, + _ => kq(w, WHAT, name)?, + }, + rows_per_expert: rows, + act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, + route, + input, + y, }, - rows_per_expert: rows, - act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, - route, - input, - y, - }, - ), + ) + } SiteTy::Q8_0 | SiteTy::F32 | SiteTy::Mxfp8 | SiteTy::Fp8B128 | SiteTy::Bf16 => { dense()?; let (site, part) = src.rows(WHAT)?; @@ -1550,92 +1570,40 @@ mod tests { Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, file_site, gemm_weight, kgemv_order, swiglu_launch, }; - use crate::gemm::GemmWeight; use crate::kquant::Act; use gguf::Split; use gguf::quant::GgmlType; - use gguf::write::{Layout, TensorDecl, Writer}; - use model::placement::kernels; - use std::fs::File; - use std::io::BufWriter; - - /// A site's type is read from the file's header alone: a Q8_0 matrix of - /// two rows of 64 is read as Q8_0 where Q8_0 is launched, and refused by - /// name where only K-quants are; a Q3_K, a Q5_K, a Q5_0 and an MXFP8 one - /// (bloomery's id, sized by the split) are read as theirs; an IQ4_XS one (a type no site launch reads) and a matrix of - /// another shape are refused by name. + use gguf::write::{Layout, TensorDecl}; + #[test] - fn a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused() { - let dir = std::env::temp_dir().join(format!("bloomery-site-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap_or_else(|e| panic!("{e}")); - let path = dir.join("m.gguf"); - let decls = [ - ("q8", vec![64, 2], 8u32, 2 * 2 * 34), - ("iq", vec![256, 1], 23, 136), - ("q3", vec![256, 2], 11, 2 * 110), - ("q5", vec![256, 2], 13, 2 * 176), - ("q50", vec![64, 2], 6, 2 * 2 * 22), - ("mx", vec![128, 2], GgmlType::MXFP8.as_u32(), 2 * 4 * 33), - ("bf", vec![64, 2], GgmlType::BF16.as_u32(), 2 * 64 * 2), - ]; - let tensors = decls - .iter() - .map(|(name, dims, type_id, nbytes)| TensorDecl { - name: (*name).to_string(), - dims: dims.clone(), - type_id: *type_id, - nbytes: *nbytes, - }) - .collect(); - let layout = Layout::new(&[], tensors).unwrap_or_else(|e| panic!("{e}")); - let file = BufWriter::new(File::create(&path).unwrap_or_else(|e| panic!("{e}"))); - let mut w = Writer::new(file, layout).unwrap_or_else(|e| panic!("{e}")); - for (name, _, _, nbytes) in &decls { - w.tensor(name, &vec![0u8; *nbytes as usize]) - .unwrap_or_else(|e| panic!("{e}")); - } - w.finish().unwrap_or_else(|e| panic!("{e}")); - let split = Split::open(&path).unwrap_or_else(|e| panic!("{e}")); - let all = [ - SiteTy::Q3K, - SiteTy::Q4K, - SiteTy::Q5K, - SiteTy::Q6K, - SiteTy::Q5_0, - SiteTy::Q8_0, - SiteTy::F32, - SiteTy::Nvfp4, - SiteTy::Mxfp8, - SiteTy::Bf16, - ]; - for (name, k, want) in [ - ("q8", 64, SiteTy::Q8_0), - ("q3", 256, SiteTy::Q3K), - ("q5", 256, SiteTy::Q5K), - ("q50", 64, SiteTy::Q5_0), - ("mx", 128, SiteTy::Mxfp8), - ("bf", 64, SiteTy::Bf16), + fn iq4_sites_keep_shape_and_launch_admission_separate() { + let decl = |name: &str, ty: GgmlType, bytes| TensorDecl { + name: name.to_owned(), + dims: vec![256, 2], + type_id: ty.as_u32(), + nbytes: bytes, + }; + let layout = Layout::new( + &[], + vec![ + decl("shared", GgmlType::IQ4_XS, 2 * 136), + decl("unsupported", GgmlType::IQ3_S, 2 * 110), + ], + ) + .unwrap(); + let file = Split::header_only("sites.gguf".into(), &layout).unwrap(); + assert_eq!( + file_site(&file, "test", "shared", (2, 256), SiteTy::ALL).ok(), + Some(SiteTy::Iq4Xs) + ); + for (name, shape, allowed) in [ + ("shared", (1, 256), SiteTy::ALL), + ("shared", (2, 512), SiteTy::ALL), + ("shared", (2, 256), &[SiteTy::Q4K][..]), + ("unsupported", (2, 256), SiteTy::ALL), ] { - let got = file_site(&split, "test", name, (2, k), &all); - assert_eq!(got.ok(), Some(want), "{name}"); + assert!(file_site(&file, "test", name, shape, allowed).is_err()); } - let refused = [ - ( - "q8", - (2, 64), - &[SiteTy::Q4K, SiteTy::Q6K][..], - "q8 is q8_0 in the file", - ), - ("iq", (1, 256), &all[..], "iq is iq4_xs in the file"), - ("q8", (4, 64), &all[..], "it takes 4 rows of 64"), - ]; - for (name, shape, allowed, want) in refused { - match file_site(&split, "test", name, shape, allowed) { - Err(e) => assert!(e.to_string().contains(want), "{name}: {e}"), - Ok(t) => panic!("{name} {shape:?} read as {t}"), - } - } - let _ = std::fs::remove_dir_all(&dir); } /// The SwiGLU's launch is the down's form under the rule: every K-quant @@ -1696,72 +1664,6 @@ mod tests { } } - /// Every family is the kernel table's dense row of one file type: a type - /// has a family exactly where the table names a dense entry for it (F16, - /// which loads widened to f32, is the f32 site of its widened rows), the - /// family maps back to that type and is named by it in upper case, its - /// K-quant GEMM is the one the table names for the type, it is a - /// `kgemv` exactly where it reads the q8_1 rows of 128 values, and it - /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or - /// the q8 blocks of 32. - #[test] - fn every_family_is_the_tables_dense_row_of_one_type() { - // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. - let own = [ - GgmlType::F8E4M3_ROWS, - GgmlType::MXFP8, - GgmlType::NVFP4_ROWS, - GgmlType::FP8_B128, - ]; - for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { - let family = SiteTy::of_ggml(ty); - if ty == GgmlType::F16 { - assert_eq!( - family, - Some(SiteTy::F32), - "{ty}: the f32 site of its widened rows" - ); - assert_eq!(kernels::dense(ty), None, "{ty}"); - continue; - } - assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); - if let Some(s) = family { - assert_eq!(s.ggml(), ty, "{s}"); - } - } - let expected = [ - (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), - (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), - (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), - (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), - (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), - (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), - (SiteTy::F32, GgmlType::F32, "F32", None), - (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), - (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), - (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), - (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - ]; - assert_eq!(SiteTy::ALL.len(), expected.len()); - for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { - assert_eq!(s, want); - assert_eq!(s.ggml(), ty); - assert_eq!(s.to_string(), label); - assert_eq!(gemm_weight(s), grouped); - assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); - let by_name = model::placement::accepts::dense_by_name(ty); - assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); - if let Some(entry) = by_name { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - } - assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); - assert_eq!(kgemv_order(SiteTy::Q5_0), None); - assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); - assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); - assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); - } - #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 4b0809668..bb3f77110 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -241,7 +241,8 @@ pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { /// is `None`. An NVFP4 site has a gemv and no wide GEMM, which the body's /// prompt units run every dense site through, and is `None`. A BF16 site /// reads the words only a load under `ModelTensors::bf16_words` keeps, which -/// the qwen3moe files do not set, and is `None`. The body's load and the +/// the qwen3moe files do not set, and is `None`. Its legacy shared-expert +/// dispatch does not feed IQ4_XS sites either. The body's load and the /// coverage census both read this, so a site the arena cannot feed is /// refused at load and listed together. #[must_use] @@ -249,7 +250,7 @@ pub const fn qwen3moe_dense(ty: GgmlType) -> Option { let Some(entry) = kernels::dense(ty) else { return None; }; - if let SiteEntry::Nvfp4 | SiteEntry::Bf16 = entry { + if let SiteEntry::Nvfp4 | SiteEntry::Bf16 | SiteEntry::Iq4xs = entry { return None; } match entry.reads() { diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index f11b23196..7ab1361b8 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -642,32 +642,6 @@ mod tests { ); } - // PIN(2026-10-16): the dense pairs gain NVFP4 (`nvfp4_gemv` through - // `site::kgemv`); every other pair is unchanged. - // PIN(2026-10-19): and bloomery's own MXFP8 (`mxfp8_gemv` through - // `site::gemv`); every other pair is unchanged. - // PIN(2026-10-20): and bloomery's own FP8_B128 (`fp8_b128_gemv` through - // `site::gemv`); BF16 also uses `bf16_gemv` and `bf16_tile_gemm` over its words. - #[test] - fn dense_admits_exactly_the_pinned_pairs() { - assert_eq!( - rows(dense), - [ - (GgmlType::F32, SiteEntry::F32), - (GgmlType::Q5_0, SiteEntry::Q5_0), - (GgmlType::Q8_0, SiteEntry::Q8_0), - (GgmlType::Q3_K, SiteEntry::Q3k), - (GgmlType::Q4_K, SiteEntry::Q4k), - (GgmlType::Q5_K, SiteEntry::Q5k), - (GgmlType::Q6_K, SiteEntry::Q6k), - (GgmlType::BF16, SiteEntry::Bf16), - (GgmlType::NVFP4, SiteEntry::Nvfp4), - (GgmlType::MXFP8, SiteEntry::Mxfp8), - (GgmlType::FP8_B128, SiteEntry::Fp8B128), - ] - ); - } - /// A dense entry is named by one type's row, so its file type is that /// row's type. #[test] @@ -707,101 +681,6 @@ mod tests { ); } - /// The layout a file tensor of each type loads in, and the one a routed - /// stack loads in under the default expert rule: every type's but q5_K's, - /// which that rule does not admit, and mxfp8's, which has no card expert - /// entry. - // PIN(2026-10-23): NVFP4_ROWS loads as NVFP4's planes (its routed card - // columns); every other pair is unchanged. - #[test] - fn the_card_layouts_are_the_pinned_pairs() { - assert_eq!( - rows(CardFormat::of), - [ - (GgmlType::F32, CardFormat::F32), - (GgmlType::F16, CardFormat::Bf16AsF32), - (GgmlType::Q5_0, CardFormat::Q5_0), - (GgmlType::Q5_1, CardFormat::Q5_1), - (GgmlType::Q8_0, CardFormat::Q8_0Planes), - (GgmlType::Q3_K, CardFormat::KQuant), - (GgmlType::Q4_K, CardFormat::KQuant), - (GgmlType::Q5_K, CardFormat::KQuant), - (GgmlType::Q6_K, CardFormat::KQuant), - (GgmlType::BF16, CardFormat::Bf16AsF32), - (GgmlType::NVFP4, CardFormat::Nvfp4Planes), - (GgmlType::IQ2_XXS, CardFormat::KQuant), - (GgmlType::MXFP8, CardFormat::Mxfp8Planes), - (GgmlType::NVFP4_ROWS, CardFormat::Nvfp4Planes), - (GgmlType::FP8_B128, CardFormat::Fp8B128Planes), - ] - ); - for ty in ALL { - let routed = - CardFormat::of(ty).filter(|_| !matches!(ty, GgmlType::Q5_K | GgmlType::MXFP8)); - assert_eq!(CardFormat::of_routed(ty), routed, "{ty:?}"); - } - } - - /// The types each layout holds: the types it is the load layout of, the - /// K-quants as decoded f32, bf16 as raw words, and in the file's blocks - /// every type a routed entry reads in them. - #[test] - fn each_layout_holds_exactly_the_pinned_types() { - let held = |f: CardFormat| -> Vec { - ALL.into_iter().filter(|&ty| f.holds(ty)).collect() - }; - assert_eq!( - held(CardFormat::KQuant), - [ - GgmlType::Q5_1, - GgmlType::Q8_0, - GgmlType::Q3_K, - GgmlType::Q4_K, - GgmlType::Q5_K, - GgmlType::Q6_K, - GgmlType::MXFP4, - GgmlType::IQ2_XXS, - GgmlType::IQ2_XS, - GgmlType::IQ3_XXS, - GgmlType::IQ4_NL, - GgmlType::IQ3_S, - GgmlType::IQ4_XS, - ] - ); - assert_eq!( - held(CardFormat::Bf16AsF32), - [ - GgmlType::F16, - GgmlType::Q3_K, - GgmlType::Q4_K, - GgmlType::Q5_K, - GgmlType::Q6_K, - GgmlType::BF16, - GgmlType::IQ2_XXS, - ] - ); - // PIN(2026-10-22): independent BF16 heads also occupy MXFP8 planes. - assert_eq!( - held(CardFormat::Mxfp8Planes), - [GgmlType::BF16, GgmlType::MXFP8] - ); - // PIN(2026-10-23): NVFP4_ROWS's routed stacks load as NVFP4's planes. - assert_eq!( - held(CardFormat::Nvfp4Planes), - [GgmlType::NVFP4, GgmlType::NVFP4_ROWS] - ); - for (f, ty) in [ - (CardFormat::Q5_0, GgmlType::Q5_0), - (CardFormat::Q5_1, GgmlType::Q5_1), - (CardFormat::Q8_0Planes, GgmlType::Q8_0), - (CardFormat::Fp8B128Planes, GgmlType::FP8_B128), - (CardFormat::F32, GgmlType::F32), - (CardFormat::Bf16Raw, GgmlType::BF16), - ] { - assert_eq!(held(f), [ty], "{f:?}"); - } - } - /// A card entry reads a layout the card loader can build: every type a /// row names has a [`CardFormat`] of its own, or is one of the file-block /// types a program's routed stack keeps as `KQuant`. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..012ef946d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -66,6 +66,31 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Ordinary IQ4_XS shared-site implementation + +IQ4_XS now has an ordinary file-layout/dense entry in QROWS and a common site family. +The loader keeps its original block words; `site::kgemv` uses the existing IQ4_XS +entry over one matrix and `site::gemm` uses the existing grouped GEMM. The common +gated block selects a split gate/up and sizes the down's q8_1 scratch from the +actual activation form. Its column walk and the shared wide-prompt owner use +those same sites. IQ4_XS is not relabeled as a routed expert or requantized. + +A compiled public `file_site` consumer on the pinned header refused all three +layer42 shared matrices before this change and accepts their IQ4_XS/Q8x128 +bindings after it. This reads no payload and proves neither loading nor arithmetic. +`hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay`, owned by +`gate-gpu-ds41-lib`, covers real packed-weight loading, unequal matrix widths, +one/eight-column block execution and the wide-prompt transition/captured replay. +Its nonnegative constant-weight case derives a scalar floating-operation bound; +it is not a whole-model accuracy pin. The hardware case has not been run here. + +Existing program-specific admission remains unchanged: in particular the legacy +Qwen projection/shared dispatch does not gain this site automatically. Flint +still has no Program, attention/router binding, body or serve seat. Run the new +consumer gate and the affected regressions on NVIDIA before claiming device +support; reconcile full-model coverage only with the actual body integration. + + ## 1. Sources and observed artifact From 974973983d51344771fb3b803fb78b08282b0def Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:37:10 +0900 Subject: [PATCH 10/85] gpu: use fixed-size rows in the IQ4 consumer gate --- crates/gpu-deepseek41/src/shared_routed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index c5be10e29..7eaf38e55 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1270,7 +1270,7 @@ mod tests { let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; let eps = f64::from(f32::EPSILON); let gamma = operations * eps / (1.0 - operations * eps); - for (c, row) in rows.chunks_exact(N).enumerate() { + for (c, row) in rows.as_chunks::().0.iter().enumerate() { let x = f64::from(input[c * N]); for (r, &got) in row.iter().enumerate() { let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; From dcb1e3a384a2277521ac22f3e25ec8dd9ca17ae5 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:34:14 +0900 Subject: [PATCH 11/85] gpu: size mixed expert scratch from selected resident forms --- crates/gpu-deepseek41/src/body.rs | 2 +- .../gpu-gates/src/bin/gate_glm5next_nodes.rs | 2 +- .../src/bin/gate_glm5next_twocard.rs | 2 +- .../gpu-gates/src/bin/gate_solar2_twocard.rs | 2 +- crates/gpu-gates/src/bin/generate_glm5next.rs | 2 +- .../src/bin/shared/glm5next_place.rs | 12 +- .../src/bin/shared/routed_legs/glm5next.rs | 3 +- .../src/bin/shared/serve_seats/glm.rs | 2 +- .../src/bin/shared/serve_seats/hybrid.rs | 15 +- .../src/bin/shared/serve_seats/solar2.rs | 9 +- .../src/bin/shared/serve_seats/tiered.rs | 12 +- .../gpu-gates/src/bin/shared/solar2_seat.rs | 2 +- crates/gpu-glm5next/src/tier.rs | 1 + crates/gpu-mimo2/src/card.rs | 1 + crates/gpu-minimaxm3/src/ffn.rs | 1 + crates/gpu-solar2/src/ffn.rs | 1 + crates/gpu-solar2/src/tier.rs | 1 + crates/gpu/src/arch/musespark/body/tier.rs | 2 + crates/gpu/src/arch/qwen3moe/body38.rs | 2 +- crates/gpu/src/arch/qwen3moe/tier_leg.rs | 34 +- crates/gpu/src/card_experts.rs | 367 +++++++++++++++++- crates/gpu/src/card_leg.rs | 15 +- crates/gpu/src/host/batch.rs | 28 +- crates/gpu/src/host/leg_tier.rs | 72 ++-- crates/gpu/src/host/node.rs | 14 +- crates/model/src/arch/deepseek41/kv.rs | 1 + crates/model/src/arch/glm5next/place.rs | 167 ++++---- crates/model/src/arch/mimo2/place.rs | 6 +- crates/model/src/arch/minimaxm3/place.rs | 12 +- crates/model/src/arch/musespark/place.rs | 3 +- crates/model/src/arch/qwen35moe/place.rs | 10 +- crates/model/src/arch/qwen3moe/place.rs | 4 +- crates/model/src/arch/solar2/place.rs | 157 ++++++-- crates/model/tests/placement.rs | 1 + crates/model/tests/qwen4exp_mtp_reserve.rs | 1 + crates/model/tests/solar_open2_meta.rs | 1 + crates/placement/src/accepts.rs | 37 ++ crates/placement/src/kernels.rs | 157 +++++++- crates/placement/src/placement.rs | 358 +++++++++++++++-- crates/placement/src/placement/nodes_scale.rs | 2 + .../placement/src/placement/qwen38_cards.rs | 2 + crates/placement/src/placement/workstation.rs | 139 +++++-- docs/research/flint-attach.md | 17 + 43 files changed, 1336 insertions(+), 343 deletions(-) diff --git a/crates/gpu-deepseek41/src/body.rs b/crates/gpu-deepseek41/src/body.rs index 0ef84a357..051d00605 100644 --- a/crates/gpu-deepseek41/src/body.rs +++ b/crates/gpu-deepseek41/src/body.rs @@ -2563,7 +2563,7 @@ impl Body { // The tiers' batch staging is reserved on their cards by the plan, // so it is made at load, not at the first prompt. hybrid.prepare_batch(gpu.context(), prefill::T_MAX)?; - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; Some(TierPiece::new(gpu, hybrid.slots(), PAIR_ROWS)?) }; let ced = ced::Ced::new(&CedLayer::table(hp), seq.ring_rows(), cfg.ced); diff --git a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs index 1b025c8c0..2e593a526 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs @@ -30,7 +30,7 @@ mod gate { let file = Split::open(&path)?; let inputs = PlanInputs::describe(&file)?; let (mut machine, node) = - crate::node_history::machine(inputs.model.layers, tier_batch(&inputs))?; + crate::node_history::machine(inputs.model.layers, tier_batch(&inputs)?)?; reserve_checkpoints( &mut machine, Seqs::priced(1, checkpoint_price(&inputs)?), diff --git a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs index 4fc7a7a82..3d30bbeda 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs @@ -517,7 +517,7 @@ mod gate { fn bp_machine(inputs: &PlanInputs) -> Result Machine + Copy, GateError> { glm_machine( Place::Bp, - Some(place::tier_batch(inputs)), + Some(place::tier_batch(inputs)?), 1, checkpoint_price(inputs)?, u64::try_from(CTX)?, diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index c32637070..af0916a7c 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs.hp)))?; + .machine(None, Some(place::tier_batch(&inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu-gates/src/bin/generate_glm5next.rs b/crates/gpu-gates/src/bin/generate_glm5next.rs index 96f430ee2..a6d0b08f2 100644 --- a/crates/gpu-gates/src/bin/generate_glm5next.rs +++ b/crates/gpu-gates/src/bin/generate_glm5next.rs @@ -799,7 +799,7 @@ mod cli { Some(lane_at(placement)?) }; let place = placement.name(); - let tier_batch = glm_place::tier_batch(placement, &inputs); + let tier_batch = glm_place::tier_batch(placement, &inputs)?; let machine = glm_machine( placement, tier_batch, diff --git a/crates/gpu-gates/src/bin/shared/glm5next_place.rs b/crates/gpu-gates/src/bin/shared/glm5next_place.rs index beccc1cab..39fa78350 100644 --- a/crates/gpu-gates/src/bin/shared/glm5next_place.rs +++ b/crates/gpu-gates/src/bin/shared/glm5next_place.rs @@ -39,10 +39,12 @@ pub fn parse(v: &str) -> Result { } /// The expert tier's prompt-batch bytes `place`'s plan reserves for the file -/// `inputs` describes (`model::arch::glm5next::place::tier_batch`), which -/// `Place::machine` takes; `None` under a placement with no tier card. -pub fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| model::arch::glm5next::place::tier_batch(inputs)) +/// described by `inputs` (`model::arch::glm5next::place::tier_batch`), +/// which `Place::machine` takes; `None` under a placement with no tier card. +pub fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| model::arch::glm5next::place::tier_batch(inputs)) + .transpose()?) } /// The experts `place`'s tier cards hold in its plan at `ctx` positions @@ -63,7 +65,7 @@ pub fn tier_experts( ) -> Result { let machine = glm_machine( place, - tier_batch(place, inputs), + tier_batch(place, inputs)?, slots, checkpoint_price(inputs)?, ctx, diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs index a9c0e2b89..5db7eebca 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs @@ -172,7 +172,7 @@ fn machine_of( slots: usize, ) -> Result { let price = checkpoint_price(inputs)?; - let tier_batch = glmplace::tier_batch(place, inputs); + let tier_batch = glmplace::tier_batch(place, inputs)?; Ok(glm_machine(place, tier_batch, slots, price, ctx)?( inputs.model.layers, )) @@ -242,7 +242,6 @@ fn place_of(p: &PlaceError) -> Outcome { PlaceError::Placement(e) => return refused_of(e), PlaceError::PastOracle { .. } => "past-oracle", PlaceError::FrontOver { .. } => "front-over", - PlaceError::TierLeg { .. } => "tier-leg", PlaceError::Broken(_) | PlaceError::NextnCards { .. } | PlaceError::Slots { .. } => { return Outcome::Red(p.to_string()); } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs index a982330a6..f670c06ff 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs @@ -503,7 +503,7 @@ impl Placed { (set, slots): (Option, usize), (images, side): (&Images<'_>, Option<&Side>), ) -> Result { - let tier_batch = glm_place::tier_batch(place, inputs); + let tier_batch = glm_place::tier_batch(place, inputs)?; // Every machine reserves its checkpoints at the slot context it is // laid for: the choice's probes each at their own, the load's at the // context it is granted; and the encoder's reserve, as the printed plan does. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs index e66507307..495f265e3 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs @@ -78,12 +78,16 @@ pub trait Family: Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s /// tier card ([`Tiered::tier_batch`]); a body that serves no tier card /// reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// The tags the model's description names for its think span, which the @@ -323,7 +327,10 @@ impl Tiered for Hybrid { F::think_tags(inputs).map(Some) } - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { F::tier_batch(place, inputs) } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs index 61af9a5b9..ce13c171c 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs @@ -182,8 +182,13 @@ impl Family for Solar { /// The tier's prompt-batch bytes of the file's widths ([`place::tier_batch`]), reserved on /// the placement's tier card and the host. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, bloomery_gpu_gates::GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(inputs)) + .transpose()?) } } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs index 98055fb10..a01293fef 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs @@ -203,12 +203,16 @@ pub trait Tiered: Seat + Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s tier card, for the file /// `inputs` describes; `None` under a placement with no tier card. A body that serves no tier /// card reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// Whether the plan of a load on `place` keeps the expert stream's ring out of the expert budget @@ -473,7 +477,7 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result, GateErro inputs.room = (PLAN_ROOM, HostRead::Given); // Plan (b′): the stage on the largest card, the next-largest an expert tier under the host tier, holding the tier's // prompt-batch bytes the file's widths give. - let machine = arm.machine(inputs.model.layers, place::tier_batch(&inputs.hp))?; + let machine = arm.machine(inputs.model.layers, place::tier_batch(&inputs)?)?; let place = PlanLevers::from_levers(&levers)?; let plan = inputs.plan(&machine, u64::try_from(ctx)?, &place)?; let mut m = Body::open_placed(split, &plan, &inputs, 0, levers.host())?; diff --git a/crates/gpu-glm5next/src/tier.rs b/crates/gpu-glm5next/src/tier.rs index e540f4ed2..aee07ed13 100644 --- a/crates/gpu-glm5next/src/tier.rs +++ b/crates/gpu-glm5next/src/tier.rs @@ -62,6 +62,7 @@ impl LegFamily for Glm { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: model::placement::accepts::Program::Glm5nextBody, } } diff --git a/crates/gpu-mimo2/src/card.rs b/crates/gpu-mimo2/src/card.rs index cf71fb1d1..506e518d3 100644 --- a/crates/gpu-mimo2/src/card.rs +++ b/crates/gpu-mimo2/src/card.rs @@ -27,6 +27,7 @@ fn stacks(l: usize) -> [String; 3] { /// MiMo's card experts: 256 routed experts, 8 a token. const FAMILY: CardFamily = CardFamily { what: "mimo2 CardExperts", + program: model::placement::accepts::Program::Mimo2Body, n_expert: N_EXPERT, n_used: N_USED, stacks, diff --git a/crates/gpu-minimaxm3/src/ffn.rs b/crates/gpu-minimaxm3/src/ffn.rs index 828b0897d..3850f6ba4 100644 --- a/crates/gpu-minimaxm3/src/ffn.rs +++ b/crates/gpu-minimaxm3/src/ffn.rs @@ -20,6 +20,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{ /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "minimaxm3 CardExperts", + program: model::placement::accepts::Program::MiniMaxM3Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/ffn.rs b/crates/gpu-solar2/src/ffn.rs index a0f1589d5..7be370e0a 100644 --- a/crates/gpu-solar2/src/ffn.rs +++ b/crates/gpu-solar2/src/ffn.rs @@ -17,6 +17,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{GATED_FROM, RoutedDims, back, f /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "solar2 CardExperts", + program: model::placement::accepts::Program::Solar2Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/tier.rs b/crates/gpu-solar2/src/tier.rs index 071ce8d20..e23773d93 100644 --- a/crates/gpu-solar2/src/tier.rs +++ b/crates/gpu-solar2/src/tier.rs @@ -64,6 +64,7 @@ impl LegFamily for Solar2 { n_used: experts.n_used, n_embd: inputs.hp.n_embd, ff: experts.ff, + program: model::placement::accepts::Program::Solar2Body, } } diff --git a/crates/gpu/src/arch/musespark/body/tier.rs b/crates/gpu/src/arch/musespark/body/tier.rs index 724730577..cce7df8ac 100644 --- a/crates/gpu/src/arch/musespark/body/tier.rs +++ b/crates/gpu/src/arch/musespark/body/tier.rs @@ -30,6 +30,7 @@ use model::arch::models::{LayerSpec, Moe}; use model::arch::musespark::place::{self, PlanInputs}; use model::moe::HostLayer; use model::placement::ModelTensors; +use model::placement::accepts::Program; use model::r8file::R8Source; use crate::card_slots::{CardAcc, CardSlotKernels, ColPlaces}; @@ -87,6 +88,7 @@ impl LegFamily for MuseSpark { n_used: hp.n_used, n_embd: hp.latent, ff: hp.expert_width(), + program: Program::MuseSparkBody, } } diff --git a/crates/gpu/src/arch/qwen3moe/body38.rs b/crates/gpu/src/arch/qwen3moe/body38.rs index 6edabca29..2d6d45b5e 100644 --- a/crates/gpu/src/arch/qwen3moe/body38.rs +++ b/crates/gpu/src/arch/qwen3moe/body38.rs @@ -1910,7 +1910,7 @@ impl Body38 { hybrid.prepare_batch(gpu.context(), host_cols)?; if tiered { let cards: Vec = tiers.iter().map(|t| t.card).collect(); - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; } // The PLE table's tier is the plan's: the host set built above holds // it on the host; on the NVMe tier its reader reads it. diff --git a/crates/gpu/src/arch/qwen3moe/tier_leg.rs b/crates/gpu/src/arch/qwen3moe/tier_leg.rs index c82babfa3..f5b383b89 100644 --- a/crates/gpu/src/arch/qwen3moe/tier_leg.rs +++ b/crates/gpu/src/arch/qwen3moe/tier_leg.rs @@ -1,8 +1,7 @@ //! The Qwen3-30B and Qwen3.6 expert tier: one card beside the stage card, //! hung through the common card leg ([`crate::host::leg_tier`]). What is -//! Qwen's here is the node family — the file's reading, `silu(g)·u`, and the -//! placed leg's arms — and the join of the tier's slots with the stage -//! card's. +//! Qwen's here is the node family — the file's reading and `silu(g)·u` — +//! and the join of the tier's slots with the stage card's. //! //! The join is the host leg's combine (`hostleg`) with the tier's slots in //! the same slot order. A slot the stage card holds is the stage card's down; @@ -22,7 +21,7 @@ use gguf::Split; use model::arch::models::{Act as SpecAct, LayerSpec, Moe}; use model::moe::HostLayer; use model::placement::ModelTensors; -use model::placement::accepts::{self, LegArms}; +use model::placement::accepts::Program; use model::r8file::R8Source; use crate::fault::{FaultSink, FaultSite}; @@ -51,13 +50,6 @@ const _: () = assert!(model::arch::qwen35moe::place::LEG_TIER_CHUNK == LEG_CHUNK /// Threads per block of both join entries. const THREADS: u32 = 256; -/// A gate or up stack the placed leg does not run. -const GATE_UP: &str = "a resident routed gate or up stack of a type the placed leg runs \ - (`accepts::prompt_leg_gate_up`)"; - -/// A down stack the placed leg does not run. -const DOWN: &str = - "a resident routed down stack of a type the placed leg runs (`accepts::prompt_leg_down`)"; /// The routed activation the tier runs: `silu(g)·u`. A clamped SwiGLU, and /// any other activation, is refused by name. @@ -78,18 +70,6 @@ fn qwen_act(moe: &Moe) -> Result { } } -/// The placed leg's arms ([`accepts::leg_arms`] of [`accepts::Program::Qwen3moeBody`]). -/// -/// # Panics -/// -/// When that program names no card-leg arms. -fn qwen_arms() -> (LegArms, (&'static str, &'static str)) { - let arms = match accepts::leg_arms(accepts::Program::Qwen3moeBody) { - Some(arms) => arms, - None => panic!("{WHAT}: Program::Qwen3moeBody names no card-leg arms"), - }; - (arms, (GATE_UP, DOWN)) -} /// The host views of the routed layers `run`, the refusal naming the run as /// `word`'s. `n_layer` is the description's layer count, so a trunk layer is @@ -154,6 +134,7 @@ impl LegFamily for Qwen3 { n_used: e.n_used, n_embd: inputs.hp.n_embd, ff: e.ff, + program: Program::Qwen3moeBody, } } @@ -175,9 +156,6 @@ impl LegFamily for Qwen3 { qwen_act(moe) } - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - Some(qwen_arms()) - } } /// Qwen3.6's facts for the common tier node service. The description's @@ -211,6 +189,7 @@ impl LegFamily for Qwen35 { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: Program::Qwen35Body, } } @@ -232,9 +211,6 @@ impl LegFamily for Qwen35 { qwen_act(moe) } - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - Some(qwen_arms()) - } } /// Qwen3-30B's tier node service ([`leg_tier::node_family`]). diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 8ca374858..7c53831d5 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -41,9 +41,11 @@ use std::mem::ManuallyDrop; use cuda_core::{CudaStream, DeviceBuffer}; use model::arch::models::{Ffn, LayerSpec}; +use model::placement::accepts::{self, LegArms, Program}; +use model::placement::kernels::{self, DownForms}; use model::arch::place_hybrid::RingForms; -use model::placement::kernels::{self, DownAct, DownForms}; +use model::placement::kernels::DownAct; use crate::card_leg::{ self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, @@ -57,6 +59,12 @@ use crate::ring_gemm::RingGemm; use crate::weights::Weights; use crate::{DeviceTensor, Gpu, GpuError}; +/// The selected program, rather than a union of unrelated launches, admits stacks. +const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected card gate·up arm runs", + "a resident routed down stack the selected card down arm runs", +); + /// What a family fixes of its card experts. #[derive(Clone, Copy)] pub struct CardFamily { @@ -68,6 +76,8 @@ pub struct CardFamily { pub n_used: usize, /// Layer `l`'s gate, up and down stacks' names. pub stacks: fn(usize) -> [String; 3], + /// The program whose selected gate·up and down launches run the stacks. + pub program: Program, } /// The kernels a run launches: the card leg's (its sum stage included), and @@ -75,6 +85,7 @@ pub struct CardFamily { struct CardKernels { leg: CardLeg, places: CardSlotKernels, + arms: LegArms, } impl CardFamily { @@ -266,8 +277,7 @@ pub struct CardExperts { ff: usize, /// The model's width. n: usize, - /// The forms the card layers' downs read, which every run's buffers hold - /// ([`card_downs`]). + /// The union of the resident layers' selected down activation forms. downs: DownForms, /// The one-token step's buffers. step: CardBufs, @@ -322,6 +332,10 @@ impl CardExperts { tiers: bool, ) -> Result { let what = family.what; + let arms = accepts::leg_arms(family.program).ok_or(GpuError::state( + what, + "the selected program's card leg arms", + ))?; let mut per = Vec::with_capacity(layers.len()); let mut tier_k = Vec::with_capacity(layers.len()); let mut downs = DownForms::NONE; @@ -362,9 +376,13 @@ impl CardExperts { per.push(None); continue; } - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; stacks.check(n_card, family.shape(n, ff, 1))?; - downs = downs.with(down_act(what, &layer, stacks.down_ty())?); + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + what, + "the selected card down's activation form", + ))?; + downs = downs.with(act); per.push(Some(layer)); } let downs = card_downs(downs); @@ -373,6 +391,7 @@ impl CardExperts { k: CardKernels { leg: CardLeg::load(gpu)?, places: CardSlotKernels::load(gpu.context())?, + arms, }, layers: per, tier: tier_k, @@ -804,7 +823,7 @@ impl CardKernels { io: &Io<'_>, sel: &DeviceBuffer, ) -> Result<(), GpuError> { - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer: l, stacks: &stacks, @@ -830,22 +849,328 @@ impl CardKernels { #[cfg(test)] mod tests { - use super::{DownAct, DownForms, card_downs}; + use super::*; + use gguf::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + use model::arch::models::{self, Mixer, Moe, Residual, Router, RouterNorm, Score}; + + use crate::card_leg::tests::stack_bytes; + use crate::host::leg_tier::LegTier; + use crate::host::tier::{TierBlock, TierExperts, TierInput, TierIo, TierSet}; + use crate::hybrid::HOST; + + const N: usize = 256; + const FF: usize = 256; + const USED: usize = 8; + const EXPERTS: usize = 16; + const CARD: usize = 4; + const CAP: usize = 3; + const FAMILY: CardFamily = CardFamily { + what: "mixed card wrapper", + n_expert: EXPERTS, + n_used: USED, + stacks: model::arch::names::routed_stacks, + program: Program::Qwen35Body, + }; + + fn layers(n: usize) -> Vec { + vec![ + LayerSpec { + mixer: Mixer::None, + ffn: Ffn::Moe(Moe { + experts: EXPERTS as u32, + top_k: USED as u32, + expert_ff: FF as u32, + latent: None, + act: models::Act::SwiGlu { limit: None }, + router: Router { + score: Score::Sigmoid, + bias: false, + norm: RouterNorm::SelectedSum, + scale: 1.0, + hash: false, + }, + shared: None, + }), + residual: Residual::Plain, + extras: Vec::new(), + }; + n + ] + } + + fn weights(gpu: &Gpu, triples: &[[GgmlType; 3]]) -> Weights { + let mut decls = Vec::new(); + let mut data = Vec::new(); + for (l, triple) in triples.iter().enumerate() { + for (part, (name, &ty)) in (FAMILY.stacks)(l).into_iter().zip(triple).enumerate() { + let (rows, k) = if part == 2 { (N, FF) } else { (FF, N) }; + let bytes = stack_bytes(ty, CARD * rows, k, 7 + (3 * l + part) as u64); + decls.push(TensorDecl { + name, + dims: vec![k as u64, rows as u64, CARD as u64], + type_id: ty.as_u32(), + nbytes: bytes.len() as u64, + }); + data.push(bytes); + } + } + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let call = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "bloomery-mixed-wrappers-{}-{call}.gguf", + std::process::id() + )); + let layout = Layout::new(&[], decls.clone()).expect("a layout"); + let file = std::io::BufWriter::new(std::fs::File::create(&path).expect("a file")); + let mut writer = Writer::new(file, layout).expect("a writer"); + for (decl, bytes) in decls.iter().zip(data) { + writer.tensor(&decl.name, &bytes).expect("a tensor"); + } + writer.finish().expect("a finished file"); + let file = gguf::Split::open(&path).expect("the file"); + let weights = Weights::load_where(gpu.stream(), &file, |_| true).expect("resident stacks"); + std::fs::remove_file(path).expect("remove the fixture"); + weights + } + + fn assert_bits(got: &[f32], want: &[f32]) { + assert_eq!(got.len(), want.len()); + for (i, (&g, &w)) in got.iter().zip(want).enumerate() { + assert!(g.is_finite() && w.is_finite(), "lane {i}: {g} vs {w}"); + assert_eq!(g.to_bits(), w.to_bits(), "lane {i}: {g} vs {w}"); + } + } - /// A load's leg buffers hold the forms its card layers' downs read — the - /// 32-value form for NVFP4's, both for a mix — and the 128-value form on a - /// load with no card layer. Mutant: the buffers back on the 128-value form - /// alone (an NVFP4 layer's down finds no q8 blocks of 32 and is refused at - /// its first run). + /// Each real wrapper uses its resident layers' union, not a model-wide + /// quant: both one-form loads and mixed loads conserve bytes and execute + /// step, pass, batch and tier chunks as the same common leg. Host slots + /// stay untouched on the tier, including a chunk's short final run. #[test] - fn the_leg_buffers_hold_the_card_downs_forms() { - let q8 = DownForms::of(DownAct::Q8Act); - let b32 = DownForms::of(DownAct::Q8Blocks32); - assert_eq!(card_downs(DownForms::NONE), q8); - assert_eq!(card_downs(q8), q8); - assert_eq!(card_downs(b32), b32); - assert!(!card_downs(b32).holds(DownAct::Q8Act)); - let both = q8.with(DownAct::Q8Blocks32); - assert_eq!(card_downs(both), both); + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_mixed_card_and_tier_wrappers_conserve_forms_and_rows() { + use GgmlType::{IQ3_S, IQ4_XS, Q5_K, Q6_K, Q8_0}; + let gpu = Gpu::with_device(0).expect("a Gpu on device 0"); + let stream = gpu.stream(); + let leg = CardLeg::load(&gpu).expect("the reference leg"); + for triples in [ + vec![[IQ3_S, IQ3_S, Q5_K], [IQ4_XS; 3]], + vec![[Q5_K, Q5_K, Q8_0]], + vec![[Q5_K, Q5_K, Q6_K]], + vec![ + [IQ3_S, IQ3_S, Q5_K], + [Q5_K, Q5_K, Q8_0], + [Q5_K, Q5_K, Q6_K], + [IQ4_XS; 3], + ], + ] { + let w = weights(&gpu, &triples); + let specs = layers(triples.len()); + let map = SlotMap::prefix(0..triples.len(), EXPERTS, CARD).expect("a map"); + let mut card = CardExperts::new(&gpu, &w, FAMILY, &specs, &map, (N, FF), CAP) + .expect("the card wrapper"); + let mut batch = card.batch(&gpu, CAP).expect("the batch"); + assert_eq!(card.batch_bytes(CAP), batch.bytes()); + let forms = triples.iter().fold(DownForms::NONE, |forms, tys| { + forms.with(kernels::down(tys[2]).expect("a down").1) + }); + let counted = |cap| { + kernels::leg_bytes(cap as u64, USED as u64, N as u64, FF as u64, forms) as usize + }; + assert_eq!( + card.bytes(), + counted(1) + counted(CAP) + 2 * size_of::() * (1 + CAP) * N, + ); + assert_eq!( + batch.bytes(), + counted(CAP) + 2 * size_of::() * CAP * N + size_of::() * CAP * USED, + ); + let shape = FAMILY.shape(N, FF, 2); + let set = TierSet::new( + 0..triples.len(), + EXPERTS, + vec![(0..CARD as u32).collect(); triples.len()], + ) + .expect("the tier set"); + let mut tier = LegTier::new( + &gpu, &specs, &set, &w, shape, FAMILY.program, + |moe| Ok(Act::SwigluClamp { limit: moe.act.swiglu_limit() }), + ) + .expect("the tier wrapper"); + assert_eq!(tier.block_bytes(), counted(shape.cap)); + let host_x: Vec = (0..CAP * N).map(|i| (i % 71) as f32 / 71.0 - 0.5).collect(); + let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); + let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); + let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); + let route = + DeviceBuffer::from_host(stream, &vec![0.125f32; CAP * USED]).expect("weights"); + let host_sel: Vec = (0..CAP * USED) + .map(|i| { + if i % USED == USED - 1 { + HOST + } else { + (i % CARD) as u32 + } + }) + .collect(); + let sel = DeviceBuffer::from_host(stream, &host_sel).expect("the places"); + let ids = DeviceBuffer::from_host( + stream, + &host_sel + .iter() + .map(|&s| if s == HOST { (EXPERTS - 1) as u32 } else { s }) + .collect::>(), + ) + .expect("the ids"); + let places = DeviceTensor::upload(stream, &map.stage_view(), triples.len(), EXPERTS) + .expect("the map's card view"); + let act = crate::Q8Act::with_tier_cols(stream, CAP, N).expect("the tier block's form"); + for (l, tys) in triples.iter().enumerate() { + let names = (FAMILY.stacks)(l); + let stacks = LegStacks::of_arms( + &w, + names.each_ref().map(String::as_str), + accepts::leg_arms(FAMILY.program).expect("the selected arms"), + STACK_ROLES, + ) + .expect("the reference stacks"); + let mut reference = LegBufs::for_down( + stream, + FAMILY.shape(N, FF, CAP), + kernels::down(tys[2]).expect("a down").1, + ) + .expect("the reference buffers"); + let mut acc = DeviceBuffer::zeroed(stream, CAP * N).expect("the reference sum"); + for m in [1, CAP] { + let io = Io { + normed: &x, + weights: &route, + x1: &x1, + m, + }; + let rule = Act::SwigluClamp { limit: 0.0 }; + leg.enqueue( + &gpu, + &mut reference, + &LegRun { + layer: l, + stacks: &stacks, + normed: &x, + m, + sel: &sel, + n_card: CARD, + rule, + }, + Some(LegSum { + w: &route, + acc: &mut acc, + }), + ) + .expect("the reference"); + let sums = acc.to_host_vec(stream).expect("the reference sums"); + let want: Vec = sums[..m * N] + .iter() + .zip(&base) + .map(|(&a, &b)| a + b) + .collect(); + let got = if m == 1 { + card.enqueue_step(&gpu, &w, l, rule, &io, &sel) + .expect("step"); + card.pre() + } else { + card.enqueue_pass(&gpu, &w, l, rule, &io, &sel) + .expect("pass"); + card.pass_pre().expect("pass output") + }; + assert_bits(&got.to_host_vec(stream).expect("output")[..m * N], &want); + card.enqueue_rows( + &gpu, + &w, + &mut batch, + l, + rule, + &io, + (&ids, &places, l * EXPERTS), + ) + .expect("batch"); + assert_bits( + &batch.pre().to_host_vec(stream).expect("batch output")[..m * N], + &want, + ); + let mut down = + DeviceBuffer::from_host(stream, &vec![1.0e30f32; CAP * USED * N]) + .expect("the tier output"); + if m == 1 { + tier.enqueue_layer( + &gpu, + &w, + l, + TierIo { + act: TierInput::F32(&x), + sel: &sel, + rows: &mut down, + cols: m, + }, + ) + .expect("tier step"); + } else { + tier.enqueue_block( + &gpu, + &w, + l, + TierBlock { + x: &x, + act: &act, + sel: &sel, + cols: m, + down: &mut down, + }, + ) + .expect("tier chunks"); + } + let got = down.to_host_vec(stream).expect("tier output"); + let want = reference + .down() + .to_host_vec(stream) + .expect("reference rows"); + for (s, &place) in host_sel[..m * USED].iter().enumerate() { + if place == HOST { + assert!(got[s * N..(s + 1) * N].iter().all(|&v| v == 1.0e30)); + } else { + assert_bits(&got[s * N..(s + 1) * N], &want[s * N..(s + 1) * N]); + } + } + } + } + assert!(gpu.take_fault().expect("the fault word").is_none()); + } + for (triple, program) in [ + ([IQ3_S, Q5_K, Q8_0], Program::Qwen35Body), + ([Q5_K, Q5_K, Q6_K], Program::Solar2Body), + ] { + let w = weights(&gpu, &[triple]); + let specs = layers(1); + let map = SlotMap::prefix(0..1, EXPERTS, CARD).expect("a map"); + let family = CardFamily { program, ..FAMILY }; + assert!( + matches!( + CardExperts::new(&gpu, &w, family, &specs, &map, (N, FF), CAP), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + let set = + TierSet::new(0..1, EXPERTS, vec![(0..CARD as u32).collect()]).expect("a tier set"); + assert!( + matches!( + LegTier::new( + &gpu, &specs, &set, &w, family.shape(N, FF, 2), program, + |moe| Ok(Act::SwigluClamp { limit: moe.act.swiglu_limit() }), + ), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + } } } diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index d153139e9..d4e28c30c 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -629,13 +629,14 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: kernels::card_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: accepts::planes_leg_down, }, (GATE_UP, DOWN), ) } + /// [`LegStacks::of`] for a body that loads every tensor in the format /// [`gguf::card::CardFormat::of`] names (`Weights::load`), whose downs may /// also be Q5_0 in the `pack_q5_0` layout ([`accepts::resident_down_runs`]). @@ -647,8 +648,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: accepts::resident_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: |ty| accepts::resident_down_runs(ty) || kernels::card_planes_runs(ty), }, (GATE_UP, DOWN), ) @@ -668,9 +669,9 @@ impl<'a> LegStacks<'a> { ) -> Result, GpuError> { let [gate, up, down] = names; let (gate, up, down) = ( - LegStack::of(w, gate, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, up, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, down, arms.down, kernels::card_planes_runs, roles.1)?, + LegStack::of(w, gate, arms.gate_up, arms.gate_up, roles.0)?, + LegStack::of(w, up, arms.gate_up, arms.gate_up, roles.0)?, + LegStack::of(w, down, arms.down, arms.down, roles.1)?, ); if up.ty != gate.ty { return Err(GpuError::Tensor { diff --git a/crates/gpu/src/host/batch.rs b/crates/gpu/src/host/batch.rs index 5a69a4707..29dc6c687 100644 --- a/crates/gpu/src/host/batch.rs +++ b/crates/gpu/src/host/batch.rs @@ -29,7 +29,7 @@ use crate::graph::{MappedHost, cu}; use crate::tensor::WindowMut; use crate::{Gpu, GpuError, Q8Act}; use cuda_core::{CudaContext, CudaEvent, CudaStream, DeviceBuffer, PinnedHostBuffer, sys}; -use model::placement::Machine; +use model::placement::Plan; use model::placement::workstation::{TIER_BATCH_HOST_RESERVE, TIER_BATCH_RESERVE}; use model::{Tensor2, Tensor2View}; use std::mem::size_of; @@ -1079,18 +1079,19 @@ impl HostTier { Ok(()) } - /// The expert tiers' prompt-batch bytes against the reserves `machine`'s + /// The expert tiers' prompt-batch bytes against the selected reserves the /// plan carries for them: on each tier's card — tier `t`'s is index - /// `cards[t]` of [`Machine::all_cards`] — its batch staging and the + /// `cards[t]` of [`model::placement::Machine::all_cards`] — its batch staging and the /// architecture's block scratch ([`TierCard::block_bytes`]) against that - /// card's [`TIER_BATCH_RESERVE`] row; on the host, the tier legs' rows and + /// card's selected [`TIER_BATCH_RESERVE`] row in `plan`; on the host, the tier legs' rows and /// places against the host's [`TIER_BATCH_HOST_RESERVE`] row. A card list /// that is not one card a tier, a missing row, a row named twice, or a /// difference is refused by name, naming the reserve. Load-time only, once /// [`HostTier::prepare_batch`] has made the legs. Step-only tiers make /// no batch leg ([`HostTier::prepare_tier_batch`]) and none is checked. - pub fn check_tier_reserves(&self, machine: &Machine, cards: &[usize]) -> Result<(), GpuError> { + pub fn check_tier_reserves(&self, plan: &Plan<'_>, cards: &[usize]) -> Result<(), GpuError> { const WHAT: &str = "HostTier::check_tier_reserves"; + let machine = plan.machine; if self.tiers.iter().any(|t| !t.supports_blocks()) { return Ok(()); } @@ -1124,6 +1125,7 @@ impl HostTier { &card.reserves, TIER_BATCH_RESERVE, port.tier_device_bytes(i) + t.block_bytes()?, + Some(index), )); } rows.push(( @@ -1131,15 +1133,27 @@ impl HostTier { &machine.host().reserves, TIER_BATCH_HOST_RESERVE, port.tier_host_bytes(), + None, )); - for (on, reserves, name, got) in rows { + for (on, reserves, name, got, card) in rows { let want = match reserves .iter() .filter(|(n, _)| n == name) .collect::>() .as_slice() { - [(_, b)] => *b, + [(_, b)] => match card { + Some(c) => plan + .card_reserve(c, name) + .map_err(|e| GpuError::plan(WHAT, e))? + .ok_or_else(|| { + GpuError::shape( + WHAT, + format!("the plan's selected reserve {name:?} on {on} is invalid"), + ) + })?, + None => *b, + }, named => { return Err(GpuError::shape( WHAT, diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 0762476e1..04572a78a 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -9,7 +9,7 @@ //! The tier's layer is the card leg ([`CardLeg::enqueue_rows`]) over the //! tier's stacks: the q8_1 of the activation, the gate·up `_sel` with the //! routed activation the family names (clamped SwiGLU unless it names -//! another), the q8_1 of the tier slots' columns and the down +//! another), the selected down forms of the tier slots' columns and the down //! `_sel` into the tier's rows; over a prompt batch's block, the same run //! chunk by chunk of `shape.cap` tokens. Each slot's down output is the one //! the stage card would write for it, so the card sum over both cards' slots @@ -30,7 +30,7 @@ use model::ModelError; use model::arch::models::{Ffn, LayerSpec, Moe}; use model::arch::names::routed_stacks; use model::moe::HostLayer; -use model::placement::accepts::LegArms; +use model::placement::accepts::{self, LegArms, Program}; use model::placement::kernels::{self, DownAct, DownForms}; use model::placement::{ModelTensors, Plan}; use model::r8file::R8Source; @@ -53,6 +53,12 @@ const WHAT: &str = "LegTier"; /// What the stage card's side of the tier layers names in its errors. const SIDE: &str = "TierSide"; +/// The selected program's launches admit the same stacks on both cards. +const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected tier gate·up arm runs", + "a resident routed down stack the selected tier down arm runs", +); + /// The tier the stage card's tier entries serve: they hand one tier its /// image and join one tier's rows, so a load holds one tier card /// ([`crate::host::refuse_tier_count`]). @@ -173,9 +179,8 @@ impl TierLayer { /// leg over the tier's stacks into the tier's rows; over a prompt batch's /// block, the same chunk by chunk. Its scratch, one [`LegBufs`] for /// `shape.cap` tokens that a step's one token and each chunk share, is made -/// at load. The activation and the stack arms are the family's -/// ([`LegFamily::tier_act`], [`LegFamily::tier_arms`]); a family that names -/// no arms uses the card leg's own. +/// at load. The activation is the family's ([`LegFamily::tier_act`]); the +/// selected program names the stack arms. pub struct LegTier { leg: CardLeg, bufs: LegBufs, @@ -183,30 +188,33 @@ pub struct LegTier { /// Per layer of `layers`, `None` off the tier. cfg: Vec>, shape: LegShape, - /// The family's stack arms, or the card leg's own when it named none. - arms: Option<(LegArms, (&'static str, &'static str))>, + arms: LegArms, } impl LegTier { /// The tier's computation on `gpu` over `set`, whose stacks `w` holds, /// for the description `layers`, through the leg of `shape` (`cap` tokens /// of `slots_per_col` slots a run, `n` wide, experts `ff` wide). The - /// activation is `F`'s ([`LegFamily::tier_act`]) and the stacks are read - /// by `F`'s arms ([`LegFamily::tier_arms`]; the card leg's own when the - /// family names none). A tier layer whose three stacks are absent, of a - /// type those arms do not read, or of other rows than its experts', and - /// a layer off the tier with resident stacks, are refused by name. The + /// activation is supplied by the family's [`LegFamily::tier_act`] and the + /// selected program names the stack arms. A tier layer whose three stacks + /// are absent, of a type those arms do not read, or of other rows than its + /// experts', and a layer off the tier with resident stacks, are refused by name. The /// leg's buffers hold the columns in the forms the tier layers' downs /// read ([`kernels::down`]), the 128-value form on a tier of no layer. /// Load-time only. - pub fn new( + pub fn new( gpu: &Gpu, layers: &[LayerSpec], set: &TierSet, w: &Weights, shape: LegShape, + program: Program, + activation: fn(&Moe) -> Result, ) -> Result { - let arms = F::tier_arms(); + let arms = accepts::leg_arms(program).ok_or(GpuError::state( + WHAT, + "the selected program's card leg arms", + ))?; let run = set.layers(); let mut cfg = Vec::with_capacity(run.len()); let mut downs = DownForms::NONE; @@ -232,17 +240,16 @@ impl LegTier { }; let layer = TierLayer { k, - rule: F::tier_act(m)?, + rule: activation(m)?, names, }; - let stacks = stacks_of(arms, w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; stacks.check(k, shape)?; - let (_, form) = kernels::down(stacks.down_ty()).ok_or_else(|| GpuError::Tensor { - what: WHAT, - name: layer.names[2].clone(), - need: "a down stack of a type the kernel table names a down entry for", - })?; - downs = downs.with(form); + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + WHAT, + "the selected tier down's activation form", + ))?; + downs = downs.with(act); cfg.push(Some(layer)); } if downs == DownForms::NONE { @@ -323,7 +330,7 @@ impl TierExperts for LegTier { .. } = *shape; let tl = layer_of(layers, cfg, layer, W)?; - let stacks = stacks_of(arms, weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), *arms, STACK_ROLES)?; for c0 in (0..io.cols).step_by(cap) { let cn = cap.min(io.cols - c0); let slots = cn * slots_per_col; @@ -367,7 +374,7 @@ impl TierExperts for LegTier { .to_string(), }); }; - let stacks = stacks_of(arms, weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer, stacks: &stacks, @@ -482,6 +489,8 @@ pub struct LegWidths { pub n_embd: usize, /// A routed expert's width. pub ff: usize, + /// The program whose card leg runs these experts. + pub program: Program, } impl LegWidths { @@ -561,12 +570,6 @@ pub trait LegFamily { }) } - /// The arms a tier layer's stacks are read by, with the gate-or-up - /// refusal's words and the down's, or `None` for the card leg's own - /// arms ([`LegStacks::of`]). - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - None - } } /// The family `F`'s tier node service: the placement tensors of a file, the @@ -609,13 +612,16 @@ fn node_experts( open: &NodeOpen, ) -> Result, GpuError> { let inputs = node_inputs::(file, open)?; - let shape = F::widths(&inputs).shape(F::CHUNK); - Ok(Box::new(LegTier::new::( + let widths = F::widths(&inputs); + let shape = widths.shape(F::CHUNK); + Ok(Box::new(LegTier::new( gpu, F::layers(&inputs), &open.set, w, shape, + widths.program, + F::tier_act, )?)) } @@ -696,5 +702,5 @@ pub fn hang_tiers( let planned: Vec = hang.tiers.iter().map(|t| t.card).collect(); hybrid.attach_tiers(cards, gpu)?; hybrid.prepare_batch(gpu.context(), hang.cols)?; - hybrid.check_tier_reserves(plan.machine, &planned) + hybrid.check_tier_reserves(plan, &planned) } diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index 0d2b3df3b..15636de56 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -321,13 +321,13 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let reserve = plan - .machine - .card(self.card) - .ok_or_else(|| { - GpuError::shape(WHAT, "the projected tier card is absent from the plan") - })? - .set_aside_bytes() + let totals = plan.cards.get(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + let reserve = totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, diff --git a/crates/model/src/arch/deepseek41/kv.rs b/crates/model/src/arch/deepseek41/kv.rs index d7fd1ebb7..8a45d8fd6 100644 --- a/crates/model/src/arch/deepseek41/kv.rs +++ b/crates/model/src/arch/deepseek41/kv.rs @@ -263,6 +263,7 @@ mod tests { let machine = Machine { cards: vec![Card { name: "pool".into(), + tier_leg: None, device: None, usable_bytes: pool, context_bytes: 0, diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 7be86b816..d497ca063 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -146,53 +146,26 @@ fn down_forms(downs: impl Iterator) -> DownForms { .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)) } -/// The down forms of the expert tier's leg buffers: those of every routed layer the card experts -/// run ([`carded_layers`]), the tier holding experts of any of them, tiles' layers included (the -/// tier runs the card leg over all of them); the 128-value form when the file has none, the form -/// the load's leg buffers default to. -fn tier_downs(model: &ModelTensors) -> DownForms { - let forms = down_forms(carded_layers(model).map(|(_, [_, _, d])| d)); - if forms == DownForms::NONE { - DownForms::of(DownAct::Q8Act) - } else { - forms - } -} - -/// The first routed layer of `model` the card experts run that the expert -/// tier card's plan does not place on its leg, with its stacks' types: the -/// plan places the file's blocks there over the 128-value form, each part by -/// its arm ([`kernels::card_gate_up_runs`] for the gate and up, -/// [`kernels::card_q8act_down_runs`] for the down, a Q6_K down among them), -/// so a plan that hangs a tier card is refused it ([`PlaceError::TierLeg`]). -fn off_tier_layer(model: &ModelTensors) -> Option<(usize, [GgmlType; 3])> { - carded_layers(model).find(|&(_, [g, u, d])| { - !(kernels::card_gate_up_runs(g) - && kernels::card_gate_up_runs(u) - && kernels::card_q8act_down_runs(d)) - }) -} /// Tokens a chunk of the tier's prompt-batch service runs at most: the /// stage card's chunk of the card experts' launches, which the body binds /// to its own constant. pub const TIER_CHUNK: u64 = 8; -/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for the file `inputs` describes and -/// blocks of the host union's columns: the tier card's staging, one run of the card leg's buffers -/// over [`TIER_CHUNK`] tokens in each form the tier layers' downs read ([`tier_downs`]; the -/// card leg's own formula, [`workstation::tier_leg_batch_bytes_of`]) and the host's rows and -/// places, which the plan reserves on the tier card and the host and the load checks its -/// allocations against. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of +/// the host union's columns: the tier card's staging, the chunked card +/// path's scratch ([`workstation::tier_chunk_scratch_bytes`]) and the host's +/// rows and places, which the plan reserves on the tier card and the host +/// and the load checks its allocations against. +pub fn tier_batch(inputs: &PlanInputs) -> Result { let hp = &inputs.hp; - workstation::tier_leg_batch_bytes_of( + workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, - (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), - tier_downs(&inputs.model), + TIER_CHUNK, + crate::moe::UNION_MAX_COLS as u64, + crate::placement::accepts::leg_down_forms(Program::Glm5nextBody, &inputs.model), ) } @@ -448,16 +421,7 @@ pub enum PlaceError { /// A plan of no resident sequence: a load serves at least one. #[error("a plan of {slots} resident sequences: a load serves at least one")] Slots { slots: usize }, - /// A machine that hangs an expert tier card beside a file with a card - /// layer the plan does not place on the tier's leg (NVFP4's planes, a down - /// over the 32-value form): the plan places the file's blocks over the - /// 128-value form alone there. - #[error( - "layer {layer}: routed stacks {tys:?} run on the stage card's leg, and the expert tier \ - card's leg reads the file's blocks over 128-value q8_1 columns alone; plan it without \ - the tier card" - )] - TierLeg { layer: usize, tys: [GgmlType; 3] }, + } impl PlanRefusal for PlaceError { @@ -683,7 +647,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(lanes); let terms = self.seq_terms(&kv, None); let n = slots as u64; @@ -730,17 +693,6 @@ impl PlanInputs { Ok(plan) } - /// Refused by name ([`PlaceError::TierLeg`]) when `machine` hangs an - /// expert tier card and a card layer of the file is one the tier's leg - /// does not run. - fn refuse_off_tier(&self, machine: &Machine) -> Result<(), PlaceError> { - match off_tier_layer(&self.model) { - Some((layer, tys)) if machine.expert_tiers().next().is_some() => { - Err(PlaceError::TierLeg { layer, tys }) - } - _ => Ok(()), - } - } } /// Card bytes the NextN walk's arena may hold beside the plan's tensors @@ -861,6 +813,7 @@ fn nextn_machine() -> Machine { Machine { cards: vec![Card { name: "nextn".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -1045,7 +998,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(KdaLanes::Two); let terms = self.seq_terms(&kv, Some(nextn)); let n = slots as u64; @@ -1622,6 +1574,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 1 << 20, @@ -2275,44 +2228,74 @@ mod tests { } } - /// A machine that hangs an expert tier card is refused, by layer - /// and types, a file with a card layer the tier's leg does not run - /// (an NVFP4 layer: its planes and 32-value form); without the tier - /// card the same file plans, the NVFP4 layer's experts on the stage - /// card, and a K-quant file is not refused with it - /// ([`node_ranges_use_the_common_loaded_assignment`]). Mutants: the - /// refusal dropped, which leaves the tier's load to refuse the - /// layer's stacks; the body's plan rule left on the common leg's, which - /// keeps the NVFP4 layer on the host, off the stage card's planes. + /// The tier reserves precisely the selected layers' down forms, + /// including a mixed 128/32-value load; a gate/up mismatch remains + /// host-only and cannot add a form. #[test] - fn a_tier_card_is_refused_a_layer_its_leg_does_not_run() { + fn tier_scratch_follows_selected_mixed_layers() { + use super::super::TIER_CHUNK; + use crate::placement::kernels::{self, DownAct, DownForms}; + use GgmlType::{IQ3_S, NVFP4, Q5_K, Q8_0}; let mut inputs = routed(64 << 30); - for t in inputs + inputs .model .tensors - .iter_mut() - .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(1)) - { - t.ty = GgmlType::NVFP4; - t.file_bytes = t.dims.iter().product::() / 64 * 36; - } - let plain = unified(64 << 30); - let mut machine = plain.clone(); - let mut tier = machine.cards[0].clone(); - tier.name = "remote".into(); - tier.head = false; - tier.layers = 0..0; - machine.tiers.push(tier); - match inputs.plan(&machine, CTX, &PlanLevers::default()) { - Err(PlaceError::TierLeg { layer, tys }) => { - assert_eq!((layer, tys), (1, [GgmlType::NVFP4; 3])); + .retain(|t| t.role != Role::RoutedExperts || t.layer.is_some_and(|l| l < 2)); + let set = |inputs: &mut PlanInputs, l, tys: [GgmlType; 3]| { + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match crate::placement::ExpertPart::of_name(&t.name) { + Some(crate::placement::ExpertPart::Gate) => 0, + Some(crate::placement::ExpertPart::Up) => 1, + Some(crate::placement::ExpertPart::Down) => 2, + None => unreachable!(), + }; + t.ty = tys[part]; } - other => panic!("{:?}", other.map(|p| p.n_l.clone())), + }; + for (layers, forms) in [ + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q5_K]], + DownForms::of(DownAct::Q8Act), + ), + ( + [[Q5_K, Q5_K, Q8_0], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [NVFP4; 3]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [IQ3_S, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act), + ), + ] { + for (l, tys) in layers.into_iter().enumerate() { + set(&mut inputs, l, tys); + } + let bytes = super::super::tier_batch(&inputs).expect("tier bytes"); + let hp = &inputs.hp; + assert_eq!( + bytes.scratch, + kernels::leg_bytes( + TIER_CHUNK, + hp.n_used as u64, + hp.n_embd as u64, + hp.expert_ff as u64, + forms, + ), + "{layers:?}", + ); } - let p = inputs - .plan(&plain, CTX, &PlanLevers::default()) - .expect("the plan without the tier card"); - assert!(p.n_l[1] > 0, "the NVFP4 layer on the card: {:?}", p.n_l); } #[test] diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 14ad7aa30..be65dc527 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -120,13 +120,14 @@ pub const TIER_CHUNK: u64 = 16; /// and places ([`workstation::tier_leg_batch_bytes`]). The plan reserves them on the tier card and /// the host, and the load checks its allocations against them. #[must_use] -pub fn tier_batch(hp: &Hparams, cols: u64) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams, cols: u64) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, TIER_CHUNK, cols, + DownForms::of(DownAct::Q8Act), ) } @@ -274,10 +275,11 @@ mod tests { hp.n_used as u64, TIER_CHUNK, COLS, + DownForms::of(DownAct::Q8Act), ) ); let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); - let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS)); + let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS).expect("tier bytes")); let plain = workstation::plan_gate(hp.n_trunk); assert_eq!( inputs.prompt_batch_bytes(&plain).expect("a plain machine"), diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 6cf405c84..64b6a8407 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -135,13 +135,14 @@ pub const TIER_CHUNK: u64 = 16; /// and places. The plan reserves them on the tier card and the host, and the load checks its /// allocations against them. #[must_use] -pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result { let hp = &inputs.hp; - workstation::tier_leg_batch_bytes_of( + workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, - (TIER_CHUNK, cols), + TIER_CHUNK, + cols, leg_downs(inputs), ) } @@ -441,9 +442,12 @@ mod tests { hp.experts.n_used as u64, TIER_CHUNK, COLS, + DownForms::of(DownAct::Q8Act), ) ); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs, COLS)); + let tiered = workstation::plan_bp( + LAYERS, None, tier_batch(&inputs, COLS).expect("tier bytes"), + ); let plain = workstation::plan_gate(LAYERS); let places = 3 * 4 * 256 * 2; assert_eq!(inputs.tier_reserve, Some(places)); diff --git a/crates/model/src/arch/musespark/place.rs b/crates/model/src/arch/musespark/place.rs index 01572dc42..de91cf878 100644 --- a/crates/model/src/arch/musespark/place.rs +++ b/crates/model/src/arch/musespark/place.rs @@ -77,7 +77,7 @@ pub const fn pass_rows_max(n_used: usize) -> usize { /// the tier's rows and its staging are the latent's width, through the card /// leg in chunks of [`TIER_CHUNK`]. #[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { let ff = hp.expert_width(); workstation::tier_leg_batch_bytes( hp.latent as u64, @@ -85,6 +85,7 @@ pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { hp.n_used as u64, TIER_CHUNK, pass_rows_max(hp.n_used) as u64, + crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act), ) } diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index a5c0bc72b..f4bd99284 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1472,13 +1472,15 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. #[must_use] -pub fn leg_tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn leg_tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, LEG_TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act) + .with(crate::placement::kernels::DownAct::Q8Blocks32), ) } @@ -1512,6 +1514,7 @@ const fn tier_batch_of(n_embd: u64, n_used: u64, ubatch: u64) -> TierBatchBytes staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_route_scratch_bytes(cols), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -2337,6 +2340,7 @@ fn draft_machine() -> Machine { Machine { cards: vec![Card { name: "mtp draft".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -2808,6 +2812,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, @@ -3578,6 +3583,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 0, @@ -4492,6 +4498,7 @@ mod tests { staging: 496_730_112, scratch: 348_980_248, host: 839_188_480, + leg: None, } ); assert_eq!( @@ -4500,6 +4507,7 @@ mod tests { staging: 62_091_264, scratch: 87_233_816, host: 104_898_560, + leg: None, } ); } diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 3c71a02d2..4926c5462 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -46,13 +46,15 @@ pub const TIER_CHUNK: u64 = 8; /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. #[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act) + .with(crate::placement::kernels::DownAct::Q8Blocks32), ) } diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 95f0e5f15..90e91e2dc 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -14,9 +14,10 @@ use super::hparams::Hparams; use super::{roles, spec}; use crate::arch::chat_of; use crate::arch::place_hybrid; +use crate::placement::accepts::{Program, leg_down_forms}; use crate::placement::kernels::q8_act_bytes; use crate::placement::workstation::TierBatchBytes; -use crate::placement::{PlacementError, workstation}; +use crate::placement::{ModelTensors, PlacementError, workstation}; pub use place_hybrid::{GEMM_TILE_COLS, GEMM32_STEP, KvLayout, PlaceError}; use place_hybrid::{route_bytes, tier_places_bytes, wide_bytes}; @@ -40,18 +41,19 @@ pub const PARTIAL_ROWS: usize = 8; /// the stage card's card experts, which the body binds to its own constant. pub const TIER_CHUNK: u64 = 8; -/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a file of hyperparameters `hp`: +/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a described file: /// the tier card's staging, the leg's chunk scratch and the host's rows and places, for blocks of /// the host union's columns. The plan reserves them on the tier card and the host, and the load /// checks its allocations against them. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + leg_down_forms(Program::Solar2Body, &inputs.model), ) } @@ -89,8 +91,12 @@ const fn acts_bytes(k: usize) -> usize { /// The wide forms are a bound: both forms at each width that allows them, where the load makes /// the ones its sites' types read. The units a prefill group past one adds are made out of the /// card's margin. -#[must_use] -pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 { +pub fn prompt_reserve_bytes( + hp: &Hparams, + spec: &ModelSpec, + model: &ModelTensors, + rows: usize, +) -> Result { let (embd, r) = (hp.n_embd, rows); let (mut heads, mut kv_heads, mut head) = (hp.n_head, 0, hp.head_dim); let (mut k_heads, mut v_heads, mut d) = (0, 0, 0); @@ -124,13 +130,21 @@ pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 let wide: usize = others.iter().map(|&k| wide_bytes(r, k)).sum::() + 2 * wide_bytes(r, embd) + wide_bytes(r, shared_ff); - let card_leg = workstation::tier_chunk_scratch_bytes( - embd as u64, - experts.ff as u64, - experts.n_used as u64, - r as u64, - ) as usize; - (4 * f32s + acts + wide + 2 * route_bytes(r) + card_leg) as u64 + let downs = leg_down_forms(Program::Solar2Body, model); + let card_leg = workstation::TierLegScratch { + n_embd: embd as u64, + ff: experts.ff as u64, + n_used: experts.n_used as u64, + chunk: r as u64, + downs, + } + .bytes(downs)?; + ((4 * f32s + acts + wide + 2 * route_bytes(r)) as u64) + .checked_add(card_leg) + .ok_or_else(|| PlacementError::Metadata { + key: "solar2 prompt reserve".into(), + detail: "bytes overflow u64".into(), + }) } impl PlanInputs { @@ -147,7 +161,7 @@ impl PlanInputs { let model = roles::classify(split, &hp)?; // No parser is declared: see `spec`. let spec = spec::spec_of(&hp, chat_of(split, None, None)?)?; - let reserve = prompt_reserve_bytes(&hp, &spec, PROMPT_ROWS); + let reserve = prompt_reserve_bytes(&hp, &spec, &model, PROMPT_ROWS)?; let tier = tier_places_bytes(PROMPT_ROWS, hp.experts.n_used); Ok(PlanInputs::new(ENGINE, hp, model, spec)? .with_prompt_reserve(reserve) @@ -247,7 +261,7 @@ mod tests { #[test] fn a_tier_card_is_planned_with_the_batch_places_beside_the_reserve() { let inputs = described("so2-place-tier"); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs.hp)); + let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs).expect("tier bytes")); let plain = workstation::plan_gate(LAYERS); let places = 12 * 512 * inputs.hp.experts.n_used as u64; assert_eq!(inputs.tier_reserve, Some(places)); @@ -305,11 +319,108 @@ mod tests { /// expert. Mutant: the reserve left out of the plan's call (the card plans, and its load stops /// at the allocation). #[test] - fn a_card_short_of_the_prompt_reserve_is_refused_by_name() { - let inputs = described("so2-place-reserve"); - let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, PROMPT_ROWS); - assert!(reserve > 0, "the synthetic header's batch has bytes"); - assert_eq!(inputs.prompt_reserve, reserve, "describe keeps the figure"); + fn a_host_only_mixture_keeps_the_exact_prompt_reserve_boundary() { + use crate::placement::{ExpertPart, Role}; + use gguf::GgmlType::{Q5_K, Q8_0}; + + let mut kv = keys(); + for (key, value) in &mut kv { + if matches!(*key, "embedding_length" | "expert_feed_forward_length") { + *value = V::U32(256); + } + } + let mut ts = tensors(); + for (name, dims) in &mut ts { + let tail = name.rsplit('.').nth(1).unwrap_or(""); + if matches!( + tail, + "ffn_gate_exps" + | "ffn_up_exps" + | "ffn_down_exps" + | "ffn_gate_shexp" + | "ffn_up_shexp" + | "ffn_down_shexp" + ) { + dims[0] = 256; + dims[1] = 256; + } else if tail == "attn_output" { + dims[1] = 256; + } else if matches!( + tail, + "token_embd" + | "output" + | "output_norm" + | "attn_norm" + | "ffn_norm" + | "attn_q" + | "attn_k" + | "attn_v" + | "attn_gate" + | "ssm_f_a" + | "ssm_g_a" + | "ssm_beta" + | "ffn_gate_inp" + ) { + dims[0] = 256; + } + } + let global = [ + ("tokenizer.ggml.pre", V::Str("solar-open")), + ("tokenizer.chat_template", V::Str("{{ messages }}")), + ]; + let path = header_shaped("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); + let split = Split::open(&path).expect("the mixed fixture opens"); + let mut inputs = PlanInputs::describe(&split).expect("valid routed geometry"); + std::fs::remove_file(path).expect("remove the header fixture"); + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.ty = Q5_K; + t.file_bytes = t.dims.iter().product::() / t.ty.blck_size().unwrap() + * t.ty.type_size().unwrap(); + } + let admitted = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("the admitted prompt reserve"); + let admitted_tier = tier_batch(&inputs).expect("the admitted tier reserve"); + let down = inputs + .model + .tensors + .iter_mut() + .find(|t| { + t.role == Role::RoutedExperts + && t.layer == Some(0) + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + }) + .expect("layer zero's down"); + down.ty = Q8_0; + down.file_bytes = down.dims.iter().product::() / down.ty.blck_size().unwrap() + * down.ty.type_size().unwrap(); + let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("prompt reserve"); + assert_eq!( + reserve, admitted, + "the host-only Q8_0 down adds no card form" + ); + assert_eq!( + tier_batch(&inputs).expect("mixed tier reserve"), + admitted_tier + ); + inputs.prompt_reserve = reserve; + let ample = workstation::plan_gate(LAYERS); + let placed = inputs + .plan(&le, 64, &PlanLevers::default()) + .expect("the mixed plan"); + assert_eq!( + placed.n_l[0], 0, + "Q8_0 stays on the host under Solar's rule" + ); + assert!( + placed.n_l[1..].iter().all(|&n| n > 0), + "other layers remain card candidates" + ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); machine.cards[0].usable_bytes = machine.cards[0].margin_bytes; @@ -346,10 +457,6 @@ mod tests { let plan = inputs .plan(&machine, 64, &levers) .expect("the card holds the terms and the reserve"); - assert_eq!( - plan.n_l.iter().sum::(), - 0, - "no room is left for an expert" - ); + assert!(plan.violations().is_empty()); } } diff --git a/crates/model/tests/placement.rs b/crates/model/tests/placement.rs index c62b0921c..394803f57 100644 --- a/crates/model/tests/placement.rs +++ b/crates/model/tests/placement.rs @@ -1371,6 +1371,7 @@ fn layered_model(layers: usize) -> ModelTensors { fn byte_card(name: &str, usable: u64, layers: Range) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, diff --git a/crates/model/tests/qwen4exp_mtp_reserve.rs b/crates/model/tests/qwen4exp_mtp_reserve.rs index 09eade30e..60c922e5c 100644 --- a/crates/model/tests/qwen4exp_mtp_reserve.rs +++ b/crates/model/tests/qwen4exp_mtp_reserve.rs @@ -298,6 +298,7 @@ fn machine(usable: u64) -> Machine { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: CONTEXT, diff --git a/crates/model/tests/solar_open2_meta.rs b/crates/model/tests/solar_open2_meta.rs index 0b55bb2ad..730f0a28b 100644 --- a/crates/model/tests/solar_open2_meta.rs +++ b/crates/model/tests/solar_open2_meta.rs @@ -398,6 +398,7 @@ fn hw_solar_open2_plan_a() { staging: 0, scratch: 0, host: 0, + leg: None, }; let bp = workstation::plan_bp(layers, None, tier_batch); let tiered = inputs diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index bb3f77110..daa05c3ec 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -449,6 +449,43 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { leg_arms(p).map(|a| gate == up && (a.gate_up)(gate) && (a.down)(down)) } +/// The union of down activation forms of the complete routed layers that +/// `p`'s placement rule admits, resolved from the file's descriptor at load. +#[must_use] +pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> kernels::DownForms { + use crate::placement::{ExpertPart, Role}; + + let RoutedRule::Leg { gate_up, down } = routed_rule(p) else { + return kernels::DownForms::NONE; + }; + let mut forms = kernels::DownForms::NONE; + for l in 0..model.layers { + let mut parts = [None; 3]; + for t in model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match ExpertPart::of_name(&t.name) { + Some(ExpertPart::Gate) => 0, + Some(ExpertPart::Up) => 1, + Some(ExpertPart::Down) => 2, + None => continue, + }; + parts[part] = Some(t.ty); + } + if let [Some(g), Some(u), Some(d)] = parts + && g == u + && gate_up(g).is_some() + && down(d).is_some() + && let Some((_, act)) = kernels::down(d) + { + forms = forms.with(act); + } + } + forms +} + /// Program `p`'s plan rule for its routed stacks ([`RoutedRule`]): on a card /// leg, each part by the arm its load reads ([`leg_runs`]), so a layer the /// plan gives card experts is one the load runs. The mimo2 and solar2 diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 7ab1361b8..61778ae70 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -100,12 +100,30 @@ pub const fn embed(ty: GgmlType) -> Option { /// it holds; a form whose last super-block is padded passes the padded width. #[must_use] pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { - let n_sb = k / 256; - 8 * cols * 64 * n_sb.div_ceil(2) - + 4 * cols * 256 * n_sb.div_ceil(4) - + 4 * cols * 128 * n_sb.div_ceil(2) - + 4 * cols * 8 * n_sb - + 4 * cols * 2 * n_sb + match try_q8_act_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_act_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_act_bytes`] with overflow returned to the placement boundary. +#[must_use] +pub(crate) const fn try_q8_act_bytes(cols: u64, k: u64) -> Option { + let n_sb = (k / 256) as u128; + // A u64 width's per-column count fits u128; its column product may not. + let per_col = 512 * n_sb.div_ceil(2) + + 1024 * n_sb.div_ceil(4) + + 512 * n_sb.div_ceil(2) + + 32 * n_sb + + 8 * n_sb; + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The card bytes of the 32-value q8_1 activation of `cols` columns of `k` @@ -114,8 +132,25 @@ pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { /// and scales, four bytes each, `k_blocks = k / 32`. #[must_use] pub const fn q8_blocks32_bytes(cols: u64, k: u64) -> u64 { - let k_blocks = k / 32; - cols * 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks) + match try_q8_blocks32_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_blocks32_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_blocks32_bytes`] with overflow returned to the placement boundary. +#[must_use] +const fn try_q8_blocks32_bytes(cols: u64, k: u64) -> Option { + let k_blocks = (k / 32) as u128; + let per_col = 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks); + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The down activation forms ([`DownAct`]) a card leg's buffers hold: the @@ -174,30 +209,68 @@ impl DownForms { /// them read it. #[must_use] pub const fn leg_bytes(cap: u64, slots_per_col: u64, n: u64, ff: u64, downs: DownForms) -> u64 { - let slots = cap * slots_per_col; + match try_leg_bytes(cap, slots_per_col, n, ff, downs) { + Some(bytes) => bytes, + None => panic!("leg_bytes: scratch bytes overflow u64"), + } +} + +/// [`leg_bytes`] with overflow returned to the placement boundary. This +/// counts the same whole blocks and padded planes as the infallible sizing. +#[must_use] +pub(crate) const fn try_leg_bytes( + cap: u64, + slots_per_col: u64, + n: u64, + ff: u64, + downs: DownForms, +) -> Option { + let Some(slots) = cap.checked_mul(slots_per_col) else { + return None; + }; + let Some(row_bytes) = slots.checked_mul(4) else { + return None; + }; + let Some(h) = row_bytes.checked_mul(ff) else { + return None; + }; + let Some(down) = row_bytes.checked_mul(n) else { + return None; + }; + let Some(input) = try_q8_act_bytes(cap, n) else { + return None; + }; let q8 = if downs.q8_act { - q8_act_bytes(slots, ff) + let Some(bytes) = try_q8_act_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; let b32 = if downs.blocks32 { - q8_blocks32_bytes(slots, ff) + let Some(bytes) = try_q8_blocks32_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; - q8_act_bytes(cap, n) + 4 * slots * ff + q8 + b32 + 4 * slots * n + let bytes = input as u128 + h as u128 + q8 as u128 + b32 as u128 + down as u128; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// Whether the common card leg (`crates/gpu/src/card_leg.rs`) launches the /// gate·up entry of a routed gate or up stack of type `ty` in the file's /// blocks ([`crate::placement::CardFormat::KQuant`]): the table's [`gate_up`] /// row, for the entries whose launchers read a `Q8Act` of more columns than a -/// run uses. The Q8_0 row has a launcher of that shape and an arm in the leg -/// (`CardLeg::launch`), which this set leaves out: the families that plan by -/// it (GLM-5.3, MiMo, Solar-Open2, MiniMax-M3, Raon) hold `Q8Act` down buffers -/// only, so a Q8_0 layer they planned on the card would fail at its first -/// launch where it runs on the host today; a program whose buffers follow the -/// down's form reads the arm through its own row ([`crate::accepts::prompt_leg_gate_up`]). +/// run uses. The Q8_0 and Q6_K rows have launchers of that shape and arms in the +/// leg, but only programs whose selected gate·up policy admits them read those +/// arms ([`crate::accepts::prompt_leg_gate_up`]). /// A stack in its planes is [`card_planes_runs`]'. #[must_use] pub const fn card_gate_up_runs(ty: GgmlType) -> bool { @@ -324,6 +397,54 @@ mod tests { use super::*; use crate::placement::CardFormat; + /// Plane-padding transitions retain their byte counts, and both sizing + /// contracts share the last representable column and aggregate boundary. + #[test] + fn activation_and_leg_bytes_keep_padding_and_overflow_boundaries() { + for (k, bytes) in [(1024, 3232), (1280, 5320)] { + assert_eq!(try_q8_act_bytes(1, k), Some(bytes)); + assert_eq!(q8_act_bytes(1, k), bytes); + } + for (k, bytes) in [(1024, 1280), (1056, 2312)] { + assert_eq!(try_q8_blocks32_bytes(1, k), Some(bytes)); + assert_eq!(q8_blocks32_bytes(1, k), bytes); + } + let q8 = DownForms::of(DownAct::Q8Act); + let b32 = DownForms::of(DownAct::Q8Blocks32); + for (forms, bytes) in [ + (DownForms::NONE, 4136), + (q8, 6224), + (b32, 5224), + (q8.with(DownAct::Q8Blocks32), 7312), + ] { + assert_eq!(try_leg_bytes(1, 1, 256, 256, forms), Some(bytes)); + assert_eq!(leg_bytes(1, 1, 256, 256, forms), bytes); + } + assert_eq!(try_leg_bytes(1, 1, 64, 24, DownForms::NONE), Some(352)); + let act_cols = u64::MAX / 2088; + assert_eq!(try_q8_act_bytes(act_cols, 256), Some(act_cols * 2088)); + assert_eq!(q8_act_bytes(act_cols, 256), act_cols * 2088); + assert_eq!(try_q8_act_bytes(act_cols + 1, 256), None); + let block_cols = u64::MAX / 1088; + assert_eq!( + try_q8_blocks32_bytes(block_cols, 256), + Some(block_cols * 1088) + ); + assert_eq!(q8_blocks32_bytes(block_cols, 256), block_cols * 1088); + assert_eq!(try_q8_blocks32_bytes(block_cols + 1, 256), None); + let cap = u64::MAX / 6224; + assert_eq!(try_leg_bytes(cap, 1, 256, 256, q8), Some(cap * 6224)); + assert_eq!(leg_bytes(cap, 1, 256, 256, q8), cap * 6224); + assert_eq!(try_leg_bytes(cap + 1, 1, 256, 256, q8), None); + assert_eq!(try_leg_bytes(1 << 61, 8, 256, 256, q8), None); + } + + #[test] + #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] + fn infallible_leg_sizing_names_an_overflow() { + let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); + } + /// Every `GgmlType` variant in declaration order, then one id the enum /// does not name. const ALL: [GgmlType; 42] = [ diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index db5f8416a..2500304ac 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -249,11 +249,10 @@ pub struct Card { /// draft model's resident bytes, say): taken from the budget with the /// context and the scratch ([`Card::set_aside_bytes`]). pub reserves: Vec<(String, u64)>, - /// A tier card with an expert cap of 0: it holds its reserves and no - /// expert, and serves no tier. The plan refuses one with no reserve - /// ([`PlacementError::IdleTier`]); a body hangs a tier only under the - /// tier cards that are not reserve-only ([`Machine::expert_tiers`]). + /// A tier card holding only reserves, not experts. pub reserve_only: bool, + /// A tier leg's scratch, jointly selected with its resident experts. + pub tier_leg: Option, } impl Card { @@ -275,10 +274,55 @@ impl Card { .try_fold(0u64, |sum, (_, bytes)| sum.checked_add(*bytes)) } - /// What the card sets aside before any tensor or cache: context, - /// scratch and reserves. The one owner of that sum: the expert rule's - /// budget, the budget floor, the headroom and [`Plan::violations`] all - /// take it from here. The margin is not in it. `None` past u64 bytes. + /// The tier leg's bound and selected scratch; absent on fixed-reserve paths. + fn selected_scratch( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result, PlacementError> { + let Some(leg) = self.tier_leg else { + return Ok(None); + }; + let forms = model + .tensors + .iter() + .filter(|t| { + t.role == Role::RoutedExperts + && t.layer + .is_some_and(|l| counts.get(l).is_some_and(|&n| n > 0)) + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + }) + .filter_map(|t| crate::kernels::down(t.ty)) + .fold(crate::kernels::DownForms::NONE, |forms, (_, act)| { + forms.with(act) + }); + Ok(Some((leg.bytes(leg.downs)?, leg.bytes(forms)?))) + } + + fn selected_reserve_bytes( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result { + let fixed = self.reserve_bytes().ok_or_else(|| self.bytes_past_u64())?; + match self.selected_scratch(model, counts)? { + Some((bound, selected)) => fixed + .checked_sub(bound) + .and_then(|n| n.checked_add(selected)) + .ok_or_else(|| self.bytes_past_u64()), + None => Ok(fixed), + } + } + + fn floor_with_reserves(&self, dense: u64, kv: u64, past: u64, reserves: u64) -> Option { + [kv, self.context_bytes, self.scratch_bytes, reserves, past] + .into_iter() + .try_fold(dense, u64::checked_add) + } + + /// The machine's context, scratch and reserve bound before uploads. + /// Placement replaces a tier leg's bound with its candidate's actual + /// scratch. The margin is not in it. `None` past u64 bytes. #[must_use] pub fn set_aside_bytes(&self) -> Option { self.context_bytes @@ -286,18 +330,13 @@ impl Card { .checked_add(self.reserve_bytes()?) } - /// The card's floor under `dense` bytes of uploads and `kv` bytes of - /// cache: those, its set-aside, and `past` — what the load keeps beside - /// them: a plan's margin and what its caller holds beside the plan - /// (`plan_floor` sums those), or a whole load's own arena and reserve - /// ([`WholeLoad::past_bytes`]). The one owner of that sum: the budget - /// floor, the free floor and the whole fit ([`whole_need`]) all take it - /// from here. `None` past u64 bytes. + /// The fixed/bounded floor under `dense` upload bytes, `kv` cache bytes + /// and `past` bytes held beside them ([`WholeLoad::past_bytes`]). + /// Selected plans use the same sum with their actual reserves. + /// `None` past u64 bytes. #[must_use] pub fn floor_bytes(&self, dense: u64, kv: u64, past: u64) -> Option { - [kv, self.set_aside_bytes()?, past] - .into_iter() - .try_fold(dense, u64::checked_add) + self.floor_with_reserves(dense, kv, past, self.reserve_bytes()?) } fn bytes_past_u64(&self) -> PlacementError { @@ -1010,7 +1049,8 @@ pub struct CardTotals { pub stage_rows_bytes: u64, pub scratch_bytes: u64, pub context_bytes: u64, - /// The card's reserves ([`Card::reserves`]). + /// The selected reserves; a tier leg's descriptor bound is replaced by + /// its actual resident layers' scratch. pub reserve_bytes: u64, /// usable − dense − experts − rounding − KV − stage rows − scratch − /// context − reserves, usable capped by the plan's card budget; the @@ -2879,6 +2919,40 @@ impl Fill<'_, '_> { .routed .unwrap_or(RoutedRule::PerType(CardFormat::of_routed)); let uploads = card_uploads(self.model, self.c, self.rows, self.eligible, format)?; + let mut down_acts = if card.tier_leg.is_some() { + vec![None; self.model.layers] + } else { + Vec::new() + }; + if card.tier_leg.is_some() { + for t in &self.model.tensors { + if t.role == Role::RoutedExperts + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + && let Some(l) = t.layer + && let Some((_, act)) = crate::kernels::down(t.ty) + { + down_acts[l] = Some(act); + } + } + } + let scratch = |counts: &[u64]| { + card.tier_leg.map_or(Ok(0), |leg| { + let forms = counts + .iter() + .zip(&down_acts) + .filter_map(|(&n, &act)| (n > 0).then_some(act).flatten()) + .fold(crate::kernels::DownForms::NONE, |forms, act| { + forms.with(act) + }); + leg.bytes(forms) + }) + }; + let bound = card + .tier_leg + .map(|leg| leg.bytes(leg.downs)) + .transpose()? + .unwrap_or(0); + let reserves = card.selected_reserve_bytes(self.model, n_l)?; let budget = i128::from(capped(card, self.card_budget)) - i128::from(self.kv) - i128::from( @@ -2886,20 +2960,23 @@ impl Fill<'_, '_> { .ok_or_else(|| card.bytes_past_u64())?, ) - i128::from(card.margin_bytes) - - i128::from(self.reserve); + - i128::from(self.reserve) + + i128::from(bound); if let Some(b) = self.card_budget { let dense = footprint(card.granule_bytes, &uploads, n_l)?; - check_floor(card, b, dense, self.kv, self.reserve)?; + check_floor(card, b, dense, self.kv, self.reserve, reserves)?; } if let Some(free) = card.free_bytes.filter(|&f| f < card.usable_bytes) { let dense = footprint(card.granule_bytes, &uploads, n_l)?; - check_free_floor(card, free, dense, self.kv, self.reserve)?; + check_free_floor(card, free, dense, self.kv, self.reserve, reserves)?; } let caps: Vec = (0..n_l.len()) .map(|l| self.model.layer_experts(l)) .collect(); spread(n_l, held, self.eligible, &caps, budget, |n| { - footprint(card.granule_bytes, &uploads, n) + footprint(card.granule_bytes, &uploads, n)? + .checked_add(scratch(n)?) + .ok_or_else(|| card.bytes_past_u64()) }) } } @@ -2909,13 +2986,11 @@ impl Fill<'_, '_> { /// those as its `past`. Both floor checks ([`check_floor`], /// [`check_free_floor`]) and the share split's floors ([`resolve_shares`]) /// take it from here, so the floor they name is the least budget at which -/// the expert rule's own budget (usable less KV, set-aside, margin and -/// `beside`, in [`Fill::run`]) reaches the dense uploads. `None` past u64 -/// bytes. -fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64) -> Option { +/// the expert rule's own budget reaches the dense uploads. `None` past u64 bytes. +fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64, reserves: u64) -> Option { card.margin_bytes .checked_add(beside) - .and_then(|past| card.floor_bytes(dense, kv, past)) + .and_then(|past| card.floor_with_reserves(dense, kv, past, reserves)) } /// Refuse `card` under the card budget `budget` when its floor — the @@ -2930,8 +3005,9 @@ fn check_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { - let floor = plan_floor(card, dense, kv, beside); + let floor = plan_floor(card, dense, kv, beside, reserves); match floor { Some(floor) if floor <= budget => Ok(()), _ => Err(PlacementError::CardBudgetFloor { @@ -2942,7 +3018,7 @@ fn check_floor( kv, context: card.context_bytes, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, }), @@ -2961,9 +3037,10 @@ fn check_free_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { let context = card.context_bytes - card.context_in_free(); - let floor = plan_floor(card, dense, kv, beside).map(|f| f - card.context_in_free()); + let floor = plan_floor(card, dense, kv, beside, reserves).map(|f| f - card.context_in_free()); match floor { Some(floor) if floor <= free => Ok(()), floor => Err(PlacementError::CardFreeFloor(Box::new(CardFreeFloor { @@ -2976,7 +3053,7 @@ fn check_free_floor( kv, context, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, held: card @@ -4059,8 +4136,12 @@ fn totals<'a>( .map(|(c, (card, heap))| { let rounding = heap.taken - (card_dense[c] + card_experts[c]); let (kv, _, stage_rows) = kv_bytes[c]; + let counts = c + .checked_sub(machine.cards.len()) + .map_or(n_l.as_slice(), |t| tier_n_l[t].as_slice()); + let reserve_bytes = card.selected_reserve_bytes(model, counts)?; let used = card - .floor_bytes(heap.taken, kv, stage_rows) + .floor_with_reserves(heap.taken, kv, stage_rows, reserve_bytes) .ok_or_else(|| card.bytes_past_u64())?; if machine.unified().is_some() && machine.on_this_node(c) @@ -4080,9 +4161,7 @@ fn totals<'a>( stage_rows_bytes: stage_rows, scratch_bytes: card.scratch_bytes, context_bytes: card.context_bytes, - reserve_bytes: card - .reserve_bytes() - .expect("the card floor checked its reserves"), + reserve_bytes, headroom_bytes: i128::from(capped(card, card_budget::of(&card_budget, card))) - i128::from(used), }) @@ -4152,11 +4231,50 @@ pub struct PoolNeed { /// `card`'s total against usable − margin: its uploads' granules, its KV, /// its stage rows and its set-aside. fn card_total(card: &Card, totals: &CardTotals, heap: &Heap) -> u64 { - card.floor_bytes(heap.taken, totals.kv_bytes, totals.stage_rows_bytes) - .expect("the plan's card totals fit u64") + card.floor_with_reserves( + heap.taken, + totals.kv_bytes, + totals.stage_rows_bytes, + totals.reserve_bytes, + ) + .expect("the plan's card totals fit u64") } impl Plan<'_> { + /// One named reserve at its selected amount, not the machine's scratch + /// bound. Missing or duplicate rows and an invalid card return `None`; + /// invalid scratch descriptors propagate their named error. + pub fn card_reserve(&self, c: usize, name: &str) -> Result, PlacementError> { + let Some(card) = self.machine.card(c) else { + return Ok(None); + }; + let mut rows = card.reserves.iter().filter(|(n, _)| n == name); + let Some((_, bytes)) = rows.next() else { + return Ok(None); + }; + if rows.next().is_some() { + return Ok(None); + } + if name != workstation::TIER_BATCH_RESERVE { + return Ok(Some(*bytes)); + } + let counts = match c.checked_sub(self.machine.cards.len()) { + None => self.n_l.as_slice(), + Some(t) => match self.tier_n_l.get(t) { + Some(counts) => counts.as_slice(), + None => return Ok(None), + }, + }; + match card.selected_scratch(self.model, counts)? { + Some((bound, selected)) => bytes + .checked_sub(bound) + .and_then(|bytes| bytes.checked_add(selected)) + .map(Some) + .ok_or_else(|| card.bytes_past_u64()), + None => Ok(Some(*bytes)), + } + } + fn reserve_beside(&mut self, reserve: u64) -> Result<(), PlacementError> { if self.machine.unified().is_some() { let here = (0..self.machine.all_cards().count()) @@ -4781,6 +4899,7 @@ mod tests { fn card(name: &str, layers: Range, head: bool, token_embedding: bool) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: 0, context_bytes: 0, @@ -5952,6 +6071,169 @@ mod tests { assert_eq!(plan.n_l, vec![8]); } + /// Scratch enters each expert candidate's footprint monotonically. + /// A boundary that admits one down form cannot pay for an unused second + /// form, and adding the first expert of that form pays its exact bytes. + #[test] + fn a_tier_jointly_selects_expert_rows_and_down_forms() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use GgmlType::{Q5_K, Q8_0}; + use workstation::{TIER_BATCH_RESERVE, tier_leg_batch_bytes}; + + let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + let batch = tier_leg_batch_bytes(256, 256, 1, 2, 3, both).expect("tier bytes"); + let leg = batch.leg.expect("a tier leg descriptor"); + let rule = routed_rule(Program::Qwen35Body); + for (downs, first) in [ + ([Q5_K, Q8_0], DownAct::Q8Act), + ([Q8_0, Q5_K], DownAct::Q8Blocks32), + ] { + let mut model = legged(&[[Q5_K, Q5_K, downs[0]], [Q5_K, Q5_K, downs[1]]]); + model.experts = 1; + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.dims[2] = 1; + t.file_bytes *= 8; + } + let expert = |l| { + model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + .map(|t| t.file_bytes / model.experts) + .sum::() + }; + let one = + batch.staging + leg.bytes(DownForms::of(first)).expect("first scratch") + expert(0); + let two = + batch.staging + leg.bytes(both).expect("mixed scratch") + expert(0) + expert(1); + for (room, counts, forms) in [ + (one - 1, [0, 0], DownForms::NONE), + (one, [1, 0], DownForms::of(first)), + (two - 1, [1, 0], DownForms::of(first)), + (two, [1, 1], both), + ] { + let mut tier = bytes_card("tier", room, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = batch.leg; + let machine = Machine { + cards: vec![bytes_card("stage", HEAD, 0..2)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + }; + let plan = plan_routed(&model, &machine, 1, &NoKv, &PlanLevers::default(), rule) + .expect("the joint plan"); + assert_eq!(plan.n_l, [0, 0]); + assert_eq!(plan.tier_n_l[0], counts, "{downs:?}, {room}"); + let selected = batch.staging + leg.bytes(forms).expect("selected scratch"); + assert_eq!( + plan.card_reserve(1, TIER_BATCH_RESERVE) + .expect("selected reserve"), + Some(selected) + ); + assert_eq!(plan.cards[1].reserve_bytes, selected); + let rows = expert(0) * counts[0] + expert(1) * counts[1]; + assert_eq!(plan.cards[1].bytes(), selected + rows); + assert!(plan.violations().is_empty(), "{downs:?}, {room}"); + } + } + } + + /// A malformed descriptor must refuse a plan and a later selected-reserve + /// read, not wrap to a small scratch or masquerade as a fixed-reserve card. + #[test] + fn a_tier_refuses_invalid_scratch_descriptors_before_placement() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use workstation::{TIER_BATCH_RESERVE, TierLegScratch, tier_leg_batch_bytes}; + + let forms = DownForms::of(DownAct::Q8Act); + let batch = tier_leg_batch_bytes(256, 256, 1, 1, 1, forms).expect("valid batch"); + let good = batch.leg.expect("a leg descriptor"); + let mut model = legged(&[[GgmlType::Q5_K; 3]]); + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.file_bytes *= 64; + } + let machine = |leg| { + let mut tier = bytes_card("tier", 1 << 30, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = Some(leg); + Machine { + cards: vec![bytes_card("stage", HEAD, 0..1)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + } + }; + let valid = machine(good); + let rule = routed_rule(Program::Qwen35Body); + let plan = plan_routed(&model, &valid, 1, &NoKv, &PlanLevers::default(), rule) + .expect("valid descriptor plans"); + for leg in [ + TierLegScratch { + chunk: 1 << 61, + n_used: 8, + ..good + }, + TierLegScratch { chunk: 0, ..good }, + TierLegScratch { ff: 0, ..good }, + TierLegScratch { n_embd: 0, ..good }, + ] { + let bad = machine(leg); + assert!( + matches!( + plan_routed(&model, &bad, 1, &NoKv, &PlanLevers::default(), rule), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + let mut stale = plan.clone(); + stale.machine = &bad; + assert!( + matches!( + stale.card_reserve(1, TIER_BATCH_RESERVE), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + assert!( + matches!( + tier_leg_batch_bytes(leg.n_embd, leg.ff, leg.n_used, leg.chunk, 1, leg.downs), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + } + assert!(tier_leg_batch_bytes(256, 256, 8, 1, 1 << 61, forms).is_err()); + let inconsistent = workstation::TierBatchBytes { + scratch: 0, + ..batch + }; + assert!(matches!( + inconsistent.card(), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg batch" + )); + } + // PIN(2026-10-22): the Qwen3.6 leg runs a Q8_0 down and (once the leg has the // arm) a Q6_K gate and up through the common leg, so the layers that held those // types hold types it still has no GEMM for (an MXFP4 down, an IQ2_XS gate and diff --git a/crates/placement/src/placement/nodes_scale.rs b/crates/placement/src/placement/nodes_scale.rs index 94e8830fe..b85b290b9 100644 --- a/crates/placement/src/placement/nodes_scale.rs +++ b/crates/placement/src/placement/nodes_scale.rs @@ -283,6 +283,7 @@ fn census_budgets_keep_each_node_and_stage_terms() { staging: 123, scratch: 456, host: 789, + leg: None, }, ) .expect("three independent nodes"); @@ -339,6 +340,7 @@ fn cases() -> &'static [Case] { staging: 0, scratch: 0, host: 0, + leg: None, }, ) .expect("two nodes with no batch bytes"); diff --git a/crates/placement/src/placement/qwen38_cards.rs b/crates/placement/src/placement/qwen38_cards.rs index 8ef748a2e..c2b009c03 100644 --- a/crates/placement/src/placement/qwen38_cards.rs +++ b/crates/placement/src/placement/qwen38_cards.rs @@ -249,6 +249,7 @@ const CHECKPOINT_38: u64 = 36 * 3_596_288 + 17 * 4 * 2560 * 4; fn stage(spec: CardSpec) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -294,6 +295,7 @@ fn machine_bp(stage_spec: CardSpec, tier: CardSpec) -> Machine { cards: vec![s], tiers: vec![Card { name: tier.name.to_string(), + tier_leg: None, device: tier.device, usable_bytes: tier.usable_bytes(), context_bytes: CONTEXT, diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index eaf29c9b1..2f6f72386 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -19,7 +19,7 @@ use super::nodes::{Fabric, Nodes}; use super::{ Card, Device, Host, HostTotals, Machine, Node, PlacementError, Plan, Reserve, UnifiedPool, }; -use crate::kernels::{DownAct, DownForms, leg_bytes, q8_act_bytes}; +use crate::kernels::{DownForms, leg_bytes, q8_act_bytes, try_leg_bytes, try_q8_act_bytes}; pub const MIB: u64 = 1 << 20; @@ -182,6 +182,7 @@ pub fn model_v41() -> String { fn card(spec: CardSpec, layers: Range, head: bool) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -210,6 +211,7 @@ pub fn tier_card(spec: CardSpec, batch: TierBatchBytes) -> Card { .card() .unwrap_or_else(|error| panic!("tier batch card reserve: {error}")), )); + tier.tier_leg = batch.leg; tier } @@ -373,6 +375,7 @@ pub const NO_TIER_BATCH: TierBatchBytes = TierBatchBytes { staging: 0, scratch: 0, host: 0, + leg: None, }; /// What the expert tier's prompt-batch service is called as a reserve: on @@ -401,11 +404,21 @@ pub struct TierBatchBytes { pub staging: u64, pub scratch: u64, pub host: u64, + /// A common leg's candidate-dependent scratch; `scratch` is its bound. + pub leg: Option, } impl TierBatchBytes { - /// The tier card's part; refuses a sum past u64. + /// The tier card's part; refuses an invalid leg bound or a sum past u64. pub fn card(&self) -> Result { + if let Some(leg) = self.leg + && leg.bytes(leg.downs)? != self.scratch + { + return Err(PlacementError::Metadata { + key: "tier leg batch".into(), + detail: "scratch reserve differs from the descriptor's bound".into(), + }); + } self.staging .checked_add(self.scratch) .ok_or(PlacementError::TierBatchCard { @@ -415,6 +428,42 @@ impl TierBatchBytes { } } +/// The common leg's scratch geometry and the descriptor's admitted forms. +/// Placement prices its candidate's selected forms jointly with expert rows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TierLegScratch { + pub n_embd: u64, + pub ff: u64, + pub n_used: u64, + pub chunk: u64, + pub downs: DownForms, +} + +impl TierLegScratch { + /// Scratch for exactly these selected forms, refusing invalid geometry + /// and arithmetic overflow before an allocation or placement candidate. + pub fn bytes(self, downs: DownForms) -> Result { + let invalid = || PlacementError::Metadata { + key: "tier leg scratch".into(), + detail: format!("invalid or overflowing geometry: {self:?}, selected {downs:?}"), + }; + if self.chunk == 0 + || self.n_used == 0 + || self.ff == 0 + || self.n_embd == 0 + || [ + crate::kernels::DownAct::Q8Act, + crate::kernels::DownAct::Q8Blocks32, + ] + .into_iter() + .any(|act| downs.holds(act) && !self.downs.holds(act)) + { + return Err(invalid()); + } + try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs).ok_or_else(invalid) + } +} + /// Experts the tile path's bucket table takes a layer at most, and the /// columns a tile holds: the tile path's shapes, which size its scratch. const TILE_BUCKET_EXPERTS: u64 = 1024; @@ -474,12 +523,18 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// The tier's chunk scratch for blocks of rows of `n_embd`, `n_used` slots a /// token, through experts of `ff`, cut in chunks of `chunk` tokens: one run of -/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`, the GLM-5.3-Flash -/// tier) with its slots' columns in the 128-value q8_1 form the tier's downs -/// read ([`leg_bytes`]), in place of the tile path's scratch. +/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`) with its slots' +/// columns in exactly the forms the selected downs read ([`leg_bytes`]), +/// in place of the tile path's scratch. #[must_use] -pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { - leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) +pub const fn tier_chunk_scratch_bytes( + n_embd: u64, + ff: u64, + n_used: u64, + chunk: u64, + downs: DownForms, +) -> u64 { + leg_bytes(chunk, n_used, n_embd, ff, downs) } /// The host's part of the tier's prompt-batch service: per exchange set, the @@ -498,6 +553,7 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_block_scratch_bytes(n_embd, ff, cols * n_used), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -505,39 +561,48 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T /// scratch is one run of the leg's buffers for `chunk` tokens ([`tier_chunk_scratch_bytes`]), in /// place of the tile path's. The one formula of those bytes for every family whose tier is the card /// leg; a family names its own `chunk` and widths. -#[must_use] -pub const fn tier_leg_batch_bytes( +pub fn tier_leg_batch_bytes( n_embd: u64, ff: u64, n_used: u64, chunk: u64, cols: u64, -) -> TierBatchBytes { - tier_leg_batch_bytes_of( + downs: DownForms, +) -> Result { + let leg = TierLegScratch { n_embd, ff, n_used, - (chunk, cols), - DownForms::of(DownAct::Q8Act), - ) -} - -/// [`tier_leg_batch_bytes`] for a tier whose layers' downs read the forms `downs`: the leg's -/// buffers hold the slots' columns in each of them (`bloomery_gpu::card_leg::LegBufs::for_downs`, -/// [`leg_bytes`]), as an NVFP4 down's 32-value blocks. -#[must_use] -pub const fn tier_leg_batch_bytes_of( - n_embd: u64, - ff: u64, - n_used: u64, - (chunk, cols): (u64, u64), - downs: DownForms, -) -> TierBatchBytes { - TierBatchBytes { - staging: tier_batch_staging_bytes(n_embd, n_used, cols), - scratch: leg_bytes(chunk, n_used, n_embd, ff, downs), - host: tier_batch_host_bytes(n_embd, n_used, cols), - } + chunk, + downs, + }; + let scratch = leg.bytes(downs)?; + let counted = || -> Option<(u64, u64)> { + if cols == 0 { + return None; + } + let slots = cols.checked_mul(n_used)?; + let places = 4u64.checked_mul(slots)?; + let down = places.checked_mul(n_embd)?; + let staging = 4u64 + .checked_mul(cols)? + .checked_mul(n_embd)? + .checked_add(places)? + .checked_add(try_q8_act_bytes(cols, n_embd)?)? + .checked_add(down)?; + let host = 2u64.checked_mul(down.checked_add(places)?)?; + Some((staging, host)) + }; + let (staging, host) = counted().ok_or_else(|| PlacementError::Metadata { + key: "tier leg batch".into(), + detail: format!("invalid or overflowing staging: {leg:?}, columns {cols}"), + })?; + Ok(TierBatchBytes { + staging, + scratch, + host, + leg: Some(leg), + }) } /// Plan (b′): the A6000 runs all `layers` and the head as in [`plan_a`]; @@ -629,6 +694,7 @@ pub fn plan_node_tiers( remote.cards[0] .reserves .extend(tier_card(spec, batch).reserves); + remote.cards[0].tier_leg = batch.leg; machines.push(remote); } Ok(machines) @@ -811,6 +877,7 @@ pub(crate) fn spark_nodes(n: usize, layers: usize, fabric: Fabric) -> Nodes { staging: 0, scratch: 0, host: 0, + leg: None, }; let mut machines = plan_node_tiers(layers, spec, &vec![spec; n - 1], batch) .expect("synthetic nodes with no batch bytes"); @@ -1688,6 +1755,7 @@ mod tests { staging: 1, scratch: 2, host: 3, + leg: None, }, ]; for layers in [0, 1, 20, 43, 47, 48, 61, 78] { @@ -1843,6 +1911,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }; for result in [ plan_tiers(4, A6000, &[RTX_3090], None, huge).map(|_| ()), @@ -1893,6 +1962,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let machines = plan_node_tiers(4, specs[0], &specs[1..], batch) .expect("four nodes of the same card name and ordinal"); @@ -1937,6 +2007,7 @@ mod tests { staging: 0, scratch: 0, host: u64::MAX / 2, + leg: None, }; let remotes = [gb10(80 << 30), gb10(60 << 30)]; let machines = @@ -1972,6 +2043,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let m = plan_tiers(4, A6000, &[t1, t2], None, batch).expect("two tiers"); let rows: Vec<&(String, u64)> = m @@ -2052,6 +2124,7 @@ mod tests { assert_eq!(counted_arena_bytes(&m.cards[0]), 7, "{}", spec.name); let want = Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -2099,6 +2172,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }, ); } @@ -2118,6 +2192,7 @@ mod tests { staging: u64::MAX - 1, scratch: 1, host: 0, + leg: None, }; assert_eq!(edge.card().expect("the sum fits u64"), u64::MAX); assert!(matches!( diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 012ef946d..a5987a823 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -89,6 +89,23 @@ Qwen projection/shared dispatch does not gain this site automatically. Flint still has no Program, attention/router binding, body or serve seat. Run the new consumer gate and the affected regressions on NVIDIA before claiming device support; reconcile full-model coverage only with the actual body integration. +### Selected expert arms and tier scratch + +Common card and tier wrappers resolve `LegArms` from their selected `Program`, rather than assuming +every family has one gate/up type or one down activation form. Their resident layers determine the +union of `Q8Act` and `Q8Blocks32` buffers. Plane layouts remain restricted to the selected program's +arms; Solar's host-only triples do not enlarge its card scratch. + +Placement charges a tier's selected down forms jointly with its expert count. The same kernels-owned, +checked sizing arithmetic is used by allocation and planning; malformed or overflowing public scratch +descriptors are refused by name. The plan records the selected reserve, and loaders check that amount +instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver +is used. + +A host-only public planner consumer exercised the one-byte boundaries before the first resident layer +and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, +not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an +exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. From c592a640cbcd5d2118f5f0b0ccff96b757a2c32f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:12:33 +0900 Subject: [PATCH 12/85] placement: test overflow refusal without pinning panic wording --- crates/placement/src/kernels.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 61778ae70..f2d85d534 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -440,8 +440,8 @@ mod tests { } #[test] - #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] - fn infallible_leg_sizing_names_an_overflow() { + #[should_panic] + fn infallible_leg_sizing_refuses_overflow() { let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); } From 7f75867ff00910cdbbc63069a55537c464de60b2 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:24:24 +0900 Subject: [PATCH 13/85] gpu: preserve fixed reserves for host-only node projections --- crates/gpu/src/host/node.rs | 21 +++++++++++++++------ docs/research/flint-attach.md | 3 +++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index 15636de56..b966046af 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -321,14 +321,23 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let totals = plan.cards.get(self.card).ok_or_else(|| { + let card = plan.machine.card(self.card).ok_or_else(|| { GpuError::shape(WHAT, "the projected tier card is absent from the plan") })?; - let reserve = totals - .context_bytes - .checked_add(totals.scratch_bytes) - .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) - .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; + // Host-only projections have no card totals; their fixed reserves + // still belong to the machine. Dynamic scratch needs selected totals. + let reserve = if card.tier_leg.is_some() { + let totals = plan.cards.get(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) + } else { + card.set_aside_bytes() + } + .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, &plan.rows, diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index a5987a823..f3437908e 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -101,6 +101,9 @@ checked sizing arithmetic is used by allocation and planning; malformed or overf descriptors are refused by name. The plan records the selected reserve, and loaders check that amount instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver is used. +Host-only projections intentionally have no card totals: fixed reserves remain owned by their machine. +Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging +budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, From 9b833fb3ad8e6a8586205442c1b38f6607d5492e Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:37:10 +0900 Subject: [PATCH 14/85] gpu: remove redundant wrapper test allocation and borrow --- crates/gpu-gates/src/bin/gate_solar2_twocard.rs | 2 +- crates/gpu/src/card_experts.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index af0916a7c..9bcd34bc7 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs)?))?; + .machine(None, Some(place::tier_batch(inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 7c53831d5..468a422bd 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -1002,8 +1002,7 @@ mod tests { let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); - let route = - DeviceBuffer::from_host(stream, &vec![0.125f32; CAP * USED]).expect("weights"); + let route = DeviceBuffer::from_host(stream, &[0.125f32; CAP * USED]).expect("weights"); let host_sel: Vec = (0..CAP * USED) .map(|i| { if i % USED == USED - 1 { From 8f2de9a9ae3780d83a256f97a2e6c8d125ec3021 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:24:29 +0900 Subject: [PATCH 15/85] gpu: support six-slot F32 tier handoff --- crates/gpu-deepseek41/src/chain/ffn/batch.rs | 215 ++++++++++++ crates/gpu/src/host/handoff.rs | 327 ++++++++++++++----- docs/research/flint-attach.md | 22 ++ tools/unsafe-ratchet.txt | 4 +- 4 files changed, 490 insertions(+), 78 deletions(-) diff --git a/crates/gpu-deepseek41/src/chain/ffn/batch.rs b/crates/gpu-deepseek41/src/chain/ffn/batch.rs index 42f350086..c3df4ec36 100644 --- a/crates/gpu-deepseek41/src/chain/ffn/batch.rs +++ b/crates/gpu-deepseek41/src/chain/ffn/batch.rs @@ -2669,3 +2669,218 @@ fn count_of(by_count: &mut [T], m: usize) -> Result<&mut T, GpuError> { detail: format!("a chunk of {m} tokens; the batch holds scratch for 1..={n}"), }) } + +#[cfg(test)] +mod tests { + use super::{CardAccTier, FfnBatchKernels}; + use bloomery_gpu::host::handoff::{Handoff, HandoffKernels}; + use bloomery_gpu::host::page::HandoffLayout; + use bloomery_gpu::host::step::HandoffTarget; + use bloomery_gpu::host::tier::{TierImageLayout, TierTarget}; + use bloomery_gpu::hybrid::HOST; + use bloomery_gpu::{FaultSite, Gpu, GpuError, Window}; + use cuda_core::DeviceBuffer; + + /// The handoff's places feed the production tier sum, not a readback + /// alone: every routed row belongs to exactly one of host, card and + /// tier, and all other copies are poisoned. Both slot entries must + /// conserve that union, including the final routed lane and partial + /// blocks, eagerly and on replay. Invalid ids must still raise. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-ds41-lib` runs it on the box"] + fn hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes() { + const EXPERTS: usize = 16; + const ROW_OFF: usize = 3; + const LAYER: usize = 7; + let gpu = Gpu::new().expect("a GPU"); + let stream = gpu.stream(); + let handoff = HandoffKernels::load(gpu.context()).expect("handoff kernels"); + let sums = FfnBatchKernels::load(gpu.context()).expect("tier sum kernels"); + let mut map = vec![HOST - 1; ROW_OFF + EXPERTS]; + let mut tmap = map.clone(); + for id in 0..EXPERTS { + map[ROW_OFF + id] = if id % 3 == 0 { (id / 3) as u32 } else { HOST }; + tmap[ROW_OFF + id] = if id % 3 == 1 { (id / 3) as u32 } else { HOST }; + } + let map = DeviceBuffer::from_host(stream, &map).expect("card map"); + let tmap = DeviceBuffer::from_host(stream, &tmap).expect("tier map"); + for n_used in [6, 8] { + for hidden in [n_used, 256, 257] { + for bad in [None, Some(EXPERTS as u32), Some(u32::MAX)] { + let mut ids = vec![15, 0, 10, 5, 8, 1, 2, 7]; + ids.truncate(n_used); + if let Some(id) = bad { + ids[n_used - 1] = id; + } + // Neither handoff may read the router's trailing lane. + ids.push(u32::MAX); + let weights: Vec = (0..n_used) + .map(|s| (s + 1) as f32 / 16.0) + .chain([f32::NAN]) + .collect(); + let x: Vec = (0..hidden).map(|d| (d % 7 + 1) as f32 / 8.0).collect(); + let row = |s: usize, d: usize| (ids[s] + 1) as f32 * x[d]; + let mut down = vec![f32::NAN; n_used * hidden]; + let mut trows = down.clone(); + let mut host = vec![0.0f32; hidden]; + let mut want = host.clone(); + for d in 0..hidden { + let mut card_tier = 0.0f32; + for s in 0..n_used { + if ids[s] >= EXPERTS as u32 { + continue; + } + let value = row(s, d); + match ids[s] % 3 { + 0 => down[s * hidden + d] = value, + 1 => trows[s * hidden + d] = value, + _ => host[d] = value.mul_add(weights[s], host[d]), + } + if ids[s] % 3 != 2 { + card_tier = value.mul_add(weights[s], card_tier); + } + } + want[d] = card_tier + host[d]; + } + let ids_d = DeviceBuffer::from_host(stream, &ids).expect("ids"); + let mut weights_d = DeviceBuffer::from_host(stream, &weights).expect("weights"); + let x_d = DeviceBuffer::from_host(stream, &x).expect("activation"); + let seq = DeviceBuffer::from_host(stream, &[19u32]).expect("sequence"); + let down = DeviceBuffer::from_host(stream, &down).expect("card rows"); + let trows = DeviceBuffer::from_host(stream, &trows).expect("tier rows"); + let doubled_host: Vec = host.iter().map(|h| 2.0 * h).collect(); + let mut host = DeviceBuffer::from_host(stream, &host).expect("host sum"); + let lay = HandoffLayout { + seq: 0, + ids: 1, + weights: 1 + n_used, + x: 1 + 2 * n_used, + n_used, + hidden, + }; + let tl = TierImageLayout { + sel: 0, + q3: 0, + d8: 0, + x: n_used, + n_used, + q3_words: 0, + d8_len: 0, + x_len: hidden, + hidden, + cols: 1, + }; + for captured in [false, true] { + let mut image = + DeviceBuffer::zeroed(stream, lay.x + hidden).expect("image"); + let mut timage = + DeviceBuffer::zeroed(stream, tl.x + hidden).expect("tier image"); + let poison = vec![HOST - 1; n_used + 1]; + let mut sel = DeviceBuffer::from_host(stream, &poison).expect("places"); + let mut tsel = + DeviceBuffer::from_host(stream, &poison).expect("tier places"); + let mut acc = DeviceBuffer::zeroed(stream, hidden).expect("sum"); + let mut result = DeviceBuffer::zeroed(stream, hidden).expect("result"); + let mut enqueue = || -> Result<(), GpuError> { + handoff.enqueue_handoff_tier( + stream, + &Handoff { + ids: &ids_d, + weights: &weights_d, + map: &map, + row_off: ROW_OFF, + n_expert: EXPERTS, + }, + &tmap, + ( + HandoffTarget { + x: &x_d, + seq: &seq, + image: &mut image, + layout: lay, + }, + TierTarget { + image: &mut timage, + rows: &trows, + layout: tl, + }, + ), + gpu.layer_sink(LAYER)?, + (&mut sel, &mut tsel), + )?; + let routed_weights = + Window::::of(&image, 4 * lay.weights, n_used)?; + sums.enqueue_card_acc_tier( + stream, + &CardAccTier { + down: &down, + trows: &trows, + w: &routed_weights, + sel: &sel, + tsel: &tsel, + n: hidden, + m: 1, + n_card: EXPERTS.div_ceil(3), + n_tier: EXPERTS.div_ceil(3), + n_used, + }, + &mut acc, + )?; + gpu.elem() + .enqueue_add(stream, &acc, &host, hidden, &mut result) + }; + if captured { + let graph = gpu.capture(|_| enqueue()).expect("capture"); + graph.launch(stream).expect("replay"); + let doubled_weights: Vec = + weights.iter().map(|w| 2.0 * w).collect(); + weights_d + .copy_from_host(stream, &doubled_weights) + .expect("new route weights"); + host.copy_from_host(stream, &doubled_host) + .expect("new host sum"); + graph.launch(stream).expect("second replay"); + stream.synchronize().expect("replays settled"); + } else { + enqueue().expect("eager handoff and consumption"); + } + let scale = if captured { 2.0 } else { 1.0 }; + let got = result.to_host_vec(stream).expect("result"); + let places = sel.to_host_vec(stream).expect("places"); + let tier_places = tsel.to_host_vec(stream).expect("tier places"); + let image = image.to_host_vec(stream).expect("host image"); + let timage = timage.to_host_vec(stream).expect("tier image"); + stream.synchronize().expect("settled"); + let fault = gpu.take_fault().expect("fault word"); + if bad.is_some() { + let fault = fault.expect("an invalid id must raise"); + assert_eq!(fault.layer, LAYER as u32); + assert_eq!(fault.code, FaultSite::ExpertId as u32); + assert_eq!(fault.sites, 1 << FaultSite::ExpertId as u32); + } else { + assert!(fault.is_none(), "{fault:?}"); + } + assert_eq!(places[n_used], HOST - 1, "card places tail"); + assert_eq!(tier_places[n_used], HOST - 1, "tier places tail"); + assert_eq!(image[lay.seq], 19, "sequence"); + assert_eq!(&image[lay.ids..lay.ids + n_used], &ids[..n_used]); + for s in 0..n_used { + assert_eq!(image[lay.weights + s], (weights[s] * scale).to_bits()); + assert_eq!(timage[tl.sel + s], tier_places[s]); + } + for d in 0..hidden { + assert_eq!(image[lay.x + d], x[d].to_bits()); + assert_eq!(timage[tl.x + d], x[d].to_bits()); + } + assert!( + got.iter() + .zip(&want) + .all(|(g, w)| g.to_bits() == (w * scale).to_bits()), + "{n_used} slots, width {hidden}, bad {bad:?}, capture {captured}: result conservation" + ); + } + } + } + } + } +} diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 1e36a2d0a..cd453226c 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,10 +18,9 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_8_tier` and -//! `ds41_ffn_handoff_4_tier` are the eight- and four-slot handoffs of a layer -//! an expert tier holds experts of: the -//! same writes, and the slots' tier places and the activation in f32 into +//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_6_tier` and +//! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an +//! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); //! `ds41_ffn_handoff_10_cols_tier` is `_10_cols`'s of such a layer, every //! column's tier places and activation into the row's tier image @@ -282,6 +281,88 @@ mod handoff_kernels { unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; } + /// [`ds41_ffn_handoff`] and the row's f32 tier image, with the same + /// per-thread writes and following go as [`ds41_ffn_handoff_8_tier`]. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + ids_in.len() >= 6, + w_in.len() >= 6, + map.len() >= row_off + n_expert, + tmap.len() >= row_off + n_expert, + x.len() >= n, + seq.len() >= 1, + n >= 6, + ids_at >= seq_at + 1, + wts_at >= ids_at + 6, + x_at >= wts_at + 6, + image.len() >= x_at + n, + tx_at >= tsel_at + 6, + timage.len() >= tx_at + n, + sel.len() >= 6, + tsel.len() >= 6 + ) + )] + pub fn ds41_ffn_handoff_6_tier( + ids_in: &[u32], + w_in: &[f32], + map: &[u32], + tmap: &[u32], + row_off: u32, + n_expert: u32, + x: &[f32], + seq: &[u32], + n: u32, + seq_at: u32, + ids_at: u32, + wts_at: u32, + x_at: u32, + tsel_at: u32, + tx_at: u32, + fault: FaultSink, + mut image: DisjointSlice, + mut timage: DisjointSlice, + mut sel: DisjointSlice, + mut tsel: DisjointSlice, + ) { + let d = thread::index_1d().get(); + if d >= n as usize { + return; + } + let a = HandoffIn { + ids_in, + w_in, + map, + row_off, + n_expert, + x, + seq, + seq_at, + ids_at, + wts_at, + x_at, + fault, + }; + // SAFETY: d < n, and the launch contract is `handoff_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; + let t = TierIn { + tmap, + tsel_at, + tx_at, + }; + // SAFETY: d < n, and the launch contract is `handoff_tier_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_tier_at::(&a, &t, d, &mut timage, &mut tsel) }; + } + /// [`ds41_ffn_handoff_8`] and the row's f32 tier image /// ([`handoff_tier_at`]): beside the handoff and the stage card's places, /// threads `s < 8` write `tsel[s] = tmap[row_off + id]` — the id's slot @@ -1265,8 +1346,9 @@ impl HandoffKernels { } impl HandoffKernels { - /// Enqueue the handoff of a tier layer of four or eight slots a token - /// (`ds41_ffn_handoff_4_tier`, `ds41_ffn_handoff_8_tier`): [`HandoffKernels::enqueue_handoff`]'s + /// Enqueue the handoff of a tier layer of six or eight slots a token + /// (`ds41_ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and /// with the activation into the f32 tier image `ttarget`. A layout of @@ -1286,79 +1368,109 @@ impl HandoffKernels { fault: FaultSink, (sel, tsel): (&mut DeviceBuffer, &mut DeviceBuffer), ) -> Result<(), GpuError> { - const WHAT: &str = "ds41_ffn_handoff_tier"; let (lay, tl) = (target.layout, ttarget.layout); - if !matches!(lay.n_used, SLOTS_4 | SLOTS_8) - || tl.n_used != lay.n_used - || tl.x_len != lay.hidden - || tl.q3_words != 0 - || tl.d8_len != 0 - { - return Err(GpuError::Shape { - what: WHAT, - detail: format!( - "the boundary carries {} slots of {} values and the tier image {} slots, {} \ - f32, {} code words, {} scales; the entries hand over {SLOTS_4} or {SLOTS_8} \ - slots, the tier image the boundary's, and the activation in f32", - lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len - ), - }); - } + let slots = check_tier(&lay, &tl)?; + let what = match slots { + HandoffSlots::Four => "ds41_ffn_handoff_4_tier", + HandoffSlots::Six => "ds41_ffn_handoff_6_tier", + _ => "ds41_ffn_handoff_8_tier", + }; let n = lay.hidden; - let grid = launch_u32(WHAT, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; + let grid = launch_u32(what, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; let cfg = LaunchConfig1D::new(grid, HANDOFF_THREADS, 0); - if lay.n_used == SLOTS_4 { - let prep = self.module.prepare_ds41_ffn_handoff_4_tier(cfg)?; - self.module.ds41_ffn_handoff_4_tier( - stream, - &prep, - h.ids, - h.weights, - h.map, - tmap, - launch_u32(WHAT, "row_off", h.row_off)?, - launch_u32(WHAT, "n_expert", h.n_expert)?, - target.x, - target.seq, - launch_u32(WHAT, "n", n)?, - launch_u32(WHAT, "seq_at", lay.seq)?, - launch_u32(WHAT, "ids_at", lay.ids)?, - launch_u32(WHAT, "wts_at", lay.weights)?, - launch_u32(WHAT, "x_at", lay.x)?, - launch_u32(WHAT, "tsel_at", tl.sel)?, - launch_u32(WHAT, "tx_at", tl.x)?, - fault, - target.image, - ttarget.image, - sel, - tsel, - )?; - } else { - let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; - self.module.ds41_ffn_handoff_8_tier( - stream, - &prep, - h.ids, - h.weights, - h.map, - tmap, - launch_u32(WHAT, "row_off", h.row_off)?, - launch_u32(WHAT, "n_expert", h.n_expert)?, - target.x, - target.seq, - launch_u32(WHAT, "n", n)?, - launch_u32(WHAT, "seq_at", lay.seq)?, - launch_u32(WHAT, "ids_at", lay.ids)?, - launch_u32(WHAT, "wts_at", lay.weights)?, - launch_u32(WHAT, "x_at", lay.x)?, - launch_u32(WHAT, "tsel_at", tl.sel)?, - launch_u32(WHAT, "tx_at", tl.x)?, - fault, - target.image, - ttarget.image, - sel, - tsel, - )?; + let row_off = launch_u32(what, "row_off", h.row_off)?; + let n_expert = launch_u32(what, "n_expert", h.n_expert)?; + let at = [ + launch_u32(what, "n", n)?, + launch_u32(what, "seq_at", lay.seq)?, + launch_u32(what, "ids_at", lay.ids)?, + launch_u32(what, "wts_at", lay.weights)?, + launch_u32(what, "x_at", lay.x)?, + launch_u32(what, "tsel_at", tl.sel)?, + launch_u32(what, "tx_at", tl.x)?, + ]; + match slots { + HandoffSlots::Four => { + let prep = self.module.prepare_ds41_ffn_handoff_4_tier(cfg)?; + self.module.ds41_ffn_handoff_4_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + HandoffSlots::Six => { + let prep = self.module.prepare_ds41_ffn_handoff_6_tier(cfg)?; + self.module.ds41_ffn_handoff_6_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + _ => { + let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; + self.module.ds41_ffn_handoff_8_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } } Ok(()) } @@ -1514,6 +1626,29 @@ impl HandoffKernels { } } +/// The single-token F32 tier handoff's layout contract, shared by the +/// slot entries. The generated launch checks the input and output spans. +fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result { + let slots = HandoffSlots::of(lay.n_used); + if !matches!(slots, Ok(HandoffSlots::Four | HandoffSlots::Six | HandoffSlots::Eight)) + || tl.n_used != lay.n_used + || tl.x_len != lay.hidden + || tl.q3_words != 0 + || tl.d8_len != 0 + { + return Err(GpuError::Shape { + what: "HandoffKernels::enqueue_handoff_tier", + detail: format!( + "the boundary carries {} slots of {} values and the tier image {} slots, {} \ + f32, {} code words, {} scales; the entries hand over matching 6 or 8 slots \ + and the activation in f32", + lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len + ), + }); + } + slots.map_err(|e| GpuError::shape("HandoffKernels::enqueue_handoff_tier", e.to_string())) +} + /// [`HandoffKernels::enqueue_handoff_cols`]'s refusals, by name before the /// launch: a layout of another slot count, `m` of 0, a pitch under ten, and a /// `sel` of `sel_len` places shorter than `m` columns of ten. @@ -1600,7 +1735,7 @@ pub struct Places<'a> { #[cfg(test)] mod tests { - use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier}; + use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier, check_tier}; use crate::host::page::HandoffLayout; use crate::host::tier::TierImageLayout; @@ -1651,6 +1786,44 @@ mod tests { } } + #[test] + fn a_token_tier_handoff_requires_matching_f32_six_or_eight_slot_images() { + for n_used in 0..=10 { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + for tier_slots in 0..=10 { + let tl = TierImageLayout { + n_used: tier_slots, + x: tier_slots, + ..image(1) + }; + assert_eq!( + check_tier(&lay, &tl).is_ok(), + matches!(n_used, 6 | 8) && tier_slots == n_used, + "host {n_used}, tier {tier_slots}" + ); + } + } + for n_used in [6, 8] { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + let tl = TierImageLayout { + n_used, + x: n_used, + ..image(1) + }; + for x_len in [0, HIDDEN - 1, HIDDEN + 1, 2 * HIDDEN] { + assert!(check_tier(&lay, &TierImageLayout { x_len, ..tl }).is_err()); + } + assert!(check_tier(&lay, &TierImageLayout { q3_words: 1, ..tl }).is_err()); + assert!(check_tier(&lay, &TierImageLayout { d8_len: 1, ..tl }).is_err()); + } + } + /// A tier handoff of `m` columns over a page of 8 refuses, by name, a /// `tsel` or a `sel` shorter than `10·m`, and takes ones of exactly that: /// the launch writes every column's places. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index f3437908e..1df60b76d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -109,6 +109,28 @@ A host-only public planner consumer exercised the one-byte boundaries before the and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. +### Six-slot F32 tier handoff implementation + +`HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry +beside the existing eight-slot entry. Both host and tier layouts must have the +same supported slot count and activation width; compressed tier images remain +refused. The entry uses the existing copy/place/fault helpers and the existing +subsequent go/publication protocol; no Flint router or Program is registered. + +The pure layout contract passed locally after an oxide build. +`hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes`, owned by +`gate-gpu-ds41-lib`, consumes the handoff's weights and emitted places through +the production tier accumulator and adds the host sum. It covers partial widths, +invalid ids, poisoned non-owning rows, trailing sentinels and changed replay +weights. Its host sum is precomputed and activation images are checked by +readback: it is not expert execution or two-card completion/publication proof. +The hardware case has not been run here. + +The new `ds41_ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +owning-gate evidence; the eight-slot and other existing entries must remain +unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. +Actual six-slot body/tier and two-card completion remain integration work, not +capabilities inferred from this primitive. diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 6071636fb..f699c71f3 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -552,7 +552,9 @@ gguf 7 2 0 # PIN(2026-10-24): gpu 2414 -> 2418 blocks, new code, not old code relaxed: the two branch pins the rebase onto main # folded away, the four-slot tier handoff (ds41_ffn_handoff_4_tier, +2) and Muse's attention branch (its fork and join, # +2); 2331 + 83 + 2 + 2, recounted by tools/check-unsafe.sh. -gpu 2418 311 10 +# PIN(2026-10-25): gpu 2418 -> 2420 blocks, the six-slot F32 tier entry adds two +# handoff body calls under its checked launch contract; the newmain baseline is preserved. +gpu 2420 311 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. From 2b8c87ce0bc3aeb9b18b1c2c43fad91c0619d413 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:59:23 +0900 Subject: [PATCH 16/85] gpu: name the new six-slot tier primitive generically --- crates/gpu/src/host/handoff.rs | 12 ++++++------ docs/research/flint-attach.md | 6 +++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index cd453226c..e42f46a61 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,7 +18,7 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_6_tier` and +//! `crate::card_slots`' `ds41_ffn_places`. `ffn_handoff_6_tier` and //! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an //! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); @@ -310,7 +310,7 @@ mod handoff_kernels { tsel.len() >= 6 ) )] - pub fn ds41_ffn_handoff_6_tier( + pub fn ffn_handoff_6_tier( ids_in: &[u32], w_in: &[f32], map: &[u32], @@ -1347,7 +1347,7 @@ impl HandoffKernels { impl HandoffKernels { /// Enqueue the handoff of a tier layer of six or eight slots a token - /// (`ds41_ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// (`ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and @@ -1372,7 +1372,7 @@ impl HandoffKernels { let slots = check_tier(&lay, &tl)?; let what = match slots { HandoffSlots::Four => "ds41_ffn_handoff_4_tier", - HandoffSlots::Six => "ds41_ffn_handoff_6_tier", + HandoffSlots::Six => "ffn_handoff_6_tier", _ => "ds41_ffn_handoff_8_tier", }; let n = lay.hidden; @@ -1418,8 +1418,8 @@ impl HandoffKernels { )?; } HandoffSlots::Six => { - let prep = self.module.prepare_ds41_ffn_handoff_6_tier(cfg)?; - self.module.ds41_ffn_handoff_6_tier( + let prep = self.module.prepare_ffn_handoff_6_tier(cfg)?; + self.module.ffn_handoff_6_tier( stream, &prep, h.ids, diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 1df60b76d..b0a9b8b4e 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -126,11 +126,15 @@ weights. Its host sum is precomputed and activation images are checked by readback: it is not expert execution or two-card completion/publication proof. The hardware case has not been run here. -The new `ds41_ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +The new `ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and owning-gate evidence; the eight-slot and other existing entries must remain unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. Actual six-slot body/tier and two-card completion remain integration work, not capabilities inferred from this primitive. +`check-ptx-names` remains red until observed spill/JIT-local measurements can pin +the new entry in the four existing architecture/capability tables. Offline +no-JIT scans cannot supply that missing column, so no placeholder row is added. + From 49110c8c5c221a83cb42561ecc404226311e0958 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:47:50 +0900 Subject: [PATCH 17/85] gpu: implement the common optional-bias 256/top6 floor router --- crates/gpu-deepseek41/src/experts_mxfp4.rs | 2 +- crates/gpu-gates/src/bin/gate_flint_router.rs | 396 +++++++++++------ crates/gpu/src/route_body.rs | 417 ++++++++++++++++-- crates/gpu/src/route_core.rs | 12 +- crates/gpu/src/routed_block.rs | 20 +- crates/model/src/arch/flint/tests.rs | 13 +- crates/model/tests/flint_meta.rs | 10 +- crates/models/src/shape.rs | 151 +++++-- docs/research/flint-attach.md | 72 +-- 9 files changed, 838 insertions(+), 255 deletions(-) diff --git a/crates/gpu-deepseek41/src/experts_mxfp4.rs b/crates/gpu-deepseek41/src/experts_mxfp4.rs index 6c2888207..033f5f291 100644 --- a/crates/gpu-deepseek41/src/experts_mxfp4.rs +++ b/crates/gpu-deepseek41/src/experts_mxfp4.rs @@ -182,7 +182,7 @@ mod dflash_kernels { // N_EXPERT = n_expert <= bias.len() by the launch contract; score and // sel_v are block-shared arrays of N_EXPERT entries; the barrier below // publishes them. - unsafe { publish_n::(probs.as_mut_ptr(), bias, e0, score, sel_v, tid, fault) }; + unsafe { publish_n::(probs.as_mut_ptr(), Some(bias), e0, score, sel_v, tid, fault) }; thread::sync_threads(); if tid < 32 { diff --git a/crates/gpu-gates/src/bin/gate_flint_router.rs b/crates/gpu-gates/src/bin/gate_flint_router.rs index 790d7f152..fbaee13b9 100644 --- a/crates/gpu-gates/src/bin/gate_flint_router.rs +++ b/crates/gpu-gates/src/bin/gate_flint_router.rs @@ -1,18 +1,6 @@ -//! Flint's 256-expert top-6 sigmoid router (`sigmoid_256_6`), model-free. -//! -//! One token. The weight's column 0 holds the logits and the other columns -//! are 0; the activation is 1 at column 0, so each logit is one product. -//! Bias is 0 and the scale is 1. The logits fall strictly through the first -//! sixteen experts and sit at −20 after that, so the sixth and the seventh -//! do not tie. -//! -//! The slot count checked here is 6, not the instance's `N_USED`. The ids -//! equal the host top 6 of the kernel's scores plus bias (larger value, and -//! a tie to the larger index) and the host sigmoid ranking of the planted -//! logits. The weights are bit-identical to the host renorm of those scores -//! (f64 slot sum, narrowed, `renorm_divisor`, times the scale) and differ -//! from the first 6 weights of a top-7 renorm. A NaN logit raises `Router`; -//! a clean launch raises nothing. +//! The actual 256/top-6 selected-sum-floor binding, through RouterEntry. +//! Optional selection-only bias, active floor/zero sums, ties, exact rows, +//! nonfinite refusal and graph replay. This gate needs an NVIDIA device. #[cfg(feature = "gpu")] fn main() -> std::process::ExitCode { @@ -21,49 +9,79 @@ fn main() -> std::process::ExitCode { #[cfg(feature = "gpu")] mod gate { - use bloomery_gpu::route_body::sigmoid_256_6; - use bloomery_gpu::route_core::{renorm_divisor, sigmoid}; + use bloomery_gpu::gemm::Gemm32Kernels; + use bloomery_gpu::route_body::sigmoid_floor_256_6::{RouterKernels, RouterOut}; + use bloomery_gpu::routed_block::{RouteIn, RoutedRows, RouterEntry}; + use bloomery_gpu::route_core::sigmoid; use bloomery_gpu::{DeviceTensor, Fault, FaultSite, Gpu, LAYER_NONE}; use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, verdict}; use cuda_core::DeviceBuffer; - /// Experts, the pick this gate names, and the activation width. The pick - /// is not read from the instance. const EXPERTS: usize = 256; const USED: usize = 6; - const TOP7: usize = 7; - const K: usize = 32; - const SCALE: f32 = 1.0; - - fn logit(expert: usize) -> f32 { - if expert < 16 { - 20.0 - expert as f32 - } else { - -20.0 - } + const K: usize = 64; + const TOKENS: usize = 5; + const SCALE: f32 = 1.75; + const FLOOR: f32 = 1.0 / 16384.0; + + struct Cx { + gpu: Gpu, + router: RouterKernels, + wide: Gemm32Kernels, + } + + #[derive(Default)] + struct Pick { + scores: Vec, + ids: Vec, + weights: Vec, } - fn weight_matrix() -> Vec { - let mut w = vec![0.0f32; EXPERTS * K]; - for e in 0..EXPERTS { - w[e * K] = logit(e); + impl Pick { + fn same(&self, other: &Self) -> bool { + bits_equal(&self.scores, &other.scores) + && self.ids == other.ids + && bits_equal(&self.weights, &other.weights) } - w } - /// The top `k` of `key`, larger value first, a tie to the larger index. fn top_k(key: &[f32], k: usize) -> Vec { - let mut idx: Vec = (0..key.len() as u32).collect(); - idx.sort_by(|&a, &b| key[b as usize].total_cmp(&key[a as usize]).then(b.cmp(&a))); - idx.truncate(k); - idx + let mut ids: Vec = (0..EXPERTS as u32).collect(); + ids.sort_by(|&a, &b| key[b as usize].total_cmp(&key[a as usize]).then(b.cmp(&a))); + ids.truncate(k); + ids } - /// Unbiased scores at `ids`, summed in slot order in f64, narrowed, guarded, scaled. + // Independent contract expression: do not reuse the device divisor. fn weights_of(scores: &[f32], ids: &[u32], scale: f32) -> Vec { - let g: Vec = ids.iter().map(|&i| scores[i as usize]).collect(); - let div = renorm_divisor(g.iter().fold(0.0f64, |s, &v| s + f64::from(v)) as f32); - g.into_iter().map(|v| v / div * scale).collect() + let sum = ids.iter().fold(0.0f64, |sum, &e| sum + f64::from(scores[e as usize])); + let div = (sum as f32).max(FLOOR); + ids.iter().map(|&e| scores[e as usize] / div * scale).collect() + } + + fn matrix(logits: &[f32]) -> Vec { + let mut w = vec![0.0; EXPERTS * K]; + for (e, &logit) in logits.iter().enumerate() { + w[e * K] = logit; + } + w + } + + fn activations(factors: &[f32]) -> Vec { + let mut x = vec![0.0; factors.len() * K]; + for (t, &factor) in factors.iter().enumerate() { + x[t * K] = factor; + } + x + } + + fn read_one(cx: &Cx, out: &RouterOut) -> Result { + let s = cx.gpu.stream(); + Ok(Pick { + scores: out.probs.to_host_vec(s)?, + ids: out.ids.to_host_vec(s)?, + weights: out.weights.to_host_vec(s)?, + }) } fn clause(name: &str, pass: bool, detail: &str) -> bool { @@ -71,101 +89,217 @@ mod gate { pass } - pub fn run() -> Result<(), GateError> { - bloomery_levers::at_main(&[])?; - let gpu = Gpu::new()?; - println!("gate_flint_router: device {}", gpu.device_name()?); - let s = gpu.stream(); - let rk = sigmoid_256_6::RouterKernels::load(gpu.context())?; - let planted: Vec = (0..EXPERTS).map(logit).collect(); - let by_logit = top_k( - &planted.iter().copied().map(sigmoid).collect::>(), - USED, - ); - let w_host = weight_matrix(); - let x_host = { - let mut x = vec![0.0f32; K]; - x[0] = 1.0; - x - }; - let bias_host = vec![0.0f32; EXPERTS]; - let w = DeviceTensor::upload(s, &w_host, EXPERTS, K)?; - let x = DeviceBuffer::from_host(s, &x_host)?; - let bias = DeviceBuffer::from_host(s, &bias_host)?; - let mut out = sigmoid_256_6::RouterOut::new(s)?; - gpu.take_fault()?; - rk.enqueue_router(s, &w, &x, &bias, SCALE, &mut out, gpu.unlabelled_sink())?; + fn valid_case(cx: &Cx, name: &str, logits: &[f32], bias: Option<&[f32]>) -> Result { + let s = cx.gpu.stream(); + let mut wh = matrix(logits); + if name == "dense" { + for e in 0..EXPERTS { + for j in 0..K { + wh[e * K + j] = ((e * 17 + j * 11) % 67) as f32 / 67.0 - 0.5; + } + } + } + let w = DeviceTensor::upload(s, &wh, EXPERTS, K)?; + let factors = [1.0, 0.0, 2.0, -1.0, 0.5]; + let mut xh = activations(&factors); + if name == "dense" { + for t in 0..TOKENS { + for j in 1..K { + xh[t * K + j] = ((t * 13 + j * 7) % 31) as f32 / 31.0 - 0.5; + } + } + } + let x = DeviceBuffer::from_host(s, &xh)?; + let b = bias.map(|b| DeviceBuffer::from_host(s, b)).transpose()?; + let input = RouteIn { w: &w, bias: b.as_ref(), scale: SCALE, fault: cx.gpu.unlabelled_sink() }; + let mut out = RouterOut::new(s)?; + let mut single = Pick::default(); + let mut first = Pick::default(); + let mut tickets = true; + cx.gpu.take_fault()?; + for t in 0..TOKENS { + let xt = DeviceBuffer::from_host(s, &xh[t * K..(t + 1) * K])?; + cx.router.route(s, &xt, &input, &mut out)?; + s.synchronize()?; + let pick = read_one(cx, &out)?; + tickets &= out.tickets(s)? == 0; + if t == 0 { + first = read_one(cx, &out)?; + } + single.scores.extend(pick.scores); + single.ids.extend(pick.ids); + single.weights.extend(pick.weights); + } + let mut probs = DeviceBuffer::zeroed(s, EXPERTS * TOKENS)?; + let mut ids = DeviceBuffer::zeroed(s, USED * TOKENS)?; + let mut weights = DeviceBuffer::zeroed(s, USED * TOKENS)?; + cx.router.route_rows( + (s, &cx.gpu, &cx.wide), &x, &input, TOKENS, + RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, + )?; s.synchronize()?; - let scores = out.probs.to_host_vec(s)?; - let ids = out.ids.to_host_vec(s)?; - let weights = out.weights.to_host_vec(s)?; - let clean = gpu.take_fault()?; - - let by_score = top_k( - &scores - .iter() - .zip(&bias_host) - .map(|(&p, &b)| p + b) - .collect::>(), - USED, - ); - let len_ok = ids.len() == USED && weights.len() == USED; - let ids_ok = len_ok && ids == by_score && ids == by_logit; - let expect = weights_of(&scores, &by_score, SCALE); - let w_ok = weights.len() == USED && expect.len() == USED && bits_equal(&weights, &expect); - let by7 = top_k( - &scores - .iter() - .zip(&bias_host) - .map(|(&p, &b)| p + b) - .collect::>(), - TOP7, - ); - let w7 = weights_of(&scores, &by7, SCALE); - let differ = - weights.len() == USED && w7.len() == TOP7 && !bits_equal(&weights, &w7[..USED]); - + let batch = Pick { + scores: probs.to_host_vec(s)?, ids: ids.to_host_vec(s)?, weights: weights.to_host_vec(s)?, + }; + let clean = cx.gpu.take_fault()?.is_none(); + let mut expected = true; + for t in 0..TOKENS { + let scores = &single.scores[t * EXPERTS..(t + 1) * EXPERTS]; + let keys: Vec = scores.iter().enumerate() + .map(|(e, &p)| bias.map_or(p, |b| p + b[e])).collect(); + let want_ids = top_k(&keys, USED); + expected &= single.ids[t * USED..(t + 1) * USED] == want_ids; + if name != "dense" { + let planted_keys: Vec = logits.iter().enumerate().map(|(e, &logit)| { + let p = sigmoid(logit * factors[t]); + bias.map_or(p, |b| p + b[e]) + }).collect(); + expected &= single.ids[t * USED..(t + 1) * USED] == top_k(&planted_keys, USED); + } + expected &= bits_equal( + &single.weights[t * USED..(t + 1) * USED], &weights_of(scores, &want_ids, SCALE), + ); + } let mut ok = clause( - "router ids top 6", - ids_ok && clean.is_none(), - &format!( - "len={} ids={ids:?} by_score={by_score:?} by_logit={by_logit:?} fault={clean:?}", - ids.len() - ), - ); - ok &= clause( - "router weights renorm of the top 6", - w_ok && clean.is_none(), - &format!("bit_identical={w_ok} slots={}", weights.len()), - ); - ok &= clause( - "router weights differ from a top-7 renorm", - differ && clean.is_none(), - &format!("differ={differ}"), + &format!("{name} selection and unbiased floor weights"), + expected && clean && tickets, + &format!("bias={} clean={clean} tickets_zero={tickets}", bias.is_some()), ); + ok &= clause(&format!("{name} token-major rows"), single.same(&batch), "bit_identical"); + if name.starts_with("tiny") { + let sum: f32 = first.ids.iter().map(|&e| first.scores[e as usize]).sum(); + let legacy: Vec = first.ids.iter() + .map(|&e| first.scores[e as usize] / (sum + 1e-20) * SCALE).collect(); + ok &= clause("tiny floor active", sum > 0.0 && sum < FLOOR + && !bits_equal(&first.weights, &legacy) + && first.weights.iter().sum::() < SCALE, "not epsilon renormalization"); + } + if name == "zero" || name == "ties" { + ok &= clause(&format!("{name} descending-id ties"), + first.ids == [255, 254, 253, 252, 251, 250], "six distinct ids"); + if name == "zero" { + ok &= clause("zero selected sum", first.weights.iter().all(|&w| w == 0.0), "finite zero weights"); + } + } + if let Some(bias) = bias { + let keys: Vec = first.scores.iter().zip(bias).map(|(&p, &b)| p + b).collect(); + ok &= clause("bias changes selection not weight source", + first.ids != top_k(&first.scores, USED) + && !bits_equal(&first.weights, &weights_of(&keys, &first.ids, SCALE)), + "unbiased selected scores consumed"); + } else if name == "normal" { + let top7 = top_k(&first.scores, 7); + ok &= clause("six not seven selected scores", + !bits_equal(&first.weights, &weights_of(&first.scores, &top7, SCALE)[..USED]), + "top-seven renormalization differs"); + } + + let x1 = DeviceBuffer::from_host(s, &xh[..K])?; + let graph = cx.gpu.capture(|st| cx.router.route(st, &x1, &input, &mut out))?; + let mut replay = graph.node_count() == 1; + for _ in 0..2 { + graph.launch(s)?; + s.synchronize()?; + replay &= read_one(cx, &out)?.same(&first) && out.tickets(s)? == 0; + } + ok &= clause(&format!("{name} single capture"), replay, "one node, two exact replays, zero tickets"); + let graph = cx.gpu.capture(|st| cx.router.route_rows( + (st, &cx.gpu, &cx.wide), &x, &input, TOKENS, + RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, + ))?; + let mut replay = graph.node_count() == 3; + for _ in 0..2 { + graph.launch(s)?; + s.synchronize()?; + let now = Pick { + scores: probs.to_host_vec(s)?, ids: ids.to_host_vec(s)?, weights: weights.to_host_vec(s)?, + }; + replay &= now.same(&single); + } + replay &= cx.gpu.take_fault()?.is_none(); + ok &= clause(&format!("{name} rows capture"), replay, "three nodes, two exact replays, no fault"); + Ok(ok) + } - let mut nan_w = w_host; - nan_w[40 * K] = f32::NAN; - let nan_w = DeviceTensor::upload(s, &nan_w, EXPERTS, K)?; - let mut nan_out = sigmoid_256_6::RouterOut::new(s)?; - gpu.take_fault()?; - rk.enqueue_router( - s, - &nan_w, - &x, - &bias, - SCALE, - &mut nan_out, - gpu.unlabelled_sink(), + fn invalid_case(cx: &Cx, name: &str, logit: Option, bias: Option, scale: f32) -> Result { + let s = cx.gpu.stream(); + let mut logits = vec![0.0; EXPERTS]; + if let Some(logit) = logit { logits[40] = logit; } + let w = DeviceTensor::upload(s, &matrix(&logits), EXPERTS, K)?; + let mut bh = vec![0.0; EXPERTS]; + if let Some(bias) = bias { bh[40] = bias; } + let b = bias.map(|_| DeviceBuffer::from_host(s, &bh)).transpose()?; + let x = DeviceBuffer::from_host(s, &activations(&[1.0, 1.0]))?; + let input = RouteIn { w: &w, bias: b.as_ref(), scale, fault: cx.gpu.unlabelled_sink() }; + let mut out = RouterOut::new(s)?; + cx.gpu.take_fault()?; + cx.router.route(s, &x, &input, &mut out)?; + s.synchronize()?; + let one = cx.gpu.take_fault()?; + let one_poisoned = out.weights.to_host_vec(s)?.iter().all(|w| !w.is_finite()); + let tickets = out.tickets(s)?; + let mut probs = DeviceBuffer::zeroed(s, EXPERTS * 2)?; + let mut ids = DeviceBuffer::zeroed(s, USED * 2)?; + let mut weights = DeviceBuffer::zeroed(s, USED * 2)?; + cx.router.route_rows( + (s, &cx.gpu, &cx.wide), &x, &input, 2, + RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, )?; s.synchronize()?; - let nan_fault = gpu.take_fault()?; + let rows = cx.gpu.take_fault()?; + let rows_poisoned = weights.to_host_vec(s)?.iter().all(|w| !w.is_finite()); let want = Some(Fault::at(LAYER_NONE, FaultSite::Router)); - ok &= clause( - "router NaN logit", - nan_fault == want, - &format!("fault={nan_fault:?} want={want:?}"), - ); + Ok(clause(name, one == want && rows == want && one_poisoned && rows_poisoned && tickets == 0, + &format!("one={one:?} rows={rows:?} poisoned={} tickets={tickets}", one_poisoned && rows_poisoned))) + } + + fn host_boundaries(cx: &Cx) -> Result { + let s = cx.gpu.stream(); + let w = DeviceTensor::upload(s, &matrix(&vec![0.0; EXPERTS]), EXPERTS, K)?; + let x = DeviceBuffer::zeroed(s, K)?; + let short_bias = DeviceBuffer::zeroed(s, EXPERTS - 1)?; + let mut out = RouterOut::new(s)?; + let sink = cx.gpu.unlabelled_sink(); + let short = cx.router.enqueue_router(s, &w, &x, Some(&short_bias), SCALE, &mut out, sink).is_err(); + let mut probs = DeviceBuffer::zeroed(s, EXPERTS)?; + let mut ids = DeviceBuffer::zeroed(s, USED)?; + let mut weights = DeviceBuffer::zeroed(s, USED)?; + let empty = cx.router.enqueue_router_rows(s, &cx.gpu, &cx.wide, &w, &x, None, + SCALE, 0, &mut probs, &mut ids, &mut weights, sink).is_err(); + let short_rows = cx.router.enqueue_router_rows(s, &cx.gpu, &cx.wide, &w, &x, None, + SCALE, 2, &mut probs, &mut ids, &mut weights, sink).is_err(); + let legacy = bloomery_gpu::route_body::sigmoid_256_6::RouterKernels::load(cx.gpu.context())?; + let mut legacy_out = bloomery_gpu::route_body::sigmoid_256_6::RouterOut::new(s)?; + let missing = legacy.route(s, &x, + &RouteIn { w: &w, bias: None, scale: SCALE, fault: sink }, &mut legacy_out).is_err(); + Ok(clause("host boundaries", short && empty && short_rows && missing, + "short bias, empty/short rows, legacy missing bias refused")) + } + + pub fn run() -> Result<(), GateError> { + bloomery_levers::at_main(&[])?; + let gpu = Gpu::new()?; + println!("gate_flint_router: device {}", gpu.device_name()?); + let router = RouterKernels::load(gpu.context())?; + let wide = Gemm32Kernels::load(gpu.context())?; + let cx = Cx { gpu, router, wide }; + let normal: Vec = (0..EXPERTS).map(|e| if e < 16 { 2.0 - e as f32 * 0.25 } else { -20.0 }).collect(); + let tiny: Vec = (0..EXPERTS).map(|e| -30.0 - (e % 16) as f32 * 0.25).collect(); + let mut bias = vec![0.0; EXPERTS]; + for (e, b) in bias.iter_mut().enumerate().skip(240) { *b = 2.0 + e as f32 / 256.0; } + let mut ok = valid_case(&cx, "normal", &normal, None)?; + ok &= valid_case(&cx, "dense", &normal, None)?; + ok &= valid_case(&cx, "biased", &normal, Some(&bias))?; + ok &= valid_case(&cx, "tiny", &tiny, None)?; + ok &= valid_case(&cx, "tiny-biased", &tiny, Some(&bias))?; + ok &= valid_case(&cx, "zero", &vec![-200.0; EXPERTS], None)?; + ok &= valid_case(&cx, "ties", &vec![0.0; EXPERTS], None)?; + for (name, value) in [("NaN", f32::NAN), ("+infinity", f32::INFINITY), ("-infinity", f32::NEG_INFINITY)] { + ok &= invalid_case(&cx, &format!("{name} logit"), Some(value), None, SCALE)?; + ok &= invalid_case(&cx, &format!("{name} selection bias"), None, Some(value), SCALE)?; + } + ok &= invalid_case(&cx, "nonfinite weight scale", None, None, f32::INFINITY)?; + ok &= host_boundaries(&cx)?; if ok { Ok(()) } else { Err(checks_failed()) } } } diff --git a/crates/gpu/src/route_body.rs b/crates/gpu/src/route_body.rs index 0bf4cd545..6058f9154 100644 --- a/crates/gpu/src/route_body.rs +++ b/crates/gpu/src/route_body.rs @@ -27,7 +27,7 @@ //! ones. use crate::q8f32::f32_lane_partial_1col_w32; -use crate::route_core::{Score, renorm_divisor, router_score, take}; +use crate::route_core::{Score, floor_divisor, renorm_divisor, router_score, take}; use crate::{FaultSink, FaultSite}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicU32}; use cuda_device::{DisjointSlice, thread, threadfence, warp}; @@ -44,20 +44,20 @@ const _: () = assert!(ROUTER_THREADS_U32 as usize == ROUTER_THREADS); // The entries' `launch_bounds` and `launch_contract` block are literals. const _: () = assert!(ROUTER_THREADS == 128); -/// V4.1's `publish` over `E` experts: thread `tid` parks `score[e] = probs[at + -/// e]` (a volatile load) and `sel_v[e] = score[e] + bias[e]` for `e = tid + -/// ROUTER_THREADS · i`, raising [`FaultSite::Router`] on a selection value -/// that is not finite. +/// Publish `E` scores: thread `tid` parks `score[e] = probs[at + e]` +/// (a volatile load) and the same value plus any supplied selection bias in +/// `sel_v[e]`, for `e = tid + ROUTER_THREADS · i`. A nonfinite selection +/// value raises [`FaultSite::Router`]. /// /// # Safety /// -/// `probs` addresses at least `at + E` values, `bias` holds `E`, +/// `probs` addresses at least `at + E` values, any `bias` holds `E`, /// `score` and `sel_v` are the block's shared arrays of `E`, and the caller's /// barrier follows before any read of them. #[inline(always)] pub unsafe fn publish_n( probs: *const f32, - bias: &[f32], + bias: Option<&[f32]>, at: usize, score: *mut f32, sel_v: *mut f32, @@ -68,8 +68,11 @@ pub unsafe fn publish_n( while e < E { // SAFETY: at + e < at + E, inside probs by this fn's contract. let p = unsafe { core::ptr::read_volatile(probs.add(at + e)) }; - // SAFETY: e < E <= bias.len() by this fn's contract. - let v = p + unsafe { *bias.get_unchecked(e) }; + // SAFETY: any supplied bias holds E entries by this fn's contract. + let v = match bias { + Some(bias) => p + unsafe { *bias.get_unchecked(e) }, + None => p, + }; if !v.is_finite() { fault.raise(FaultSite::Router); } @@ -86,12 +89,12 @@ pub unsafe fn publish_n( /// V4.1's `select` over `E` experts and `U` slots: `U` rounds of the butterfly /// argmax under [`take`]`::`, lane `L` scanning experts `L + 32 j < E` /// (`j < ⌈E / 32⌉`) no earlier round took; lane 0 writes each round's id to -/// `ids[at + s]`, then sums the `U` scores in f64 in slot order, takes -/// [`renorm_divisor`] of the sum narrowed to f32, and writes each weight -/// `score / sum · scale` to `weights[at + s]`, raising [`FaultSite::Router`] -/// on one that is not finite. With `REFUSE_NAN`, a non-finite selection -/// value writes NaN weights instead: [`take`] never selects a NaN, so the -/// ids stay the finite rank order and no slot's weight is a defined pick. +/// `ids[at + s]`, then sums the `U` scores in f64 in slot order. The divisor +/// is [`floor_divisor`] with `FLOOR`, [`renorm_divisor`] otherwise, of the +/// sum narrowed to f32. Each weight is `score / divisor · scale`, raising +/// [`FaultSite::Router`] on one that is not finite. With `REFUSE_NAN`, any +/// nonfinite selection value writes NaN weights instead, so the ids are +/// not a defined pick. /// /// # Safety /// @@ -104,7 +107,7 @@ pub unsafe fn publish_n( reason = "the selecting block's shared arrays, its lane and scale, the fault and the outputs" )] #[inline(always)] -pub unsafe fn select_n( +pub unsafe fn select_n( score: *const f32, sel_v: *const f32, slot_p: *mut f32, @@ -196,7 +199,11 @@ pub unsafe fn select_n( sum += f64::from(unsafe { *slot_p.add(s) }); s += 1; } - let sum = renorm_divisor(sum as f32); + let sum = if const { FLOOR } { + floor_divisor(sum as f32) + } else { + renorm_divisor(sum as f32) + }; let mut s = 0usize; while s < U { // SAFETY: s < U, this lane's own write. @@ -222,7 +229,7 @@ pub unsafe fn select_n( /// /// the grid is `E / ROWS_PER_BLOCK` blocks of [`ROUTER_THREADS`] /// (the launcher's), every thread arrives converged; `n_expert == E`, `w` -/// holds `n_expert · k` values, `x` `k`, `bias`, `logits` and `probs` +/// holds `n_expert · k` values, `x` `k`, any `bias`, `logits` and `probs` /// `n_expert`, `ids` and `weights` `U`, `done` one, zero before the launch; /// `k` is a positive multiple of 32; `last`, `score`, `sel_v` and `slot_p` /// are this block's own shared arrays of 1, `E`, `E` and `U` entries. @@ -231,10 +238,10 @@ pub unsafe fn select_n( reason = "a kernel entry's arguments and the shared arrays it declares (rust-quality R8)" )] #[inline(always)] -pub unsafe fn router_body( +pub unsafe fn router_body( w: &[f32], x: &[f32], - bias: &[f32], + bias: Option<&[f32]>, n_expert: u32, k: u32, scale: f32, @@ -297,7 +304,7 @@ pub unsafe fn router_body(score, sel_v, slot_p, lane, scale, fault, ids, weights, 0) + select_n::(score, sel_v, slot_p, lane, scale, fault, ids, weights, 0) }; if lane == 0 { // SAFETY: `ticket` is done[0], inside `done` by this fn's @@ -316,7 +323,7 @@ pub unsafe fn router_body( +pub unsafe fn pick_body( probs: &[f32], - bias: &[f32], + bias: Option<&[f32]>, n_expert: u32, t: u32, scale: f32, @@ -352,7 +359,7 @@ pub unsafe fn pick_body( // ids and weights hold U entries from U · token by this fn's // contract. unsafe { - select_n::( + select_n::( score, sel_v, slot_p, @@ -538,10 +545,10 @@ macro_rules! sigmoid_router_instance { // contract's, the launcher passes k a positive multiple of 32, // and `done[0]` is zero before the launch. unsafe { - router_body::( + router_body::( w, x, - bias, + Some(bias), n_expert, k, scale, @@ -599,9 +606,9 @@ macro_rules! sigmoid_router_instance { // reaches each `static mut` without a reference); the block is // 128 threads, converged; the lengths are the launch contract's. unsafe { - pick_body::( + pick_body::( probs, - bias, + Some(bias), n_expert, t, scale, @@ -851,7 +858,11 @@ macro_rules! sigmoid_router_instance { r: &$crate::routed_block::RouteIn<'_>, out: &mut RouterOut, ) -> Result<(), GpuError> { - self.enqueue_router(stream, r.w, x, r.bias, r.scale, out, r.fault) + let bias = r.bias.ok_or($crate::GpuError::State { + what: concat!(stringify!($m), "::route"), + missing: "the required selection bias", + })?; + self.enqueue_router(stream, r.w, x, bias, r.scale, out, r.fault) } fn route_rows( @@ -862,13 +873,17 @@ macro_rules! sigmoid_router_instance { t: usize, rows: $crate::routed_block::RoutedRows<'_>, ) -> Result<(), GpuError> { + let bias = r.bias.ok_or($crate::GpuError::State { + what: concat!(stringify!($m), "::route_rows"), + missing: "the required selection bias", + })?; self.enqueue_router_rows( stream, gpu, wide, r.w, x, - r.bias, + bias, r.scale, t, rows.probs, @@ -946,3 +961,343 @@ crate::sigmoid_router_instance! { refuse_nan: true, } } + +/// A fixed selected-sum-floor sigmoid binding over 256 experts and six +/// slots, with an actual optional selection-only bias. Shares the score, +/// publication, selection and sum order with the legacy instances above. +pub mod sigmoid_floor_256_6 { + use super::{ROUTER_THREADS_U32, ROWS_PER_BLOCK, pick_body, router_body}; + use crate::gemm::Gemm32Kernels; + use crate::route_core::Sigmoid; + use crate::routed_block::{RouteIn, RoutedRows, RouterEntry}; + use crate::{DeviceTensor, FaultSink, Gpu, GpuError, launch_u32}; + use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; + use cuda_device::{DisjointSlice, SharedArray, kernel, launch_bounds, launch_contract}; + use cuda_host::cuda_module; + use model::arch::models::shape::{self, RouterBody, RouterInst, rules}; + use std::sync::Arc; + + pub const N_EXPERT: usize = 256; + pub const N_USED: usize = 6; + /// Both metadata bindings name the same optional-bias device entries. + pub const ROW: RouterInst = + shape::router_row_rule(RouterBody::SigmoidFloor, rules::SIGMOID_FLOOR, 256, 6); + pub const BIASED_ROW: RouterInst = + shape::router_row_rule(RouterBody::SigmoidFloor, rules::BIASED_SIGMOID_FLOOR, 256, 6); + const _: () = assert!(shape::router_row_is(ROW, rules::SIGMOID_FLOOR, false, (6, 6))); + const _: () = assert!(shape::router_row_is( + BIASED_ROW, rules::BIASED_SIGMOID_FLOOR, false, (6, 6) + )); + const _: () = assert!(N_EXPERT.is_multiple_of(ROWS_PER_BLOCK)); + const _: () = assert!(N_EXPERT.div_ceil(32) <= 32); + + #[allow( + dead_code, + reason = "#[cuda_module] derives unchecked launchers that no caller uses" + )] + #[cuda_module] + mod sigmoid_floor_256_6_router_kernels { + use super::*; + + /// One token, with `has_bias` distinguishing an unused borrowed + /// input slice from a real selection bias. No zero-bias storage. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + w.len() >= 256 * k, + x.len() >= k, + bias.len() >= 256, + logits.len() >= 256, + probs.len() >= 256, + ids.len() >= 6, + weights.len() >= 6, + done.len() >= 1 + ) + )] + pub fn sigmoid_floor_256_6_router( + w: &[f32], + x: &[f32], + bias: &[f32], + has_bias: u32, + k: u32, + scale: f32, + mut logits: DisjointSlice, + mut probs: DisjointSlice, + mut ids: DisjointSlice, + mut weights: DisjointSlice, + mut done: DisjointSlice, + fault: FaultSink, + ) { + static mut SCORE: SharedArray = SharedArray::UNINIT; + static mut SEL_V: SharedArray = SharedArray::UNINIT; + static mut LAST: SharedArray = SharedArray::UNINIT; + static mut SLOT_P: SharedArray = SharedArray::UNINIT; + // SAFETY: the launch contract supplies every input/output length; + // the launcher uses k a positive multiple of 32, 64 blocks of 128 + // converged threads, and done is zero before each ordered launch. + // Shared arrays belong to this block and have the body's sizes. + unsafe { + router_body::( + w, x, if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, k, scale, + &mut logits, &mut probs, &mut ids, &mut weights, &mut done, fault, + SharedArray::as_raw_mut_ptr(&raw mut LAST), + SharedArray::as_raw_mut_ptr(&raw mut SCORE), + SharedArray::as_raw_mut_ptr(&raw mut SEL_V), + SharedArray::as_raw_mut_ptr(&raw mut SLOT_P), + ); + } + } + + /// Each block selects one token's scores, with the same optional + /// bias and floor as the one-token entry's final block. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + probs.len() >= 256 * t, + bias.len() >= 256, + ids.len() >= 6 * t, + weights.len() >= 6 * t + ) + )] + pub fn sigmoid_floor_256_6_router_pick( + probs: &[f32], + bias: &[f32], + has_bias: u32, + t: u32, + scale: f32, + mut ids: DisjointSlice, + mut weights: DisjointSlice, + fault: FaultSink, + ) { + static mut SCORE: SharedArray = SharedArray::UNINIT; + static mut SEL_V: SharedArray = SharedArray::UNINIT; + static mut SLOT_P: SharedArray = SharedArray::UNINIT; + // SAFETY: launch-contract lengths and 128 converged threads per + // block; shared arrays belong to this block at the body's sizes. + unsafe { + pick_body::( + probs, if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, t, scale, &mut ids, &mut weights, fault, + SharedArray::as_raw_mut_ptr(&raw mut SCORE), + SharedArray::as_raw_mut_ptr(&raw mut SEL_V), + SharedArray::as_raw_mut_ptr(&raw mut SLOT_P), + ); + } + } + } + + /// One token's expert logits/scores and six rank-ordered ids/weights. + /// Launches sharing this output must be ordered on one stream. + pub struct RouterOut { + pub logits: DeviceBuffer, + pub probs: DeviceBuffer, + pub ids: DeviceBuffer, + pub weights: DeviceBuffer, + done: DeviceBuffer, + } + + impl RouterOut { + /// Allocate result storage at load time, including the zero ticket. + pub fn new(stream: &CudaStream) -> Result { + Ok(Self { + logits: DeviceBuffer::zeroed(stream, N_EXPERT)?, + probs: DeviceBuffer::zeroed(stream, N_EXPERT)?, + ids: DeviceBuffer::zeroed(stream, N_USED)?, + weights: DeviceBuffer::zeroed(stream, N_USED)?, + done: DeviceBuffer::zeroed(stream, 1)?, + }) + } + + /// The ticket is zero between ordered launches, including replay. + pub fn tickets(&self, stream: &CudaStream) -> Result { + Ok(self.done.to_host_vec(stream)?[0]) + } + } + + /// Loaded floor router entries; all enqueue paths are allocation-free + /// and capturable, and borrow the optional bias rather than materialize it. + pub struct RouterKernels { + module: sigmoid_floor_256_6_router_kernels::LoadedModule, + } + + fn weight_k(what: &'static str, w: &DeviceTensor) -> Result { + let k = w.cols(); + if w.rows() != N_EXPERT || k == 0 || !k.is_multiple_of(32) { + return Err(GpuError::shape( + what, + format!("router weight is {} x {k}, want 256 rows of a positive multiple of 32", w.rows()), + )); + } + Ok(k) + } + + fn check_bias(what: &'static str, bias: Option<&DeviceBuffer>) -> Result<(), GpuError> { + if let Some(bias) = bias { + if bias.len() < N_EXPERT { + return Err(GpuError::shape( + what, format!("selection bias has {} entries, want at least 256", bias.len()), + )); + } + } + Ok(()) + } + + impl RouterKernels { + /// Load the embedded module once, before capture. + pub fn load(ctx: &Arc) -> Result { + // SAFETY: this crate owns the module's embedded device bundle. + let module = unsafe { crate::shared_module!(sigmoid_floor_256_6_router_kernels, ctx)? }; + Ok(Self { module }) + } + + /// Six largest sigmoid scores, larger id on ties. Bias changes + /// selection only; unbiased selected scores are summed in f64 slot + /// order, narrowed to f32, divided by max(sum, 2^-14), then scaled. + /// Nonfinite logits, bias-derived keys or weights raise Router and + /// an invalid key poisons every slot weight instead of routing it. + #[allow( + clippy::too_many_arguments, + reason = "the launcher's inputs are the kernel's (rust-quality R8)" + )] + pub fn enqueue_router( + &self, + stream: &CudaStream, + w: &DeviceTensor, + x: &DeviceBuffer, + bias: Option<&DeviceBuffer>, + scale: f32, + out: &mut RouterOut, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "sigmoid_floor_256_6::enqueue_router"; + let k = weight_k(what, w)?; + check_bias(what, bias)?; + if x.len() < k { + return Err(GpuError::shape(what, format!("x.len() {} < k {k}", x.len()))); + } + let k = launch_u32(what, "k", k)?; + let prep = self.module.prepare_sigmoid_floor_256_6_router( + LaunchConfig1D::new((N_EXPERT / ROWS_PER_BLOCK) as u32, ROUTER_THREADS_U32, 0), + )?; + let RouterOut { logits, probs, ids, weights, done } = out; + // With no bias the weight slice is only an ABI borrow: the + // kernel takes None and never loads it as a selection bias. + self.module.sigmoid_floor_256_6_router( + stream, &prep, w.buf(), x, bias.unwrap_or(w.buf()), u32::from(bias.is_some()), + k, scale, logits, probs, ids, weights, done, fault, + )?; + Ok(()) + } + + /// Token-major batch equivalent to ordered single-token launches. + /// The common F32 tile requires k a positive multiple of 64 and + /// aligned weight/input buffers. All buffers are caller-provided; + /// no optional-bias scratch is needed. + #[allow( + clippy::too_many_arguments, + reason = "router inputs, token count and the batch's output buffers (rust-quality R8)" + )] + pub fn enqueue_router_rows( + &self, + stream: &CudaStream, + gpu: &Gpu, + wide: &Gemm32Kernels, + w: &DeviceTensor, + x: &DeviceBuffer, + bias: Option<&DeviceBuffer>, + scale: f32, + t: usize, + probs: &mut DeviceBuffer, + ids: &mut DeviceBuffer, + weights: &mut DeviceBuffer, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "sigmoid_floor_256_6::enqueue_router_rows"; + let k = weight_k(what, w)?; + check_bias(what, bias)?; + let tt = launch_u32(what, "t", t)?; + let input = k.checked_mul(t).ok_or_else(|| GpuError::shape(what, "input size overflow"))?; + let scores = N_EXPERT.checked_mul(t).ok_or_else(|| GpuError::shape(what, "score size overflow"))?; + let slots = N_USED.checked_mul(t).ok_or_else(|| GpuError::shape(what, "slot size overflow"))?; + if t == 0 || x.len() < input || probs.len() < scores || ids.len() < slots || weights.len() < slots { + return Err(GpuError::shape( + what, + format!("{t} tokens of {k} (at least one): x {}, probs {}, ids {}, weights {}", + x.len(), probs.len(), ids.len(), weights.len()), + )); + } + // The tile GEMM uses the one-token row walk and warp sum tree. + wide.enqueue_f32_tile(stream, w, x, t, probs)?; + gpu.elem().enqueue_sigmoid_rows(stream, probs, scores)?; + let prep = self.module.prepare_sigmoid_floor_256_6_router_pick( + LaunchConfig1D::new(tt, ROUTER_THREADS_U32, 0), + )?; + self.module.sigmoid_floor_256_6_router_pick( + stream, &prep, probs, bias.unwrap_or(w.buf()), u32::from(bias.is_some()), + tt, scale, ids, weights, fault, + )?; + Ok(()) + } + } + + impl RouterEntry for RouterKernels { + const N_EXPERT: usize = N_EXPERT; + const N_USED: usize = N_USED; + const OPTIONAL_BIAS: bool = true; + type Out = RouterOut; + + fn load(ctx: &Arc) -> Result { + Self::load(ctx) + } + fn new_out(stream: &CudaStream) -> Result { + RouterOut::new(stream) + } + fn out_ids(out: &RouterOut) -> &DeviceBuffer { + &out.ids + } + fn out_weights(out: &RouterOut) -> &DeviceBuffer { + &out.weights + } + fn out_bytes(out: &RouterOut) -> usize { + out.logits.num_bytes() + out.probs.num_bytes() + out.ids.num_bytes() + + out.weights.num_bytes() + out.done.num_bytes() + } + fn route( + &self, + stream: &CudaStream, + x: &DeviceBuffer, + r: &RouteIn<'_>, + out: &mut RouterOut, + ) -> Result<(), GpuError> { + self.enqueue_router(stream, r.w, x, r.bias, r.scale, out, r.fault) + } + fn route_rows( + &self, + (stream, gpu, wide): (&CudaStream, &Gpu, &Gemm32Kernels), + x: &DeviceBuffer, + r: &RouteIn<'_>, + t: usize, + rows: RoutedRows<'_>, + ) -> Result<(), GpuError> { + self.enqueue_router_rows( + stream, gpu, wide, r.w, x, r.bias, r.scale, t, + rows.probs, rows.ids, rows.weights, r.fault, + ) + } + } +} diff --git a/crates/gpu/src/route_core.rs b/crates/gpu/src/route_core.rs index 4c00bfe9b..b32595b86 100644 --- a/crates/gpu/src/route_core.rs +++ b/crates/gpu/src/route_core.rs @@ -18,9 +18,9 @@ //! [`Sigmoid`]). What a router stores is [`router_score`] of the logit, never //! the score alone. //! -//! A weight stage that renormalizes its selected scores divides by -//! [`renorm_divisor`] of their sum: the one guard every such stage and its -//! host simulation share. +//! A weight stage uses either [`renorm_divisor`] (the legacy epsilon-added +//! policy) or [`floor_divisor`] (Flint's fixed selected-sum floor). The +//! compiled instance chooses the policy, never a per-token launch flag. use crate::elem::argmax_take; @@ -58,6 +58,12 @@ pub fn renorm_divisor(sum: f32) -> f32 { sum + 1e-20 } +/// Flint's selected-score divisor: the f32 sum floored at exactly `2^-14`. +#[inline(always)] +pub fn floor_divisor(sum: f32) -> f32 { + sum.max(1.0 / 16384.0) +} + /// The score a router stores for `logit`: `S::score(logit)`, or NaN for a /// logit that is not finite. A logit overflows to ±infinity when a row's /// finite products sum past f32, and the scores give that a finite value diff --git a/crates/gpu/src/routed_block.rs b/crates/gpu/src/routed_block.rs index b965c0b79..7ad972410 100644 --- a/crates/gpu/src/routed_block.rs +++ b/crates/gpu/src/routed_block.rs @@ -48,11 +48,11 @@ use crate::weights::Weights; use crate::{DeviceTensor, FaultSink, Gpu, GpuError}; /// What a router launch reads besides its activations: the layer's router -/// weight and selection bias, the file's `expert_weights_scale` and the -/// layer's fault sink. +/// weight and optional selection bias, the file's `expert_weights_scale` +/// and the layer's fault sink. pub struct RouteIn<'a> { pub w: &'a DeviceTensor, - pub bias: &'a DeviceBuffer, + pub bias: Option<&'a DeviceBuffer>, pub scale: f32, pub fault: FaultSink, } @@ -72,6 +72,8 @@ pub trait RouterEntry: Sized { const N_EXPERT: usize; /// Experts each token routes to. const N_USED: usize; + /// Whether an absent selection bias is a native binding, not a zero buffer. + const OPTIONAL_BIAS: bool = false; /// Where one token's launch leaves its results. type Out; @@ -111,7 +113,7 @@ pub struct RouteLayer { pub norm: String, /// The router weight. pub router: String, - /// The selection bias; `None` routes with a zero bias. + /// The selection bias; `None` routes without a correction bias. pub bias: Option, /// The layer's row in the slot map's card copy, as a word offset /// (`SlotMap::row_offset`). @@ -141,7 +143,7 @@ pub struct RouteBlock { handoff: HandoffKernels, /// The wide F32 tile GEMM the rows router's scores run on. g32: Gemm32Kernels, - /// The zero selection bias of a layer the file gives none. + /// Legacy bindings' zero selection bias for a layer the file gives none. no_bias: Option>, } @@ -270,7 +272,7 @@ impl RouteBlock { Some(name) => { w.f32_buf(what, name)?; } - None => zero_bias = true, + None => zero_bias |= !R::OPTIONAL_BIAS, } } let ctx = gpu.context(); @@ -335,7 +337,11 @@ impl RouteBlock { gpu: &Gpu, ) -> Result, GpuError> { let layer = self.layer(l)?; - let bias = self.bias_of(w, l)?; + let bias = if layer.bias.is_none() && R::OPTIONAL_BIAS { + None + } else { + Some(self.bias_of(w, l)?) + }; Ok(RouteIn { w: w.f32_tensor(self.what, &layer.router)?, bias, diff --git a/crates/model/src/arch/flint/tests.rs b/crates/model/src/arch/flint/tests.rs index ac2bca5dc..698806d85 100644 --- a/crates/model/src/arch/flint/tests.rs +++ b/crates/model/src/arch/flint/tests.rs @@ -283,10 +283,15 @@ fn normalization_policy_is_architecture_specific_and_bias_is_selection_only() { let shape = models::shape::MoeShape::of(m); assert_eq!(shape.rule.norm, expected); assert_eq!(shape.rule.bias, bias); - assert_eq!( - models::shape::select_router(shape).unwrap_err().why, - models::shape::Refusal::NoRouterRule - ); + let refusal = models::shape::select_router(shape).unwrap_err().why; + if norm { + assert!(matches!( + refusal, + models::shape::Refusal::RouterWidth { ref served } if served == &[256] + )); + } else { + assert_eq!(refusal, models::shape::Refusal::NoRouterRule); + } assert!( uncovered .iter() diff --git a/crates/model/tests/flint_meta.rs b/crates/model/tests/flint_meta.rs index d6121431c..bde81addf 100644 --- a/crates/model/tests/flint_meta.rs +++ b/crates/model/tests/flint_meta.rs @@ -109,12 +109,10 @@ fn hw_flint_header() { let coverage = model::arch::coverage::check(s, &r.tensors); for (need, layer) in &needs { if let Need::Router(shape) = need { - assert_eq!( - model::arch::models::shape::select_router(*shape) - .unwrap_err() - .why, - model::arch::models::shape::Refusal::NoRouterRule - ); + let binding = model::arch::models::shape::select_router(*shape).unwrap(); + assert_eq!(binding.body, model::arch::models::shape::RouterBody::SigmoidFloor); + assert_eq!(binding.rule, shape.rule); + assert_eq!((binding.experts(), binding.top_k_min, binding.top_k_max), (256, 6, 6)); assert!(coverage.iter().any(|u| { u.layer == layer.map(|l| l as usize) && u.feature == need.to_string() })); diff --git a/crates/models/src/shape.rs b/crates/models/src/shape.rs index b70773254..6fd7f1592 100644 --- a/crates/models/src/shape.rs +++ b/crates/models/src/shape.rs @@ -83,6 +83,9 @@ pub enum RouterBody { /// `mimo2_router*`, `sigmoid_320_8_router*`, `sigmoid_128_4_router*`; /// `gpu/src/arch/nemotron_h_moe/router.rs` `nemotron_moe_router_*`. BiasedSigmoid, + /// `gpu/src/route_body.rs` `sigmoid_floor_256_6_router*`: sigmoid, + /// optional selection-only bias and a fixed `2^-14` selected-sum floor. + SigmoidFloor, } /// One compiled router instance. @@ -160,6 +163,16 @@ const BIASED_SIGMOID: RouterRule = RouterRule { norm: RouterNorm::SelectedSum, gated: false, }; +const SIGMOID_FLOOR: RouterRule = RouterRule { + score: Score::Sigmoid, + bias: false, + norm: RouterNorm::SelectedSumFloor, + gated: false, +}; +const BIASED_SIGMOID_FLOOR: RouterRule = RouterRule { + bias: true, + ..SIGMOID_FLOOR +}; const BIASED_SQRT_SOFTPLUS: RouterRule = RouterRule { score: Score::SqrtSoftplus, bias: true, @@ -326,6 +339,24 @@ pub const ROUTERS: &[RouterInst] = &[ top_k_max: 6, at: "gpu/src/route_body.rs sigmoid_256_6_router*", }, + RouterInst { + body: RouterBody::SigmoidFloor, + rule: SIGMOID_FLOOR, + norm_arg: false, + experts: 256, + top_k_min: 6, + top_k_max: 6, + at: "gpu/src/route_body.rs sigmoid_floor_256_6_router*", + }, + RouterInst { + body: RouterBody::SigmoidFloor, + rule: BIASED_SIGMOID_FLOOR, + norm_arg: false, + experts: 256, + top_k_min: 6, + top_k_max: 6, + at: "gpu/src/route_body.rs sigmoid_floor_256_6_router*", + }, // Muse Spark's fixture: 16 experts, the real file's top 8. RouterInst { body: RouterBody::BiasedSigmoid, @@ -531,6 +562,31 @@ pub const fn router_row_k(body: RouterBody, experts: u32, top_k: u32) -> RouterI panic!("no router row for this body at this width and pick count") } +/// A compiled binding whose body, rule, expert count and pick count all +/// agree. Unlike [`router_row_k`], distinguishes optional-bias bindings. +#[must_use] +pub const fn router_row_rule( + body: RouterBody, + rule: RouterRule, + experts: u32, + top_k: u32, +) -> RouterInst { + let mut i = 0; + while i < ROUTERS.len() { + let r = ROUTERS[i]; + if body_eq(r.body, body) + && r.runs(rule) + && r.experts == experts + && r.top_k_min <= top_k + && top_k <= r.top_k_max + { + return r; + } + i += 1; + } + panic!("no router row for this body, rule, width and pick count") +} + /// Whether row `r` is body `body` at `per_lane`, with rule `rule` and the /// pick range `min..=max`: the whole of what a device crate compiled in. #[must_use] @@ -564,6 +620,11 @@ pub mod rules { /// Sigmoid scores, a selection bias steering the choice only, the kept /// unbiased scores renormalized: GLM-5.3's and MiMo-V2's noaux_tc router. pub const BIASED_SIGMOID: RouterRule = super::BIASED_SIGMOID; + /// Sigmoid, no selection bias, selected scores divided by + /// `max(sum_selected, 2^-14)`. + pub const SIGMOID_FLOOR: RouterRule = super::SIGMOID_FLOOR; + /// [`SIGMOID_FLOOR`] with a bias steering selection only. + pub const BIASED_SIGMOID_FLOOR: RouterRule = super::BIASED_SIGMOID_FLOOR; } const fn norm_eq(a: RouterNorm, b: RouterNorm) -> bool { @@ -937,66 +998,62 @@ mod tests { assert!(matches!(e.why, Refusal::RouterWidth { ref served } if served == &[128])); } + /// Legacy rows retain their epsilon-added or unnormalized arithmetic; + /// only the real floor binding accepts the floor policy. #[test] - fn a_selected_sum_floor_is_not_a_legacy_device_policy() { + fn floor_bindings_are_distinct_from_legacy_policies() { for row in ROUTERS { - let rule = RouterRule { - norm: RouterNorm::SelectedSumFloor, - ..row.rule - }; assert_eq!( select_router(moe(row.rule, row.experts(), row.top_k_min)).unwrap(), *row ); - assert!(!row.runs(rule)); - assert!(!router_row_is( - *row, - rule, - row.norm_arg, - (row.top_k_min, row.top_k_max) - )); - let shape = moe(rule, row.experts(), row.top_k_min); - let refusal = select_router(shape).expect_err("a legacy body bound the floor policy"); - assert_eq!(refusal.shape, Shape::Moe(shape)); - assert_eq!(refusal.why, Refusal::NoRouterRule); + if row.body == RouterBody::SigmoidFloor { + assert_eq!(row.rule.norm, RouterNorm::SelectedSumFloor); + assert!(!row.norm_arg); + } else { + assert!(!row.runs(RouterRule { + norm: RouterNorm::SelectedSumFloor, + ..row.rule + })); + } } } - /// Flint's sigmoid top-six is not the biased top-eight binding, and no - /// compiled body runs its rule: the 256-expert top-6 instance runs the - /// biased sigmoid with the epsilon-added `SelectedSum`, so every other - /// pair of bias and policy, Flint's `SelectedSumFloor` with or without a - /// selection bias among them, is refused by name. - // PIN(2026-10-25): the biased `SelectedSum` pair was refused as - // `TopK { min: 8, max: 8 }` while only MiMo's top-8 row held 256 experts; - // the `sigmoid_256_6` instance's row now serves exactly that pair - // (`sigmoid_256_6_router_is_its_row_beside_mimos_top_8` pins its identity). - // The other five pairs keep `NoRouterRule`. #[test] - fn flints_sigmoid_top_six_is_not_the_biased_top_eight_binding() { + fn floor_top_six_serves_both_bias_modes_and_only_its_shape() { for bias in [false, true] { - for norm in [ - RouterNorm::None, - RouterNorm::SelectedSum, - RouterNorm::SelectedSumFloor, - ] { - let rule = RouterRule { - bias, - norm, - ..rules::BIASED_SIGMOID - }; - let selected = select_router(moe(rule, 256, 6)); - if bias && norm == RouterNorm::SelectedSum { - assert!( - selected.is_ok(), - "the 256/6 instance's own rule: {selected:?}" - ); - continue; - } - let refusal = selected.expect_err("a sigmoid top-six binding was fabricated"); - assert_eq!(refusal.why, Refusal::NoRouterRule); + let rule = RouterRule { bias, ..rules::SIGMOID_FLOOR }; + let row = select_router(moe(rule, 256, 6)).unwrap(); + assert_eq!(row.body, RouterBody::SigmoidFloor); + assert_eq!(row.rule, rule); + assert_eq!(row, router_row_rule(RouterBody::SigmoidFloor, rule, 256, 6)); + for top_k in [0, 5, 7, 8, 257] { + assert_eq!( + select_router(moe(rule, 256, top_k)).unwrap_err().why, + Refusal::TopK { min: 6, max: 6 } + ); + } + assert!(matches!( + select_router(moe(rule, 128, 6)).unwrap_err().why, + Refusal::RouterWidth { ref served } if served == &[256] + )); + for (score, gated) in [(Score::Softmax, false), (Score::Sigmoid, true)] { + let wrong = RouterRule { score, gated, ..rule }; + assert_eq!( + select_router(moe(wrong, 256, 6)).unwrap_err().why, + Refusal::NoRouterRule + ); } } + let legacy = select_router(moe(rules::BIASED_SIGMOID, 256, 6)).unwrap(); + assert_eq!(legacy.body, RouterBody::BiasedSigmoid); + assert_eq!( + select_router(moe(RouterRule { + norm: RouterNorm::SelectedSum, + ..rules::SIGMOID_FLOOR + }, 256, 6)).unwrap_err().why, + Refusal::NoRouterRule + ); } /// GLM-5.3-Flash (model/tests/glm5next_meta.rs:83, "router: sigmoid, 288 diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index b0a9b8b4e..5eb89f92d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,9 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. The current tree additionally has a **header-only reader** and a real common +256/top6 floor-router binding; it still has no Flint body, app session or serve seat. +No GPU, inference, performance or weight-content validation is claimed. ## Host reader status @@ -15,10 +16,10 @@ GemmaEmbedding2's sliding mask: missing keys, scalars, wrong lengths and non-boo Present `expert_weights_scale` uses `arch::finite_positive`; only an absent key defaults to 1, with that default recorded. The common needs walk emits q/k normalization even on NoPE layers. `program_of(Flint)` remains -`None`; missing program, head128 sinks/SWA, NoPE normalization and 256/top6 bindings remain coverage -refusals. `Router.norm` and `RouterRule.norm` distinguish no normalization, the existing selected-sum -rule and Flint's selected-sum floor; the floor description does not supply a device implementation -or activated-value scaling. +`None`; missing program, head128 sinks/SWA and NoPE normalization remain coverage refusals. +`Router.norm` and `RouterRule.norm` distinguish no normalization, the existing selected-sum +rule and Flint's selected-sum floor. The real floor-router binding is selectable in both bias +modes; selecting that operation does not supply a whole program or activated-value scaling. Fault formatting uses `bloomery_device::fault`'s shared layout/decoder and the `gpu::arch::step_order` registry. Flint's registry row uses neutral code order; @@ -50,19 +51,41 @@ No full weights or GPU were needed for that check. The maintainer [assigned the model port and offered card runs](https://github.com/midagedev/bloomery/issues/632#issuecomment-6352066886), and named a forthcoming head128 sink/SWA attention, QK RMS and sigmoid256/top6 train. -Those complete contracts are not in this main snapshot or the inspected public heads of #726 (`1723485602c1`) -and #747 (`f883e84f8d2c`). Existing ring and sink-merge ingredients do not establish the combined attention -operation; `route_body` still lacks 256/top6 and its normalization-floor policy. -Obtain the train's fetchable ref before duplicating those primitives. - -The router description/selection contract now distinguishes `RouterNorm::None`, `SelectedSum` and -`SelectedSumFloor`. Flint requests `max(sum_selected, 2^-14)`; existing model descriptions and arithmetic -keep their rules. Compiled rows, including boolean `norm_arg` bindings, refuse the floor policy rather -than borrow an epsilon-added implementation. The absent-bias, 256/top6 and `Need::Program` refusals stay -until real device bindings are supplied. Reconcile the policy interface with the pending train before device work. -Then close shared IQ4_XS, selected card/tier down forms and six-slot F32 handoff in their existing common owners, +Those complete attention contracts are not in this main snapshot or the inspected public heads of +#726 (`1723485602c1`) and #747 (`f883e84f8d2c`). Existing ring and sink-merge ingredients do not +establish the combined attention operation. Obtain the train's fetchable ref before duplicating +those primitives. + +The common `route_body::sigmoid_floor_256_6` binding now supplies Flint's fixed +`max(sum_selected, 2^-14)` rule. `RouterKernels::enqueue_router` and `enqueue_router_rows` +take `Option<&DeviceBuffer>` for a real optional selection-only bias. The common +`RouterEntry`/`RouteIn` consuming paths carry that option; `RouteBlock` allocates no zero-bias +buffer for this binding. Absent bias borrows the existing router-weight slice only as an unused +device ABI argument; no bias data is fabricated, copied or read. The floor policy is a compile-time +specialization of the same score, publication and top-k bodies, not a per-step normalization flag. + +Both metadata bindings are `RouterBody::SigmoidFloor` at exactly 256/top6: sigmoid, no shared +sigmoid gate, `SelectedSumFloor`, with or without selection bias. Bias changes only the ranking. +The six selected unbiased scores are summed in f64 slot order, narrowed to f32, divided by the +floored sum and then scaled. Equal selection keys choose larger expert ids. Nonfinite logits, +selection keys or resulting weights raise `FaultSite::Router`; an invalid selection key poisons +all six weights. Existing epsilon-added entries, launch ABIs and legacy norm-argument rules remain +unchanged. A legacy `RouterEntry` given no required bias returns a named state error. + +`gate-flint-router` now consumes the actual floor binding through `RouterEntry` for planted and +dense rows, both bias modes, active tiny-sum floors, zero sums, ties, nonfinite logits/bias/scale, +one-token/batch bit equivalence and eager/captured replay. Its independent host weight expression +uses the declared floor and rejects an epsilon-added or biased-weight implementation. +**needs-gpu:** this implementation has no local NVIDIA numerical, capture, fault, allocation or +JIT-resource evidence. The two new entries (`sigmoid_floor_256_6_router` and +`sigmoid_floor_256_6_router_pick`) have no invented resource pins; the add-class PTX comparison, +spill/JIT pinning and hardware gate remain required. + +The `Need::Program` refusal stays: a callable router is not a Flint body or session. Close shared +IQ4_XS, selected card/tier down forms and six-slot F32 handoff in their existing common owners, followed by paging/resident slots, reference provenance and the body/app/seat. -Each device slice needs its numerical/fault gates and the maintainer's offered hardware proof before admission. +Each device slice needs its numerical/fault gates and the maintainer's offered hardware proof before +whole-program admission. Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. @@ -230,13 +253,12 @@ source of the gathered weights. The shared SwiGLU output is added without a shar Three numerical obligations must stay explicit: -1. Current [common router normalization][route-core] uses `sum + 1e-20`, not the reference's floor. +1. Legacy [common router normalization][route-core] uses `sum + 1e-20`, not the reference's floor. This is a semantic difference for tiny scores, not a width-instance problem or a measured GPU - failure. The existing `Router`/`RouterRule` contract now carries the explicit policy, and the - reader requests `SelectedSumFloor` for Flint. Selection refuses that policy on every current - router row; `norm_arg` serves only the two legacy policies. The device floor binding is still - missing. Add it in the common operation without changing the global divisor, existing model - arithmetic, or hiding the difference in a Flint-only wrapper. + failure. `Router`/`RouterRule` carries the explicit policy; the reader requests + `SelectedSumFloor` and the common `sigmoid_floor_256_6` binding implements it. Legacy instances + keep their exact arithmetic and `norm_arg` serves only the two legacy policies. The floor + gate checks active tiny sums independently; GPU numerical and resource evidence is still needed. 2. Mainline [rescales Flint's activated expert input before down projection][ref-down] with `build_amax_src1_f16`, then restores its scale before router weighting. This protects F16/Q8 headroom around a maximum magnitude of 1024; it is not an architectural activation clamp. @@ -272,7 +294,7 @@ Do not claim runtime support by adding the name to an enum or relaxing a shape/t |---|---|---| | G1 | NoPE normalization-need generation is present; the executable head128 norm-only binding is missing | Preserve the emitted normalization **or** rotation need; bind head128 normalization with `mode=None` truthfully. No identity-RoPE table. | | G2 | Existing head128/group8 attention lacks the complete sink/SWA contract; MiMo's window pass is K192/V128 | Complete the described head128 operation in `rope_neox` / `flash_gqa`, including genuine NoPE append, sink merge, window/ring positions, prompt and slot paths. Never pad heads to 192 or copy MiMo's body. | -| G3 | The host description/selection policy is explicit; sigmoid256/top6 and its device floor binding are absent, and higher-level `shared_routed` admission requires bias | Add the real instance/floor operation; bind absent bias through `RouteLayer.bias=None`, which the lower `RouteBlock` already supports. Price/load the BF16 router through its supported F32 widening. | +| G3 | Real common sigmoid256/top6 floor operation and both optional-bias metadata bindings are implemented; whole-program binding remains separate | Attach `sigmoid_floor_256_6` through `RouterEntry`, with `RouteLayer.bias=None` retaining genuine absence and no zero buffer. Price/load the BF16 router through its supported F32 widening. Run `gate-flint-router` and obtain genuine new-entry JIT pins on NVIDIA hardware. | | G4 | Layer42 shared IQ4_XS has routed/GEMM operations but no ordinary dense site, and `BlockPlan` rejects it | Extend the **common** dense/gated block's real layout, load, launch, scratch and prompt binding. Keep the role `SharedExpert`; do not relabel it as routed or silently requantize it. | | G5 | The common card wrapper uses Q8Act-only buffers and `LegStacks::of`; Q6_K/Q8_0 down do not both pass that path | Carry selected arms and actual down forms through `CardExperts`, `LegBufs::for_downs`, loader, planner, residency and tier. Existing Q6_K/Q8_0 arithmetic is not the missing piece. | | G6 | Common tier accumulation supports six slots, but F32 `enqueue_handoff_tier` accepts eight | Add the six-slot F32 handoff instance in the existing owner; preserve the eight-slot entry. A DeepSeek Q8_1 handoff is not an interchangeable input contract. | From 1c9432b84e0c6d89945ad15fb3660e2248c3bc4b Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:51:45 +0900 Subject: [PATCH 18/85] gpu: complete current-main selected tier prerequisite cutover --- crates/gpu-gates/src/bin/gate_mimo2_serve.rs | 5 +++-- .../src/bin/gate_musespark_prefill.rs | 2 +- .../gpu-gates/src/bin/shared/mimo2_twocard.rs | 2 +- .../gpu-gates/src/bin/shared/minimaxm3_seat.rs | 4 ++-- .../gpu-gates/src/bin/shared/qwen3moe_place.rs | 4 ++-- .../src/bin/shared/serve_seats/mimo2.rs | 6 ++++-- .../src/bin/shared/serve_seats/minimaxm3.rs | 9 +++++++-- .../src/bin/shared/serve_seats/musespark.rs | 9 +++++++-- crates/gpu-mimo2/src/tier.rs | 5 +++-- crates/gpu-minimaxm3/src/tier.rs | 5 +++-- crates/gpu/src/host/handoff.rs | 18 +++++++++--------- crates/model/src/arch/mimo2/place.rs | 6 ++++-- crates/model/src/arch/minimaxm3/place.rs | 4 ++-- crates/model/src/arch/musespark/place.rs | 1 + crates/model/tests/mimo2_meta.rs | 2 +- crates/model/tests/musespark_meta.rs | 2 +- crates/model/tests/qwen3_bp_plan.rs | 8 ++++++-- crates/placement/src/placement.rs | 5 +++-- crates/placement/src/placement/card_budget.rs | 1 + docs/research/flint-attach.md | 8 +++++++- 20 files changed, 68 insertions(+), 38 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs index 2245c05d3..da2c6271b 100644 --- a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs +++ b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs @@ -892,8 +892,9 @@ mod gate { let mut inputs = PlanInputs::describe(&split)?; inputs.room = (PLAN_ROOM, HostRead::Given); let place = unset_place(cards).on_host()?; - let batch = - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)); + let batch = (!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)) + .transpose()?; let machine = place.machine(None, batch)?(inputs.model.layers); let slots = NonZeroUsize::new(usize::try_from(DEFAULT_SLOTS)?).ok_or("no default slot")?; let plan = diff --git a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs index 6674f694b..5137e5828 100644 --- a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs @@ -530,7 +530,7 @@ mod gate { tier_now: Tier, ) -> Result<(Model, bool, Vec), GateError> { let inputs = PlanInputs::read(&open_split(Arch::MuseSpark, RECIPE)?)?; - let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp))?; + let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp)?)?; place::stage_placed(&mut two)?; let ctx = u64::try_from(ctx)?; // The stage card alone, the reference's machine: the budget reads its unbudgeted plan, which a plan of the two diff --git a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs index 3b8425314..cbf918642 100644 --- a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs +++ b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs @@ -111,7 +111,7 @@ pub fn run(path: &Path, levers: &bloomery_levers::Levers) -> Result Result workstation::placed_with_arena(*stage, layers, arena), [stage, tier] => { let batch = match inputs { - Inputs::Qwen3(i) => q3::tier_batch(&i.hp), - Inputs::Qwen35(i) => q35::leg_tier_batch(&i.hp), + Inputs::Qwen3(i) => q3::tier_batch(&i.hp)?, + Inputs::Qwen35(i) => q35::leg_tier_batch(&i.hp)?, }; let mut machine = workstation::plan_tiers(layers, *stage, &[*tier], None, batch)?; q3::stage_placed(&mut machine, arena); diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs index 240784af9..039f020c4 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs @@ -504,8 +504,10 @@ impl Tiered for Mimo { /// The tier card's prompt-batch bytes ([`bloomery_gpu_mimo2::tier::tier_batch`]), under a /// placement with one. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs)) + fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs)) + .transpose()?) } fn flags(flag: &str, value: &str, out: &mut Flags) -> Result { diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs index ec2676feb..4c4654de9 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs @@ -427,8 +427,13 @@ impl Tiered for MiniMax { /// The tier's prompt-batch bytes of the file's widths and down forms over the body's prompt /// unit ([`bloomery_gpu_minimaxm3::tier::tier_batch`]), reserved on the tier card and the /// host; `None` on a placement with no tier card. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs)) + .transpose()?) } /// The tags the model's description names for its think span. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs index 73ac85ac8..7cf800441 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs @@ -384,8 +384,13 @@ impl Tiered for Muse { /// The tier's prompt-batch bytes of the file's widths /// ([`place::tier_batch`]), reserved on the tier card and the host; `None` /// on a placement with no tier card. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(&inputs.hp)) + .transpose()?) } /// The ring is kept out of the expert budget where the common rule starts diff --git a/crates/gpu-mimo2/src/tier.rs b/crates/gpu-mimo2/src/tier.rs index 748c7889e..4a431ba3f 100644 --- a/crates/gpu-mimo2/src/tier.rs +++ b/crates/gpu-mimo2/src/tier.rs @@ -34,7 +34,7 @@ use gguf::Split; use model::ModelError; use model::arch::mimo2::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; +use model::placement::{ModelTensors, PlacementError}; use model::placement::workstation::TierBatchBytes; use model::r8file::R8Source; use models::LayerSpec; @@ -57,7 +57,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. #[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(&inputs.hp, T_MAX as u64) } @@ -91,6 +91,7 @@ impl LegFamily for Mimo2 { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: model::placement::accepts::Program::Mimo2Body, } } diff --git a/crates/gpu-minimaxm3/src/tier.rs b/crates/gpu-minimaxm3/src/tier.rs index c591db72c..b937c9291 100644 --- a/crates/gpu-minimaxm3/src/tier.rs +++ b/crates/gpu-minimaxm3/src/tier.rs @@ -20,7 +20,7 @@ use gguf::Split; use model::arch::minimaxm3::host as minimax_host; use model::arch::minimaxm3::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; +use model::placement::{ModelTensors, PlacementError}; use model::placement::workstation::TierBatchBytes; use model::r8file::R8Source; use models::{LayerSpec, Moe}; @@ -39,7 +39,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. #[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(inputs, T_MAX as u64) } @@ -73,6 +73,7 @@ impl LegFamily for MiniMax { n_used: experts.n_used, n_embd: inputs.hp.n_embd, ff: experts.ff, + program: model::placement::accepts::Program::MiniMaxM3Body, } } diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index e42f46a61..51df65d8d 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,9 +18,9 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ffn_handoff_6_tier` and -//! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an -//! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into +//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_4_tier`, +//! `ffn_handoff_6_tier` and `ds41_ffn_handoff_8_tier` hand over four, six or +//! eight slots of a tier layer: the same writes, its tier places and f32 activation into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); //! `ds41_ffn_handoff_10_cols_tier` is `_10_cols`'s of such a layer, every //! column's tier places and activation into the row's tier image @@ -1346,8 +1346,8 @@ impl HandoffKernels { } impl HandoffKernels { - /// Enqueue the handoff of a tier layer of six or eight slots a token - /// (`ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// Enqueue the handoff of a tier layer of four, six or eight slots a token + /// (`ds41_ffn_handoff_4_tier`, `ffn_handoff_6_tier`, `ds41_ffn_handoff_8_tier`): /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and @@ -1640,7 +1640,7 @@ fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result u64 { hp.expert_ff as u64, hp.n_used as u64, cap as u64, + DownForms::of(DownAct::Q8Act), ) as usize; leg + 2 * size_of::() * cap * n + size_of::() * cap * hp.n_used } else { @@ -268,7 +270,7 @@ mod tests { let _ = std::fs::remove_file(&path); let hp = &inputs.hp; assert_eq!( - tier_batch(hp, COLS), + tier_batch(hp, COLS).expect("tier bytes"), workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, @@ -276,7 +278,7 @@ mod tests { TIER_CHUNK, COLS, DownForms::of(DownAct::Q8Act), - ) + ).expect("tier bytes") ); let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS).expect("tier bytes")); diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 64b6a8407..9ee262b4f 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -435,7 +435,7 @@ mod tests { let inputs = described("mm3-place-tier"); let hp = &inputs.hp; assert_eq!( - tier_batch(&inputs, COLS), + tier_batch(&inputs, COLS).expect("tier bytes"), workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, @@ -443,7 +443,7 @@ mod tests { TIER_CHUNK, COLS, DownForms::of(DownAct::Q8Act), - ) + ).expect("tier bytes") ); let tiered = workstation::plan_bp( LAYERS, None, tier_batch(&inputs, COLS).expect("tier bytes"), diff --git a/crates/model/src/arch/musespark/place.rs b/crates/model/src/arch/musespark/place.rs index de91cf878..273ce4c2b 100644 --- a/crates/model/src/arch/musespark/place.rs +++ b/crates/model/src/arch/musespark/place.rs @@ -629,6 +629,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, diff --git a/crates/model/tests/mimo2_meta.rs b/crates/model/tests/mimo2_meta.rs index 64ecda04a..db975961b 100644 --- a/crates/model/tests/mimo2_meta.rs +++ b/crates/model/tests/mimo2_meta.rs @@ -401,7 +401,7 @@ fn hw_mimo2_plan() { let tiered = workstation::plan_bp( layers, None, - model::arch::mimo2::place::tier_batch(&inputs.hp, 512), + model::arch::mimo2::place::tier_batch(&inputs.hp, 512).expect("tier batch bytes"), ); let bp = inputs .plan(&tiered, ctx, &PlanLevers::default()) diff --git a/crates/model/tests/musespark_meta.rs b/crates/model/tests/musespark_meta.rs index d3cabdd9c..e765e3d56 100644 --- a/crates/model/tests/musespark_meta.rs +++ b/crates/model/tests/musespark_meta.rs @@ -368,7 +368,7 @@ fn hw_musespark_plan() { workstation::A6000, &[workstation::RTX_3090], None, - place::tier_batch(&hp), + place::tier_batch(&hp).expect("tier batch bytes"), ) .unwrap_or_else(|e| panic!("the tiered machine: {e}")); place::stage_placed(&mut m).unwrap_or_else(|e| panic!("the tiered machine's stage: {e}")); diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index dc4fb39a6..972d82125 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -528,7 +528,9 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); - let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.hp)); + let machine = tiered( + Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.hp).expect("tier batch bytes"), + ); let plan = inputs.plan_rule( &machine, CTX, @@ -543,7 +545,9 @@ fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place #[test] fn qwen3_30b_q8_bp_tier_experts() { let inputs = qwen3_inputs(); - let machine = tiered(Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp)); + let machine = tiered( + Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp).expect("tier batch bytes"), + ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) .expect("the A6000 stage plans a tier"); diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 2500304ac..68156e9c9 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -2769,7 +2769,8 @@ fn resolve_shares( let uploads = card_uploads(model, c, rows, &own, routed)?; let dense = footprint(card.granule_bytes, &uploads, &zeros)?; let kv = kv_card.saturating_add(stage_rows); - let floor = plan_floor(card, dense, kv, reserve); + let reserves = card.selected_reserve_bytes(model, &zeros)?; + let floor = plan_floor(card, dense, kv, reserve, reserves); let cap = capped(card, None); let Some((floor, room)) = floor.and_then(|f| Some((f, cap.checked_sub(f)?))) else { return Err(refuse(format!( @@ -2781,7 +2782,7 @@ fn resolve_shares( kv, card.context_bytes, card.scratch_bytes, - card.reserve_bytes().unwrap_or(u64::MAX), + reserves, card.margin_bytes, ))); }; diff --git a/crates/placement/src/placement/card_budget.rs b/crates/placement/src/placement/card_budget.rs index 3f8d2e734..52830b93c 100644 --- a/crates/placement/src/placement/card_budget.rs +++ b/crates/placement/src/placement/card_budget.rs @@ -209,6 +209,7 @@ mod tests { fn card(name: &str, ordinal: Option) -> Card { Card { name: name.to_owned(), + tier_leg: None, device: ordinal.map(|o| DeviceId { ordinal: o, uuid: [u8::try_from(o).expect("small") + 1; 16], diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 5eb89f92d..64ad6ee9b 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -128,6 +128,12 @@ Host-only projections intentionally have no card totals: fixed reserves remain o Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. +The current-main cutover keeps reserve-only draft holders, per-card budgets and each +family's activation contract. `LegTier` receives its activation converter at load; +selected `Program` arms replace the redundant family arm hook. Qwen, Muse, MiMo and +MiniMax sizing callers propagate checked tier descriptors rather than bypassing +their validation. Share-budget floors use the same selected reserve calculation. + A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an @@ -135,7 +141,7 @@ exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain abse ### Six-slot F32 tier handoff implementation `HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry -beside the existing eight-slot entry. Both host and tier layouts must have the +beside the existing four- and eight-slot entries. Both host and tier layouts must have the same supported slot count and activation width; compressed tier images remain refused. The entry uses the existing copy/place/fault helpers and the existing subsequent go/publication protocol; no Flint router or Program is registered. From e28830ddfa136761258f74fca35f905a29362368 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:17:57 +0900 Subject: [PATCH 19/85] feat(flint): attach floor router, ring attention and hosted runtime --- Cargo.lock | 16 + Cargo.toml | 2 +- crates/app/Cargo.toml | 2 + crates/app/src/arch/flint.rs | 76 ++ crates/app/src/arch/minimaxm3/mod.rs | 8 +- crates/app/src/arch/mod.rs | 106 ++- crates/app/src/arch/musespark/mod.rs | 4 +- crates/app/src/arch/solar2/mod.rs | 4 +- crates/app/src/lib.rs | 9 +- crates/gpu-deepseek41/src/experts_mxfp4.rs | 4 +- crates/gpu-deepseek41/src/prompt_walk.rs | 18 +- crates/gpu-deepseek41/src/shared_routed.rs | 366 ++++++-- crates/gpu-deepseek41/src/tier_join.rs | 16 +- crates/gpu-flint/Cargo.toml | 19 + crates/gpu-flint/src/body.rs | 583 ++++++++++++ crates/gpu-flint/src/lib.rs | 13 + crates/gpu-flint/src/prefill.rs | 128 +++ crates/gpu-flint/src/step.rs | 183 ++++ crates/gpu-flint/src/tests.rs | 252 +++++ crates/gpu-flint/src/tier.rs | 65 ++ crates/gpu-gates/Cargo.toml | 4 +- crates/gpu-gates/src/bin/bloomery_serve.rs | 27 +- crates/gpu-gates/src/bin/gate_flint_attn.rs | 318 +++++++ crates/gpu-gates/src/bin/gate_flint_router.rs | 302 ++++-- .../gpu-gates/src/bin/gate_solar2_twocard.rs | 1 + .../src/bin/shared/serve_seats/hybrid.rs | 42 +- .../src/bin/shared/serve_seats/mod.rs | 4 +- .../src/bin/shared/serve_seats/tiered.rs | 11 +- crates/gpu-gates/src/record.rs | 68 +- crates/gpu-mimo2/src/tier.rs | 3 +- crates/gpu-minimaxm3/src/tier.rs | 3 +- crates/gpu/src/arch/qwen3moe/tier_leg.rs | 4 - crates/gpu/src/attention_h128.rs | 617 ++++++++++++ crates/gpu/src/attention_h128_ring.rs | 880 ++++++++++++++++++ crates/gpu/src/card_experts.rs | 122 ++- crates/gpu/src/card_leg.rs | 119 ++- crates/gpu/src/expert_input.rs | 257 +++++ crates/gpu/src/flash_gqa_h128_swa.rs | 2 +- crates/gpu/src/gemm/route.rs | 5 + crates/gpu/src/host/handoff.rs | 10 +- crates/gpu/src/host/leg_tier.rs | 39 +- crates/gpu/src/host/run.rs | 11 +- crates/gpu/src/lib.rs | 3 + crates/gpu/src/ring_gemm.rs | 33 +- crates/gpu/src/route_body.rs | 210 ++++- crates/gpu/src/routed_block.rs | 2 + crates/model/src/arch/coverage.rs | 131 ++- crates/model/src/arch/flint/host.rs | 83 ++ crates/model/src/arch/flint/mod.rs | 13 +- crates/model/src/arch/flint/place.rs | 478 ++++++++++ crates/model/src/arch/flint/tests.rs | 12 +- crates/model/src/arch/glm5next/place.rs | 9 +- crates/model/src/arch/mimo2/place.rs | 10 +- crates/model/src/arch/minimaxm3/place.rs | 12 +- crates/model/src/arch/musespark/place.rs | 2 +- crates/model/src/arch/names.rs | 6 + crates/model/src/arch/place_hybrid.rs | 42 +- crates/model/src/arch/qwen35moe/place.rs | 2 +- crates/model/src/arch/qwen3moe/place.rs | 2 +- crates/model/src/arch/solar2/place.rs | 2 + crates/model/src/moe.rs | 148 ++- crates/model/src/ops.rs | 113 +++ crates/model/tests/flint_meta.rs | 48 +- crates/model/tests/qwen3_bp_plan.rs | 10 +- crates/models/src/shape.rs | 40 +- crates/placement/src/accepts.rs | 12 +- crates/placement/src/kernels.rs | 3 + crates/placement/src/kernels/attention.rs | 79 ++ crates/placement/src/kernels/ffn.rs | 234 +++++ crates/placement/src/placement.rs | 10 +- crates/placement/src/placement/workstation.rs | 48 +- docs/research/flint-attach.md | 95 +- justfile | 56 +- tools/families.tsv | 2 +- tools/gate-class.tsv | 1 + tools/gate-paths.tsv | 2 +- tools/hf-arch-check.py | 8 +- tools/levers-direct.txt | 1 + tools/model-tax.tsv | 8 +- tools/recipes.py | 2 - tools/ref/dense-logits.py | 2 +- tools/release/build.sh | 2 +- tools/unsafe-ratchet.txt | 16 +- 83 files changed, 6230 insertions(+), 485 deletions(-) create mode 100644 crates/app/src/arch/flint.rs create mode 100644 crates/gpu-flint/Cargo.toml create mode 100644 crates/gpu-flint/src/body.rs create mode 100644 crates/gpu-flint/src/lib.rs create mode 100644 crates/gpu-flint/src/prefill.rs create mode 100644 crates/gpu-flint/src/step.rs create mode 100644 crates/gpu-flint/src/tests.rs create mode 100644 crates/gpu-flint/src/tier.rs create mode 100644 crates/gpu/src/attention_h128.rs create mode 100644 crates/gpu/src/attention_h128_ring.rs create mode 100644 crates/gpu/src/expert_input.rs create mode 100644 crates/model/src/arch/flint/host.rs create mode 100644 crates/model/src/arch/flint/place.rs create mode 100644 crates/placement/src/kernels/attention.rs create mode 100644 crates/placement/src/kernels/ffn.rs diff --git a/Cargo.lock b/Cargo.lock index e95e1197b..0b9fbced7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,6 +74,7 @@ dependencies = [ "bloomery-gguf", "bloomery-gpu", "bloomery-gpu-deepseek41", + "bloomery-gpu-flint", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-minimaxm3", @@ -197,6 +198,20 @@ dependencies = [ "cuda-host", ] +[[package]] +name = "bloomery-gpu-flint" +version = "0.1.0" +dependencies = [ + "bloomery-gguf", + "bloomery-gpu", + "bloomery-gpu-deepseek41", + "bloomery-levers", + "bloomery-model", + "bloomery-models", + "bloomery-runtime", + "cuda-core", +] + [[package]] name = "bloomery-gpu-gates" version = "0.3.9" @@ -208,6 +223,7 @@ dependencies = [ "bloomery-gguf", "bloomery-gpu", "bloomery-gpu-deepseek41", + "bloomery-gpu-flint", "bloomery-gpu-gemma-embedding2", "bloomery-gpu-gemma4a", "bloomery-gpu-glm5next", diff --git a/Cargo.toml b/Cargo.toml index ed941f89b..4313a222a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/qdot", "crates/gguf", "crates/models", "crates/model", "crates/model-hf", "crates/placement", "crates/threads", "crates/levers", "crates/verbs", "crates/gpu", "crates/gpu-deepseek41", "crates/gpu-glm5next", "crates/gpu-mimo2", "crates/gpu-solar2", "crates/gpu-minimaxm3", "crates/gpu-gemma-embedding2", "crates/gpu-gates", "crates/engram", "crates/engram-lab", "crates/sampler", "crates/tokenizer", "crates/serve", "crates/jinja", "crates/vision", "crates/gpu-vision", "crates/gpu-gemma4a", "crates/refset", "crates/runtime", "crates/app", "crates/decision", "crates/embed", "crates/hf", "crates/safetensors", "crates/os", "crates/device", "crates/metal", "crates/record", "crates/chain"] +members = ["crates/qdot", "crates/gguf", "crates/models", "crates/model", "crates/model-hf", "crates/placement", "crates/threads", "crates/levers", "crates/verbs", "crates/gpu", "crates/gpu-deepseek41", "crates/gpu-glm5next", "crates/gpu-mimo2", "crates/gpu-solar2", "crates/gpu-minimaxm3", "crates/gpu-flint", "crates/gpu-gemma-embedding2", "crates/gpu-gates", "crates/engram", "crates/engram-lab", "crates/sampler", "crates/tokenizer", "crates/serve", "crates/jinja", "crates/vision", "crates/gpu-vision", "crates/gpu-gemma4a", "crates/refset", "crates/runtime", "crates/app", "crates/decision", "crates/embed", "crates/hf", "crates/safetensors", "crates/os", "crates/device", "crates/metal", "crates/record", "crates/chain"] # 의도적으로 컴파일되지 않는 재현기다(MUL-7). 워크스페이스에 넣으면 --workspace가 영구히 빨갛다. exclude = ["crates/oxide-ice-unroll", "crates/oxide-ice-unroll/variants/unroll_smoke", "fuzz", "labs/hip-native"] diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index b828c35a2..38f3d0dd0 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -29,6 +29,7 @@ bloomery-gpu-mimo2 = { path = "../gpu-mimo2", optional = true } bloomery-gpu-minimaxm3 = { path = "../gpu-minimaxm3", optional = true } # The Solar-Open2 session (`arch::solar2`). bloomery-gpu-solar2 = { path = "../gpu-solar2", optional = true } +bloomery-gpu-flint = { path = "../gpu-flint", optional = true } # The levers the sessions' own configurations carry (the plan's card budget, the host set's load # settings). The crate has no dependency of its own, so every session reads it. bloomery-levers = { path = "../levers" } @@ -41,3 +42,4 @@ glm5next = ["dep:bloomery-gpu-glm5next"] mimo2 = ["dep:bloomery-gpu-mimo2"] minimaxm3 = ["dep:bloomery-gpu-minimaxm3"] solar2 = ["dep:bloomery-gpu-solar2"] +flint = ["dep:bloomery-gpu-flint"] diff --git a/crates/app/src/arch/flint.rs b/crates/app/src/arch/flint.rs new file mode 100644 index 000000000..5587fe585 --- /dev/null +++ b/crates/app/src/arch/flint.rs @@ -0,0 +1,76 @@ +//! Flint's session over its actual attention-ring body and common hybrid planner. + +use std::num::NonZeroUsize; + +use bloomery_gpu::head::Head; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError, GpuModel}; +use bloomery_gpu_flint::Body; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::PlanInputs; +use model::placement::Plan; + +use super::{Batch, ResidentSequences}; + +const WHAT: &str = "flint session"; + +/// A Flint session, with resident slots backed by the body's own stores. +pub type FlintSession = crate::Session; + +impl Batch for Body { + type Inputs = PlanInputs; + type Sequences = ResidentSequences; + + const WHAT: &'static str = self::WHAT; + + fn read(file: &Split) -> Result { + PlanInputs::read(file).map_err(|e| GpuError::plan(WHAT, e)) + } + + fn configure(inputs: &mut PlanInputs, cfg: &super::HybridCfg) -> Result<(), GpuError> { + model::arch::flint::place::configure_group(inputs, cfg.group) + .map_err(|e| GpuError::plan(WHAT, e)) + } + + fn open_placed( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + host: HostCfg, + residency: Residency, + slots: NonZeroUsize, + ) -> Result, GpuError> { + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) + } + + fn batch( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, + ids: &[u32], + at: u32, + ) -> Result { + bloomery_gpu_flint::prefill::prefill(gpu, w, body, head, ids, at) + } + + fn group(m: &mut GpuModel, g: usize) -> Result { + bloomery_gpu_flint::set_prefill_group(m, g) + } + + fn keepable(m: &GpuModel, n: u32) -> u32 { + m.body(WHAT) + .map_or(0, |b| b.keepable_at(m.pos(), n.min(m.pos()))) + } + + fn cut(m: &mut GpuModel, n: u32) -> Result<(), GpuError> { + if n == 0 { + return m.reset(); + } + let pos = m.pos(); + m.body_parts(WHAT)?.2.stand_for_cut(pos); + m.rollback(n) + } +} diff --git a/crates/app/src/arch/minimaxm3/mod.rs b/crates/app/src/arch/minimaxm3/mod.rs index ba4cadc72..765a0ce3d 100644 --- a/crates/app/src/arch/minimaxm3/mod.rs +++ b/crates/app/src/arch/minimaxm3/mod.rs @@ -9,8 +9,8 @@ //! every layer's key and value rows at those positions, every sparse layer's //! index keys, and the last position's logits. Every store is a plane of //! positions a later position never reads before it writes, so a cut keeps -//! every held position. This session holds one sequence; the serve seat parks -//! more ([`bloomery_gpu_minimaxm3`]'s `Slots`). +//! every held position. The plan counts the configured resident sequences; +//! [`crate::Session::add_slots`] parks them through the body's actual `Slots`. //! //! The session's shape is the common owner's ([`super::Batch`]): what the //! placement plans, how the prompt runs and how a cut is kept are shared with @@ -37,6 +37,7 @@ const WHAT: &str = "minimaxm3 session"; impl Batch for Body { type Inputs = PlanInputs; + type Sequences = super::ResidentSequences; const WHAT: &'static str = self::WHAT; @@ -52,8 +53,9 @@ impl Batch for Body { inputs: &PlanInputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError> { - Body::open_placed_slots(file, plan, inputs, 0, host, residency, NonZeroUsize::MIN) + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) } fn batch( diff --git a/crates/app/src/arch/mod.rs b/crates/app/src/arch/mod.rs index 04b2b3bd2..bff1125c4 100644 --- a/crates/app/src/arch/mod.rs +++ b/crates/app/src/arch/mod.rs @@ -8,10 +8,12 @@ //! and the cut rule — and [`Batch`] holds the rest: the [`Open`] the placement //! plans by, the [`Prompt`] the batch runs and the [`Keep`] the cut follows. +use std::num::NonZeroUsize; + use bloomery_gpu::head::Head; use bloomery_gpu::host::served::prompt_pass; use bloomery_gpu::host::swap::Residency; -use bloomery_gpu::model::{ChainBody, Rollback}; +use bloomery_gpu::model::{ChainBody, Rollback, Slots}; use bloomery_gpu::weights::Weights; use bloomery_gpu::{Gpu, GpuError, GpuModel}; use bloomery_levers::HostCfg; @@ -24,6 +26,8 @@ use crate::{Keep, Open, Prompt}; #[cfg(feature = "deepseek41")] pub mod deepseek41; +#[cfg(feature = "flint")] +pub mod flint; #[cfg(feature = "glm5next")] pub mod glm5next; #[cfg(feature = "mimo2")] @@ -56,6 +60,8 @@ pub struct HybridCfg { pub residency: Residency, /// The batches a prompt group runs layer by layer. pub group: usize, + /// The resident sequences charged by the plan before any allocation. + pub slots: NonZeroUsize, } /// The plan inputs of a hybrid body: the family's own hyperparameters under @@ -71,6 +77,7 @@ pub trait HybridInputs { machine: &'a Machine, ctx: u64, levers: &PlanLevers, + slots: NonZeroUsize, what: &'static str, ) -> Result, GpuError>; } @@ -85,9 +92,11 @@ impl HybridInputs for PlanInputs { machine: &'a Machine, ctx: u64, levers: &PlanLevers, + slots: NonZeroUsize, what: &'static str, ) -> Result, GpuError> { - PlanInputs::plan(self, machine, ctx, levers).map_err(|e| GpuError::plan(what, e)) + self.plan_with_slots(machine, ctx, levers, slots) + .map_err(|e| GpuError::plan(what, e)) } } @@ -98,6 +107,8 @@ impl HybridInputs for PlanInputs { pub trait Batch: ChainBody + Rollback + Sized { /// The plan inputs the body is loaded by. type Inputs: HybridInputs; + /// The sequence capability the body actually implements. + type Sequences: SequenceSlots; /// The session's name in its refusals. const WHAT: &'static str; @@ -105,6 +116,11 @@ pub trait Batch: ChainBody + Rollback + Sized { /// The plan inputs of `file`. fn read(file: &Split) -> Result; + /// Price configuration-dependent buffers before placement; unchanged for a fixed schedule. + fn configure(_inputs: &mut Self::Inputs, _cfg: &HybridCfg) -> Result<(), GpuError> { + Ok(()) + } + /// The body resident by `plan`, which `inputs` made: the layers the plan /// puts on the card, the host tier over the routed ones. fn open_placed( @@ -113,6 +129,7 @@ pub trait Batch: ChainBody + Rollback + Sized { inputs: &Self::Inputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError>; /// The prompt batch of `ids` from `at`: the head's row after the last. @@ -154,6 +171,10 @@ impl Open for B { B::read(file) } + fn configure_inputs(inputs: &mut B::Inputs, cfg: &HybridCfg) -> Result<(), GpuError> { + B::configure(inputs, cfg) + } + fn layer_count(inputs: &B::Inputs) -> usize { inputs.layers() } @@ -169,8 +190,9 @@ impl Open for B { ctx: usize, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, B::WHAT)?; let ctx = one_card_ctx(B::WHAT, machine, ctx)?; - inputs.plan_at(machine, ctx, &cfg.place, B::WHAT) + inputs.plan_at(machine, ctx, &cfg.place, cfg.slots, B::WHAT) } fn open( @@ -179,7 +201,8 @@ impl Open for B { plan: &Plan<'_>, cfg: &HybridCfg, ) -> Result, GpuError> { - B::open_placed(file, plan, inputs, cfg.host, cfg.residency) + >::check(cfg.slots, B::WHAT)?; + B::open_placed(file, plan, inputs, cfg.host, cfg.residency, cfg.slots) } /// The configuration's group; the units it needs are made here. @@ -222,6 +245,81 @@ impl Keep for B { } } +/// The sequence operations a common hybrid seat may offer. +pub trait SequenceSlots { + /// Whether several resident sequences are backed by body-owned stores. + const RESIDENT: bool; + /// Refuse an unsupported count before planning or allocation. + fn check(slots: NonZeroUsize, what: &'static str) -> Result<(), GpuError>; + /// Park the planned sequences through the session's actual slot operations. + fn add(s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError>; + /// The number of resident sequences the session currently holds. + fn count(s: &crate::Session) -> usize; + /// Select the actual sequence every following call acts on. + fn select(s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError>; +} + +/// A body that holds one sequence; other slots are refused, not simulated. +pub struct SingleSequence; + +impl SequenceSlots for SingleSequence { + const RESIDENT: bool = false; + + fn check(slots: NonZeroUsize, what: &'static str) -> Result<(), GpuError> { + if slots.get() != 1 { + return Err(GpuError::Shape { + what, + detail: format!( + "the body holds one sequence, not {} resident slots", + slots.get() + ), + }); + } + Ok(()) + } + + fn add(_s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError> { + >::check(slots, "single-sequence session").map_err(Into::into) + } + + fn count(_s: &crate::Session) -> usize { + 1 + } + + fn select(_s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError> { + if slot == 0 { + Ok(()) + } else { + Err(crate::SessionError::Refused(format!( + "select_slot {slot}: the body holds one sequence" + ))) + } + } +} + +/// Resident sequences backed by the body's actual parked stores. +pub struct ResidentSequences; + +impl> SequenceSlots for ResidentSequences { + const RESIDENT: bool = true; + + fn check(_slots: NonZeroUsize, _what: &'static str) -> Result<(), GpuError> { + Ok(()) + } + + fn add(s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError> { + s.add_slots(slots.get()) + } + + fn count(s: &crate::Session) -> usize { + s.slots() + } + + fn select(s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError> { + s.select_slot(slot) + } +} + /// `ctx` positions as a plan takes them, for a body whose chain runs on one /// card: refused by name (`what` the body's session) when `machine` puts the /// layers on any other number of cards or `ctx` passes u64. diff --git a/crates/app/src/arch/musespark/mod.rs b/crates/app/src/arch/musespark/mod.rs index 91194dd31..a66a0ca01 100644 --- a/crates/app/src/arch/musespark/mod.rs +++ b/crates/app/src/arch/musespark/mod.rs @@ -47,8 +47,9 @@ impl Open for Body { ctx: usize, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, WHAT)?; let ctx = one_card_ctx(WHAT, machine, ctx)?; - inputs.plan_at(machine, ctx, &cfg.place, WHAT) + inputs.plan_at(machine, ctx, &cfg.place, cfg.slots, WHAT) } /// The model resident by `plan`, on the plan's one card, under the @@ -60,6 +61,7 @@ impl Open for Body { plan: &Plan<'_>, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, WHAT)?; GpuModel::::open_placed_with(file, plan, 0, cfg.host, cfg.residency) } diff --git a/crates/app/src/arch/solar2/mod.rs b/crates/app/src/arch/solar2/mod.rs index 482533bfe..56eac5094 100644 --- a/crates/app/src/arch/solar2/mod.rs +++ b/crates/app/src/arch/solar2/mod.rs @@ -40,6 +40,7 @@ const WHAT: &str = "solar2 session"; impl Batch for Body { type Inputs = PlanInputs; + type Sequences = super::SingleSequence; const WHAT: &'static str = self::WHAT; @@ -53,8 +54,9 @@ impl Batch for Body { inputs: &PlanInputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError> { - Body::open_placed_slots(file, plan, inputs, 0, host, residency, NonZeroUsize::MIN) + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) } fn batch( diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 5eee1b184..df3396b79 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -161,6 +161,12 @@ pub trait Open: ChainBody + Sized { /// Read `file`'s headers. fn inputs(file: &Split) -> Result; + /// Apply load settings that change planned buffers, before the plan borrows its inputs. + /// Bodies with no configuration-dependent buffers keep the header-derived inputs. + fn configure_inputs(_inputs: &mut Self::Inputs, _cfg: &Self::Cfg) -> Result<(), GpuError> { + Ok(()) + } + /// The layer count a placement's machine is laid out for. fn layer_count(inputs: &Self::Inputs) -> usize; @@ -303,7 +309,8 @@ impl Loaded { log: &mut impl OpenLog, edit: impl FnOnce(&B::Inputs, &mut Plan<'_>) -> Result<(), SessionError>, ) -> Result>, SessionError> { - let inputs = B::inputs(&file)?; + let mut inputs = B::inputs(&file)?; + B::configure_inputs(&mut inputs, &args.cfg)?; let machine = (args.machine)(B::layer_count(&inputs)); let mut plan = B::plan(&inputs, &machine, args.ctx, &args.cfg)?; edit(&inputs, &mut plan)?; diff --git a/crates/gpu-deepseek41/src/experts_mxfp4.rs b/crates/gpu-deepseek41/src/experts_mxfp4.rs index 033f5f291..51f15c799 100644 --- a/crates/gpu-deepseek41/src/experts_mxfp4.rs +++ b/crates/gpu-deepseek41/src/experts_mxfp4.rs @@ -182,7 +182,9 @@ mod dflash_kernels { // N_EXPERT = n_expert <= bias.len() by the launch contract; score and // sel_v are block-shared arrays of N_EXPERT entries; the barrier below // publishes them. - unsafe { publish_n::(probs.as_mut_ptr(), Some(bias), e0, score, sel_v, tid, fault) }; + unsafe { + publish_n::(probs.as_mut_ptr(), Some(bias), e0, score, sel_v, tid, fault) + }; thread::sync_threads(); if tid < 32 { diff --git a/crates/gpu-deepseek41/src/prompt_walk.rs b/crates/gpu-deepseek41/src/prompt_walk.rs index 905cba000..5b182c96f 100644 --- a/crates/gpu-deepseek41/src/prompt_walk.rs +++ b/crates/gpu-deepseek41/src/prompt_walk.rs @@ -214,7 +214,14 @@ impl BatchBufs { })?; let stream = gpu.stream(); let sets = group_sets(group); - Ok(Self { + let expected = model::placement::kernels::ffn::PromptWalk { + hidden: embd as u64, + sets: sets as u64, + } + .bytes(t_max as u64) + .and_then(|bytes| usize::try_from(bytes).ok()) + .ok_or_else(|| GpuError::shape("prompt batch buffers", "buffer geometry overflows"))?; + let buffers = Self { group, t_max, units: (0..sets) @@ -225,7 +232,14 @@ impl BatchBufs { stream: PromptStream::default(), facts: None, timing: None, - }) + }; + if buffers.bytes() != expected { + return Err(GpuError::shape( + "prompt batch buffers", + format!("allocated {} bytes, planned {expected}", buffers.bytes()), + )); + } + Ok(buffers) } /// What the next prompt calls move (`BLOOMERY_XSTREAM`), behind the guards every family shares diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index 7eaf38e55..2d9457022 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -4,20 +4,17 @@ //! [`shadow`], [`back`]) or a unit's rows (the batch: [`Routed::front_rows`] and its three). Both feeds leave the same //! bits but the shared expert's rows of a unit of [`GATED_FROM`] rows or more, which its GEMMs write. //! -//! Every layer is a routed layer with one shared expert, so the block has no dense arm. The router, the handoff, the -//! go, the card experts' call and the wait and add around the host leg are the common routed block -//! ([`bloomery_gpu::routed_block`]); this file holds the shared expert and the base the card sum is added to. Its -//! parts around the host leg: -//! - the front: `ffn_norm` RMS of the rows of the residual after the mixer `x1`, and the router over them: the -//! scores, the selection bias (the choice only), the top `R::N_USED`, the weights from the unbiased scores -//! normalized and scaled. The step goes on to the handoff into the host-mapped image and the go; four launches. -//! The batch downloads the rows, weights and ids to the host ([`Routed::download`]). -//! - the shadow: while the host computes, the shared expert, its output summed into `x1` (`x1s`), then the card -//! experts the slot map puts on the card with `x1s` as the base their sum is added to; none of the latter on a -//! layer with no card experts. The base is `x1 + shared` before the card sum is added, in the step and the batch -//! alike. -//! - the back: the wait, then the add of the host's routed sum and the shadow's `card + x1s` (`x1s` alone on a -//! layer with no card experts). One launch after the wait. +//! Dense leading layers run the common dense block; routed layers have one ungated shared expert. The router, +//! handoff, go, card experts and host wait are the common routed block ([`bloomery_gpu::routed_block`]). +//! [`FfnConfig`] resolves the residual association and expert-input range protection once at load: +//! - the front normalizes the attention residual, scores and selects through `R`'s normalization and bias +//! policy, then hands the normed row and picks to the host; +//! - the shadow computes the shared expert and card experts while the host runs. The default saves +//! `residual + shared` as the base. [`ResidualOrder::RoutedSharedResidual`] keeps the residual separate; +//! - the back joins the host and card routed sums. The default adds the host to the card-plus-base rows; +//! the other association computes `(routed + shared) + residual`, after the complete routed sum is ready. +//! +//! The step and prompt batch use the same loaded association, including tier-card joins. //! //! A load whose slot map names an expert tier card ([`TierSide`]) runs a layer the tier holds experts of as a tier //! layer: the front's handoff also writes the tier's image and each slot's tier place and the go is the host's and the @@ -28,15 +25,15 @@ //! [`Routed::join_rows`] (the host leg's [`bloomery_gpu::host::BatchLeg::join_tiered`] hands the tier's rows to the //! join, then [`Routed::back_rows`]). //! -//! What a family fixes is its router instance, its [`CardFamily`] (stack names, counts) and what its description -//! says: the routed experts' and the shared expert's activation ([`rule_of`]: a SwiGLU clamp or SwiGLU-OAI), the -//! weights' scale and widths, which every layer agrees on ([`describe`]). The shared expert's down is Q6_K or Q4_K -//! by layer, so its block plan is read per layer. The shared expert's Q8_0 pair kernel runs the clamp alone, so a -//! SwiGLU-OAI shared expert on Q8_0 weights is refused by name at the first launch. +//! What a family fixes is its router instance, its [`CardFamily`] (stack names, counts), the loaded arithmetic +//! facts, and its description: the routed and shared activations ([`rule_of`]), scales and widths +//! ([`describe`]). Shared and dense block plans are read per layer and support mixed sites, including IQ4_XS. +//! A Q8_0 shared pair runs the clamp alone; a SwiGLU-OAI shared expert on it is refused before launch. use std::sync::Arc; use bloomery_gpu::card_experts::{CardBatch, CardExperts, CardFamily}; +use bloomery_gpu::card_leg::ExpertInput; use bloomery_gpu::gemm::{Gemm32Kernels, GemmInput, GemmKernels, GemmRoute}; use bloomery_gpu::host::leg_tier::TierSide; use bloomery_gpu::host::run::HostRun; @@ -58,9 +55,13 @@ use crate::chain::ffn::FfnBatchKernels; use crate::experts::ExpertKernels; use crate::span::{span, span_mut}; use crate::tier_join::{self, TierBatch, TierSlots}; -use cuda_core::{CudaStream, DeviceBuffer}; +use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; +use cuda_device::float::add_rn_f32; +use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread}; +use cuda_host::cuda_module; use gguf::Split; use model::arch::names; +use model::placement::kernels::ffn; use models::{Act as ActRule, Ffn, LayerSpec, Score}; /// What the refusals of this file name. @@ -70,7 +71,125 @@ const WHAT: &str = "shared routed block"; /// GEMM a site): below it `enqueue_block_cols` runs the step's gemvs on each /// column (at most 8 columns), so a unit of up to 8 rows writes the step's /// bits and from it a launch count of 6 replaces 4 a column. -pub const GATED_FROM: usize = 9; +pub const GATED_FROM: usize = ffn::GATED_FROM as usize; + +/// The association of a routed block's shared output and attention residual. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ResidualOrder { + /// The shared output joins the residual before the routed sum. + #[default] + ResidualSharedRouted, + /// The complete routed sum joins the shared output before the residual. + RoutedSharedResidual, +} + +/// Arithmetic facts resolved once when the routed block is loaded. +#[derive(Clone, Copy, Debug)] +pub struct FfnConfig { + pub residual: ResidualOrder, + pub expert_input: ExpertInput, +} + +impl Default for FfnConfig { + fn default() -> Self { + Self { + residual: ResidualOrder::ResidualSharedRouted, + expert_input: ExpertInput::Direct, + } + } +} + +#[cuda_module] +mod residual_kernels { + use super::*; + + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + host.len() >= n, + card.len() >= n, + shared.len() >= n, + residual.len() >= n, + dest.len() >= n + ) + )] + pub fn routed_shared_residual( + host: &[f32], + card: &[f32], + shared: &[f32], + residual: &[f32], + has_card: u32, + n: u32, + mut dest: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: the launch bounds every input and output by n; i is guarded. + unsafe { + let mut routed = *host.get_unchecked(i); + if has_card != 0 { + routed = add_rn_f32(routed, *card.get_unchecked(i)); + } + let ffn = add_rn_f32(routed, *shared.get_unchecked(i)); + *dest.get_unchecked_mut(i) = add_rn_f32(ffn, *residual.get_unchecked(i)); + } + } +} + +struct ResidualKernels { + module: residual_kernels::LoadedModule, +} + +impl ResidualKernels { + fn load(ctx: &Arc) -> Result { + // SAFETY: this package owns the embedded bundle; enqueue checks its contract. + let module = unsafe { bloomery_gpu::shared_module!(residual_kernels, ctx)? }; + Ok(Self { module }) + } + + fn enqueue( + &self, + stream: &CudaStream, + (host, card): (&DeviceBuffer, Option<&DeviceBuffer>), + (shared, residual): (&DeviceBuffer, &DeviceBuffer), + n: usize, + dest: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + if n == 0 + || host.len() < n + || card.is_some_and(|c| c.len() < n) + || shared.len() < n + || residual.len() < n + || dest.len() < n + { + return Err(GpuError::shape( + WHAT, + "routed residual join outside its buffers", + )); + } + let n = bloomery_gpu::launch_u32(WHAT, "residual values", n)?; + let prep = self + .module + .prepare_routed_shared_residual(LaunchConfig1D::new(n.div_ceil(256), 256, 0))?; + self.module.routed_shared_residual( + stream, + &prep, + host, + card.unwrap_or(host), + shared, + residual, + u32::from(card.is_some()), + n, + dest, + )?; + Ok(()) + } +} /// What the load takes of the model besides the description and the file. #[derive(Clone, Copy, Debug)] @@ -146,16 +265,15 @@ impl DenseFfn { } } -/// Layer `l`'s routed block for the common one: its norm, router and bias -/// tensors (every layer has a bias) and its row in `map`. -fn route_layer(map: &SlotMap, l: usize) -> Result { +/// Layer `l`'s norm, router, optional selection bias and row in `map`. +fn route_layer(map: &SlotMap, l: usize, bias: bool) -> Result { let row_off = map .row_offset(l) .ok_or_else(|| GpuError::shape(WHAT, format!("layer {l}: no row in the slot map")))?; Ok(RouteLayer { norm: names::ffn_norm(l), router: names::ffn_gate_inp(l), - bias: Some(names::exp_probs_b(l)), + bias: bias.then(|| names::exp_probs_b(l)), row_off, }) } @@ -169,6 +287,8 @@ struct Kernels { experts: ExpertKernels, /// The K-quant block's kernels: the shared expert is K-quant in this file. block: Option, + /// Loaded only for the routed-plus-shared, then residual association. + residual: Option, } /// The step's buffers: the router's outputs and each slot's place, and the @@ -191,6 +311,31 @@ struct SharedWide { u: DeviceBuffer, } +fn wide_layout( + (gate_up, down): (&[SiteTy], &[SiteTy]), + (embd, ff): (usize, usize), +) -> ffn::WideBlock { + let forms = |types: &[SiteTy]| { + types.iter().fold(ffn::WideForms::default(), |forms, ty| { + forms.with(ty.reads()) + }) + }; + ffn::WideBlock { + hidden: embd as u64, + ff: ff as u64, + input: forms(gate_up).without_f32(), + down: forms(down), + } +} + +fn device_forms(forms: ffn::WideForms) -> site::WideForms { + site::WideForms { + q128: forms.quant.q128, + q32: forms.quant.q32, + f32: forms.f32, + } +} + impl SharedWide { /// Buffers for up to `cap` rows over a block of width `ff` on `embd` values, whose gate and up read /// the types `gate_up` and whose down reads the types `down`. Load-time only. @@ -200,11 +345,12 @@ impl SharedWide { (cap, embd, ff): (usize, usize, usize), ) -> Result { let stream = gpu.stream(); + let layout = wide_layout((gate_up, down), (embd, ff)); Ok(SharedWide { gemm: GemmKernels::load(gpu.context())?, route: GemmRoute::new(stream, cap, 1)?, - x: WideActs::new(stream, WideActs::wants_of(gate_up).without_f32(), cap, embd)?, - h: WideActs::new(stream, WideActs::wants_of(down), cap, ff)?, + x: WideActs::new(stream, device_forms(layout.input), cap, embd)?, + h: WideActs::new(stream, device_forms(layout.down), cap, ff)?, g: DeviceBuffer::zeroed(stream, cap * ff)?, u: DeviceBuffer::zeroed(stream, cap * ff)?, }) @@ -325,7 +471,21 @@ impl BatchBufs { let z = |n: usize| DeviceBuffer::::zeroed(stream, n); let (shared_gate_up, shared_down) = block_types(per.iter().flatten().map(|l| &l.plan)); let (dense_gate_up, dense_down) = block_types(dense.iter().flatten().map(|l| &l.plan)); - Ok(BatchBufs { + let layout = ffn::RoutedBatch { + hidden: embd as u64, + experts: R::N_EXPERT as u64, + used: R::N_USED as u64, + shared: wide_layout((&shared_gate_up, &shared_down), (embd, ff)), + dense: (dense_ff > 0) + .then(|| wide_layout((&dense_gate_up, &dense_down), (embd, dense_ff))), + card: (card.any() || card.any_tier()).then(|| card.batch_layout(cap)), + tiered, + }; + let expected = layout + .bytes(cap as u64) + .and_then(|bytes| usize::try_from(bytes).ok()) + .ok_or_else(|| GpuError::shape(WHAT, "routed batch scratch byte overflow"))?; + let buffers = BatchBufs { cap, normed: z(cap * embd)?, probs: z(cap * R::N_EXPERT)?, @@ -348,7 +508,14 @@ impl BatchBufs { tier: tiered .then(|| TierBatch::new(stream, cap, R::N_USED)) .transpose()?, - }) + }; + if buffers.bytes() != expected { + return Err(GpuError::shape( + WHAT, + "routed batch allocation differs from its common scratch descriptor", + )); + } + Ok(buffers) } fn bytes(&self) -> usize { @@ -448,8 +615,8 @@ struct Described { } /// The description of `layers` as the compiled router, handoff and shared -/// block run it: `R`'s experts and picks, sigmoid scores with a bias and -/// normalized weights, a SwiGLU or SwiGLU-OAI shared expert without a gate. +/// block run it: `R`'s experts, picks, normalization and bias policy; sigmoid scores and a +/// SwiGLU or SwiGLU-OAI shared expert without a gate. /// Anything else is refused by name with its layer. fn describe(layers: &[LayerSpec]) -> Result { let (mut scale, mut expert_ff, mut shared_ff) = (None, None, None); @@ -469,8 +636,8 @@ fn describe(layers: &[LayerSpec]) -> Result || m.top_k as usize != R::N_USED || m.latent.is_some() || r.score != Score::Sigmoid - || !r.bias - || r.norm != models::RouterNorm::SelectedSum + || (!r.bias && !R::OPTIONAL_BIAS) + || r.norm != R::NORM || r.hash || shared.sigmoid_gate { @@ -478,9 +645,11 @@ fn describe(layers: &[LayerSpec]) -> Result WHAT, format!( "layer {l}: {m:?} is not the {}-expert top-{} sigmoid router \ - with a bias, normalized weights and an ungated shared expert", + with {:?} normalization, optional bias={} and an ungated shared expert", R::N_EXPERT, - R::N_USED + R::N_USED, + R::NORM, + R::OPTIONAL_BIAS ), )); } @@ -530,6 +699,28 @@ impl Routed { (layers, map, slots): (&[LayerSpec], &SlotMap, &Arc>), d: RoutedDims, card: CardFamily, + ) -> Result, GpuError> { + Self::new_configured( + gpu, + file, + w, + (layers, map, slots), + d, + card, + FfnConfig::default(), + ) + } + + /// Load the same common block with explicit arithmetic facts. The default keeps the legacy + /// association and expert input unchanged; the selected policies apply to every routed layer. + pub fn new_configured( + gpu: &Gpu, + file: &Split, + w: &Weights, + (layers, map, slots): (&[LayerSpec], &SlotMap, &Arc>), + d: RoutedDims, + card: CardFamily, + config: FfnConfig, ) -> Result, GpuError> { if (card.n_expert, card.n_used) != (R::N_EXPERT, R::N_USED) { return Err(GpuError::shape( @@ -564,7 +755,11 @@ impl Routed { }) .collect::, GpuError>>()?; let routes = (0..layers.len()) - .map(|l| routed(l).then(|| route_layer(map, l)).transpose()) + .map(|l| { + moe_of(layers, l) + .map(|m| route_layer(map, l, m.router.bias)) + .transpose() + }) .collect::, _>>()?; let route_dims = RouteDims { embd: d.embd, @@ -577,6 +772,9 @@ impl Routed { let k = Kernels { g32: Gemm32Kernels::load(ctx)?, experts: ExpertKernels::load(ctx)?, + residual: (config.residual == ResidualOrder::RoutedSharedResidual) + .then(|| ResidualKernels::load(ctx)) + .transpose()?, block: per .iter() .flatten() @@ -606,10 +804,12 @@ impl Routed { }) .transpose()?; let dims = (d.embd, desc.expert_ff); - let card = match tier { + let mut card = match tier { Some(_) => CardExperts::tiered(gpu, w, card, layers, map, dims, 1)?, None => CardExperts::new(gpu, w, card, layers, map, dims, 1)?, }; + card.set_expert_input(gpu, config.expert_input)?; + card.set_add_base(config.residual == ResidualOrder::ResidualSharedRouted); let dense_ff = dense.iter().flatten().map(|l| l.ff).max().unwrap_or(0); let batch = (d.batch_rows > 1) .then(|| { @@ -703,9 +903,8 @@ impl Routed { self.step.route.ids() } - /// Layer `l`'s selection bias, the values the router adds to the scores - /// before it ranks them: the ranking a reference engine's picks are read - /// at. + /// Layer `l`'s selection bias for reference-pick inspection. A layer with no bias is refused by + /// name; execution handles it through the router's optional-bias policy. pub fn route_bias<'a>( &'a self, w: &'a Weights, @@ -766,10 +965,8 @@ fn shared_launch<'a>( (kernels, blk) } -/// Enqueue layer `l`'s shared expert and card experts of the step in its host -/// leg's shadow, after its [`front`]: `x1s` is `x1` plus the shared expert's -/// output, and the card sum plus `x1s` goes to the card experts' `pre` where -/// the layer has any. +/// Enqueue layer `l`'s shared and card experts after [`front`], while the host runs. `x1s` holds the +/// legacy residual-plus-shared base, or the saved residual when the loaded association joins it last. pub fn shadow( gpu: &Gpu, w: &Weights, @@ -786,8 +983,13 @@ pub fn shadow( y: &mut r.step.sh_y, }; enqueue_block_act(gpu, w, &kernels, &mut r.block, (&blk, r.rules.1), io)?; - gpu.elem() - .enqueue_add(stream, x1, &r.step.sh_y, r.d.embd, &mut r.step.x1s)?; + if r.k.residual.is_some() { + span_mut(WHAT, &mut r.step.x1s, 0, r.d.embd)? + .copy_from_device_async(&*span(WHAT, x1, 0, r.d.embd)?, stream)?; + } else { + gpu.elem() + .enqueue_add(stream, x1, &r.step.sh_y, r.d.embd, &mut r.step.x1s)?; + } let rule = r.rules.0; r.route.shadow( (gpu, w), @@ -812,6 +1014,19 @@ pub fn back( ) -> Result<(), GpuError> { let n_tier = r.card.tier_k(l); if n_tier == 0 { + if let Some(join) = &r.k.residual { + hybrid.boundary().enqueue_back_of(gpu.stream(), 0, l)?; + return join.enqueue( + gpu.stream(), + ( + hybrid.boundary().hsum_of(0)?, + r.card.has(l).then(|| r.card.acc()), + ), + (&r.step.sh_y, &r.step.x1s), + r.d.embd, + dest, + ); + } return r.route.back(gpu, hybrid, &r.card, l, (&r.step.x1s, dest)); } const W: &str = "shared routed back (tier layer)"; @@ -827,6 +1042,22 @@ pub fn back( n_tier, n_used: R::N_USED, }; + if let Some(residual) = &r.k.residual { + hybrid.enqueue_tier_back(gpu.stream(), l, 0)?; + let mut join = r.card.step_join(r.step.route.sel()); + tier_join::enqueue_acc( + (gpu.stream(), kernels), + (&mut join, &slots), + (hybrid.tier_rows(0)?, 1), + )?; + return residual.enqueue( + gpu.stream(), + (hybrid.boundary().hsum_of(0)?, Some(join.acc)), + (&r.step.sh_y, &r.step.x1s), + r.d.embd, + dest, + ); + } tier_join::enqueue_step_join( gpu, kernels, @@ -961,10 +1192,8 @@ impl Routed { self.card_rows(gw, lm, (&[], None)) } - /// Layer `l`'s shared expert over the `m` rows [`Routed::front_rows`] - /// wrote: the shared expert's rows (per column below [`GATED_FROM`] rows, - /// on `site::gated` from it) and `x1s = x1 + shared`, the base - /// [`Routed::card_rows`] adds the card sum to. + /// Layer `l`'s shared expert over the `m` normed rows, per column below [`GATED_FROM`] and on + /// `site::gated` from it. Keep the residual-plus-shared base or the separate residual, as loaded. pub fn shared_rows( &mut self, (gpu, w): (&Gpu, &Weights), @@ -1007,8 +1236,14 @@ impl Routed { (&b.normed, &mut b.sh), )?; } - gpu.elem() - .enqueue_add(gpu.stream(), x1, &b.sh, m * n, &mut b.x1s) + if k.residual.is_some() { + span_mut(WHAT, &mut b.x1s, 0, m * n)? + .copy_from_device_async(&*span(WHAT, x1, 0, m * n)?, gpu.stream())?; + Ok(()) + } else { + gpu.elem() + .enqueue_add(gpu.stream(), x1, &b.sh, m * n, &mut b.x1s) + } } /// Layer `l`'s card experts, where the slot map puts any, over the `m` @@ -1097,6 +1332,13 @@ impl Routed { }); }; let slots = tb.slots(n, (n_card, n_tier, R::N_USED)); + if self.k.residual.is_some() { + return tier_join::enqueue_acc( + (gpu.stream(), kernels), + (&mut cb.join(), &slots), + (trows, m), + ); + } tier_join::enqueue_join( gpu, (gpu.stream(), kernels), @@ -1106,10 +1348,8 @@ impl Routed { ) } - /// `x = hsum +` the shadow's rows (the card sum plus `x1s`, or `x1s` on a - /// layer with no card experts) over the unit's `m` rows, after - /// [`Routed::shadow_rows`] and the host sums' upload; on a tier layer after - /// [`Routed::join_rows`], which wrote the sum. + /// Join the uploaded host sums and shadow results over `m` rows with the loaded association. + /// On a tier layer, [`Routed::join_rows`] first forms the raw routed sum over both cards. pub fn back_rows( &self, gpu: &Gpu, @@ -1118,6 +1358,20 @@ impl Routed { ) -> Result<(), GpuError> { let b = self.batch.as_ref().ok_or_else(no_batch)?; b.check(l, m)?; + if let Some(join) = &self.k.residual { + let card = b + .card + .as_ref() + .filter(|_| self.card.sums(l)) + .map(CardBatch::acc); + return join.enqueue( + gpu.stream(), + (hsum, card), + (&b.sh, &b.x1s), + m * self.d.embd, + x, + ); + } let shadowed = match b.card.as_ref().filter(|_| self.card.sums(l)) { Some(cb) => cb.pre(), None => &b.x1s, diff --git a/crates/gpu-deepseek41/src/tier_join.rs b/crates/gpu-deepseek41/src/tier_join.rs index e02e500ba..ddaff2392 100644 --- a/crates/gpu-deepseek41/src/tier_join.rs +++ b/crates/gpu-deepseek41/src/tier_join.rs @@ -53,6 +53,18 @@ pub fn enqueue_join( (join, slots): (&mut TierJoin<'_>, &TierSlots<'_>), (trows, m): (&DeviceBuffer, usize), base: &DeviceBuffer, +) -> Result<(), GpuError> { + enqueue_acc((stream, kernels), (&mut *join, slots), (trows, m))?; + gpu.elem() + .enqueue_add(stream, join.acc, base, m * slots.n, join.pre) +} + +/// Sum both cards' routed slots without adding a base. A caller whose shared output and residual +/// join after the complete routed sum uses this result directly. +pub fn enqueue_acc( + (stream, kernels): (&CudaStream, &FfnBatchKernels), + (join, slots): (&mut TierJoin<'_>, &TierSlots<'_>), + (trows, m): (&DeviceBuffer, usize), ) -> Result<(), GpuError> { let a = CardAccTier { down: join.down, @@ -66,9 +78,7 @@ pub fn enqueue_join( n_tier: slots.n_tier, n_used: slots.n_used, }; - kernels.enqueue_card_acc_tier(stream, &a, join.acc)?; - gpu.elem() - .enqueue_add(stream, join.acc, base, m * slots.n, join.pre) + kernels.enqueue_card_acc_tier(stream, &a, join.acc) } /// Layer `l`'s back of the step on a tier layer at the page's row `row`: the wait for the host and diff --git a/crates/gpu-flint/Cargo.toml b/crates/gpu-flint/Cargo.toml new file mode 100644 index 000000000..2e1678700 --- /dev/null +++ b/crates/gpu-flint/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bloomery-gpu-flint" +version = "0.1.0" +edition = "2024" +publish = false +license = "MIT" + +[lints] +workspace = true + +[dependencies] +cuda-core.workspace = true +bloomery-gpu = { path = "../gpu" } +bloomery-gpu-deepseek41 = { path = "../gpu-deepseek41" } +bloomery-gguf = { path = "../gguf" } +bloomery-levers = { path = "../levers" } +bloomery-model = { path = "../model" } +bloomery-models = { path = "../models" } +bloomery-runtime = { path = "../runtime" } diff --git a/crates/gpu-flint/src/body.rs b/crates/gpu-flint/src/body.rs new file mode 100644 index 000000000..27b7654db --- /dev/null +++ b/crates/gpu-flint/src/body.rs @@ -0,0 +1,583 @@ +//! Checked load, sequence ownership and common model seam for Flint. + +use std::num::NonZeroUsize; +use std::ops::Range; +use std::sync::Arc; + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch as AttnScratch}; +use bloomery_gpu::card_experts::CardFamily; +use bloomery_gpu::card_leg::ExpertInput; +use bloomery_gpu::head::Head; +use bloomery_gpu::host::leg_tier::{Hang, LegOpen}; +use bloomery_gpu::host::refuse_tier_count; +use bloomery_gpu::host::run::{HostRun, HostWidths}; +use bloomery_gpu::host::served::{ResidencyParts, TierBody}; +use bloomery_gpu::host::shell::TierShell; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::host::swap_source::{ + DEADLINE, LIVE_DELAY, ResidencyGlue, ResidencySpec, hosted_stacks, +}; +use bloomery_gpu::host::tier::TierOpen; +use bloomery_gpu::hybrid::{Chain, HostResidency, Hybrid, SlotMap}; +use bloomery_gpu::model::{ + CardBytes, ChainBody, HostServed, Rollback, Slots, StepMode, hosted_hybrid_tiered, + refuse_reserve_past, refuse_scratch_past, stage_run, +}; +use bloomery_gpu::rope_table::{RopeRows, RopeSpec, RopeTable}; +use bloomery_gpu::route_body::sigmoid_floor_256_6::RouterKernels; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{DeviceTensor, Gpu, GpuError, GpuModel}; +use bloomery_gpu_deepseek41::prompt_walk::{self, BatchBufs}; +use bloomery_gpu_deepseek41::shared_routed::{FfnConfig, ResidualOrder, Routed, RoutedDims}; +use bloomery_levers::{ChurnHosting, HostCfg}; +use cuda_core::{CudaStream, DeviceBuffer}; +use gguf::Split; +use model::arch::Arch; +use model::arch::flint::{host, place::PlanInputs}; +use model::arch::names; +use model::embed::EmbedTable; +use model::placement::Plan; +use model::placement::accepts::Program; +use models::{Ffn, Mixer, RopeMode}; +use runtime::layer::{Layer, MixerKind, ResidualKind, hosted}; + +pub(crate) const WHAT: &str = "flint Body"; +pub type FlintModel = GpuModel; +pub(crate) type FfnBody = Routed; + +/// The per-replay position. The embedding row is staged before refresh. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StepInput { + pub pos: u32, +} + +pub(crate) struct StepRows { + pub x: DeviceBuffer, + pub x1: DeviceBuffer, + pub xn: DeviceBuffer, + pub delta: DeviceBuffer, + pub pos: DeviceBuffer, + pub n_keys: DeviceBuffer, +} + +impl StepRows { + fn new(stream: &CudaStream, hidden: usize) -> Result { + Ok(Self { + x: DeviceBuffer::zeroed(stream, hidden)?, + x1: DeviceBuffer::zeroed(stream, hidden)?, + xn: DeviceBuffer::zeroed(stream, hidden)?, + delta: DeviceBuffer::zeroed(stream, hidden)?, + pos: DeviceBuffer::zeroed(stream, 1)?, + n_keys: DeviceBuffer::zeroed(stream, 1)?, + }) + } + + fn bytes(&self) -> usize { + [&self.x, &self.x1, &self.xn, &self.delta] + .into_iter() + .map(DeviceBuffer::num_bytes) + .sum::() + + self.pos.num_bytes() + + self.n_keys.num_bytes() + } +} + +/// Weights, kernels and scratch belong to the load; only caches and their high-water mark belong to a sequence. +pub struct Body { + pub(crate) shell: TierShell, + pub(crate) cfg: Vec, + pub(crate) bindings: Vec, + pub(crate) stores: Vec, + pub(crate) k: Kernels, + pub(crate) attention: AttnScratch, + pub(crate) batch_attention: AttnScratch, + pub(crate) s: StepRows, + pub(crate) rope: RopeRows, + pub(crate) ffn: FfnBody, + pub(crate) units: BatchBufs, + pub(crate) embed: EmbedTable, + row: Vec, + pub(crate) slots: Arc>, + pub(crate) at: u32, + pub(crate) ctx: usize, + pub(crate) hidden: usize, + eps: f32, + slots_planned: usize, + slots_made: usize, + prompt_reserve: u64, +} + +impl Body { + /// A single-sequence placed load with fixed expert residency. + pub fn open_placed( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + card: usize, + host: HostCfg, + ) -> Result { + Self::open_placed_slots( + file, + plan, + inputs, + card, + host, + Residency::Off, + NonZeroUsize::MIN, + ) + } + + /// Load exactly the number of resident sequences counted by the placement. + #[allow( + clippy::too_many_arguments, + reason = "the placed load's file, plan, card and host policy, residency and sequence count" + )] + pub fn open_placed_slots( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + card: usize, + host: HostCfg, + residency: Residency, + slots: NonZeroUsize, + ) -> Result { + let tiers = TierOpen::of_machine(plan.machine)?; + refuse_tier_count(WHAT, tiers.len())?; + let (layers, _) = stage_run(WHAT, plan, card, inputs.spec.layers.len())?; + let one = inputs.kv.bytes(layers, plan.ctx_max); + refuse_store_count(plan, card, one, slots)?; + let run = hosted(&inputs.spec.layers).map_err(|e| shape(e.to_string()))?; + let residency = ResidencySpec { + lever: residency, + pool: ChurnHosting::Hosted, + delay: LIVE_DELAY, + deadline: DEADLINE, + top_k: 6, + max_rows: 1, + stacks: Arc::new(hosted_stacks(&inputs.model, WHAT, (run.len(), true))?), + }; + GpuModel::load_placed_with( + file, + plan, + card, + host, + residency, + |_, _, _, _| Ok(()), + |gpu, file, w, set, glue| { + Self::load( + gpu, + file, + w, + (plan, inputs, card), + host, + (set, glue), + (&tiers, slots), + ) + }, + ) + } + + #[allow( + clippy::too_many_arguments, + reason = "the checked placed load's disjoint common loader parts" + )] + fn load( + gpu: &Gpu, + file: &Arc, + w: &Weights, + (plan, inputs, card): (&Plan<'_>, &PlanInputs, usize), + host_cfg: HostCfg, + (set, glue): (HostResidency, ResidencyGlue), + (tiers, seqs): (&[TierOpen], NonZeroUsize), + ) -> Result { + gpu.context().bind_to_thread()?; + let spec = &inputs.spec; + let (layers, ctx) = stage_run(WHAT, plan, card, spec.layers.len())?; + let hidden = spec.hidden as usize; + let cfg: Vec<_> = spec.layers.iter().map(Layer::of).collect(); + for (l, kind) in cfg.iter().enumerate() { + if kind.mixer != MixerKind::Gqa || kind.residual != ResidualKind::Plain { + return Err(shape(format!( + "layer {l}: {kind:?}, not plain-residual GQA" + ))); + } + match &spec.layers[l].ffn { + Ffn::Moe(m) if m.experts != 256 || m.top_k != 6 || m.expert_ff != 512 => { + return Err(shape(format!( + "layer {l}: expert geometry {m:?}, not the compiled floor256top6 block" + ))); + } + _ => {} + } + } + let run = hosted(&spec.layers).map_err(|e| shape(e.to_string()))?; + let tier_cards: Vec<_> = tiers.iter().map(|t| t.card).collect(); + let map = SlotMap::of_plan_tiers(plan, card, &tier_cards, run.clone(), 256)?; + let bindings = spec + .layers + .iter() + .enumerate() + .map(|(l, s)| Binding::load(file, w, l, s, hidden, spec.rms_eps, ctx)) + .collect::, _>>()?; + let stores = bindings + .iter() + .map(|b| Cache::new(gpu, b)) + .collect::, _>>()?; + let held: usize = stores.iter().map(Cache::bytes).sum(); + let planned = inputs.kv.bytes(layers, plan.ctx_max); + if held as u64 != planned { + return Err(shape(format!( + "stores hold {held} bytes; the plan counts {planned}" + ))); + } + for (l, b) in bindings.iter().enumerate() { + let counted = inputs.kv.rows_at(l, plan.ctx_max); + if b.capacity() as u64 != counted { + return Err(shape(format!( + "layer {l}: {} physical rows; the plan counts {counted}", + b.capacity() + ))); + } + } + let rope_base = spec + .layers + .iter() + .find_map(|s| match &s.mixer { + Mixer::Gqa(g) if g.rope.mode == RopeMode::Neox => Some(g.rope.base), + _ => None, + }) + .ok_or_else(|| shape("no NeoX layer in the validated description"))?; + let table = RopeTable::new(&RopeSpec::window(rope_base, 128))?; + let rope = RopeRows::new(gpu.stream(), &table, 128, ctx)?; + let slots = Arc::new(DeviceTensor::upload( + gpu.stream(), + &map.stage_view(), + run.len(), + 256, + )?); + let ffn = FfnBody::new_configured( + gpu, + file, + w, + (&spec.layers, &map, &slots), + RoutedDims { + embd: hidden, + rms_eps: spec.rms_eps, + batch_rows: crate::prefill::T_MAX, + }, + CardFamily { + what: WHAT, + n_expert: 256, + n_used: 6, + stacks: names::routed_stacks, + program: Program::FlintBody, + }, + FfnConfig { + residual: ResidualOrder::RoutedSharedResidual, + expert_input: ExpertInput::AmaxF16, + }, + )?; + let hang = Hang { + tiers, + family: &crate::tier::NODE_FAMILY, + open: LegOpen { + n_expert: 256, + layers: cfg.len(), + hidden, + n_used: 6, + rows: 1, + }, + cols: crate::prefill::T_MAX, + card_dontneed: host_cfg.card_dontneed, + bytes: CardBytes::File, + }; + let mut hybrid = hosted_hybrid_tiered( + (gpu, file, plan), + (run.clone(), map), + (host_cfg, set, &glue), + ( + HostWidths { + embd: hidden, + ff: 512, + n_used: 6, + }, + 1, + ), + Some(&hang), + |src| { + run.clone() + .map(|l| host::layer(src, spec, l)) + .collect::, _>>() + }, + )?; + prompt_walk::prepare_host(gpu, &mut hybrid, crate::prefill::T_MAX)?; + let embed = EmbedTable::new(Arc::clone(file), &names::token_embd(), hidden)?; + if embed.n_vocab() != spec.vocab as usize { + return Err(shape(format!( + "embedding vocabulary {} differs from {}", + embed.n_vocab(), + spec.vocab + ))); + } + let prompt_reserve = inputs + .prompt_batch_bytes(plan.machine) + .map_err(|e| GpuError::plan(WHAT, e))?; + let mut body = Self { + shell: TierShell::new(hybrid, glue, hidden, WHAT), + k: Kernels::load(gpu)?, + attention: AttnScratch::new(gpu, &bindings, 1)?, + batch_attention: AttnScratch::new(gpu, &bindings, crate::prefill::T_MAX)?, + s: StepRows::new(gpu.stream(), hidden)?, + units: BatchBufs::new(gpu, hidden, crate::prefill::T_MAX, 1)?, + cfg, + bindings, + stores, + rope, + ffn, + embed, + row: vec![0.0; hidden], + slots, + at: 0, + ctx, + hidden, + eps: spec.rms_eps, + slots_planned: seqs.get(), + slots_made: 1, + prompt_reserve, + }; + refuse_scratch_past( + WHAT, + plan, + card, + body.scratch_bytes(), + "step rows, attention, routed blocks, rope, host boundary and slot map", + )?; + body.check_prompt_reserve()?; + body.start_residency(gpu)?; + Ok(body) + } + + pub fn ctx(&self) -> usize { + self.ctx + } + pub fn kinds(&self) -> Vec { + self.cfg.clone() + } + pub fn store_bytes(&self) -> usize { + self.stores.iter().map(Cache::bytes).sum() + } + pub fn scratch_bytes(&self) -> usize { + self.s.bytes() + + self.attention.bytes() + + self.k.bytes() + + self.ffn.bytes() + + self.rope.table.num_bytes() + + self.shell.hybrid.boundary().device_bytes() + + self.slots.buf().num_bytes() + } + pub fn prompt_bytes(&self) -> usize { + self.batch_attention.bytes() + self.units.bytes() + self.ffn.batch_bytes() + } + pub(crate) fn check_prompt_reserve(&self) -> Result<(), GpuError> { + refuse_reserve_past( + WHAT, + "prompt batch buffers", + self.prompt_reserve, + self.prompt_bytes() as u64, + "attention rows, common walk units and shared routed batch buffers", + ) + } + + /// Rollback never restores a ring. Its depth is measured from the greatest written end, not the last cut. + pub fn cut_keeps(&self, pos: u32, n: u32) -> bool { + let end = self.at.max(pos); + n <= end + && self.bindings.iter().all(|b| { + runtime::stores::cut_depth_keeps( + u64::from(end - n), + b.capacity() as u64, + self.ctx as u64, + b.window() as u64, + ) + }) + } + pub fn keepable_at(&self, pos: u32, n: u32) -> u32 { + if self.cut_keeps(pos, n) { n } else { 0 } + } + pub fn stand_for_cut(&mut self, pos: u32) { + self.at = self.at.max(pos); + } + pub fn hybrid(&self) -> &Hybrid { + self.shell.hybrid() + } + pub fn hybrid_mut(&mut self) -> &mut Hybrid { + self.shell.hybrid_mut() + } + pub fn taps(&self, gpu: &Gpu) -> Result, GpuError> { + self.shell.taps(gpu) + } +} + +impl ChainBody for Body { + type Input = StepInput; + type Host = Body; + const UNHOSTED_POOL: bloomery_levers::UnhostedPool = bloomery_levers::UnhostedPool::NotCarried; + fn arch() -> Arch { + Arch::Flint + } + fn decode_input(&mut self, token: u32, pos: u32) -> Result { + if pos as usize >= self.ctx { + return Err(shape(format!("position {pos} past context {}", self.ctx))); + } + self.embed.row_into(token, &mut self.row)?; + Ok(StepInput { pos }) + } + fn refresh(&mut self, stream: &CudaStream, input: &StepInput) -> Result<(), GpuError> { + self.s.x.copy_from_host(stream, &self.row)?; + self.s.pos.copy_from_host(stream, &[input.pos])?; + self.s.n_keys.copy_from_host(stream, &[input.pos + 1])?; + self.at = self.at.max(input.pos + 1); + Ok(()) + } + fn enqueue_chain(&mut self, gpu: &Gpu, w: &Weights, head: &mut Head) -> Result<(), GpuError> { + crate::step::enqueue(gpu, w, self, head) + } + fn reset(&mut self, gpu: &Gpu) -> Result<(), GpuError> { + self.shell.hybrid.settle(gpu.stream())?; + self.at = 0; + Ok(()) + } + fn head_eps(&self) -> f32 { + self.eps + } + fn resident_bytes(&self) -> usize { + self.store_bytes() + + self.scratch_bytes() + + self.prompt_bytes() + + self + .shell + .taps + .as_ref() + .map_or(0, |t| t.iter().map(DeviceBuffer::num_bytes).sum()) + } + fn layers(&self) -> Range { + 0..self.cfg.len() + } + fn host(&mut self) -> Option<&mut Body> { + Some(self) + } +} + +impl TierBody for Body { + type Experts = HostRun; + fn hybrid(&self) -> &Hybrid { + self.shell.hybrid() + } + fn hybrid_mut(&mut self) -> &mut Hybrid { + self.shell.hybrid_mut() + } + fn residency_parts(&mut self) -> Option> { + self.shell.residency_parts(&self.slots) + } + fn serve_chain(&mut self, chain: Chain) -> Result<(), GpuError> { + self.shell.hybrid.serve_captured_of(chain) + } +} + +impl Rollback for Body { + fn rollback(&mut self, pos: u32) -> Result<(), GpuError> { + if !self.cut_keeps(self.at, pos) { + return Err(shape(format!( + "rollback to {pos} from written end {} loses ring keys; reset and refeed", + self.at + ))); + } + Ok(()) + } +} + +/// One resident sequence's device stores and greatest written end. +pub struct Seq { + stores: Vec, + at: u32, +} + +impl Slots for Body { + type Seq = Seq; + fn new_seq(&mut self, gpu: &Gpu) -> Result { + if self.slots_made >= self.slots_planned { + return Err(shape(format!( + "sequence {} exceeds {} planned resident sequences", + self.slots_made + 1, + self.slots_planned + ))); + } + let need = self.store_bytes(); + let free = gpu.alloc_room()?; + if need > free { + return Err(shape(format!( + "sequence needs {need} store bytes, card has {free}" + ))); + } + let stores = self + .bindings + .iter() + .map(|b| Cache::new(gpu, b)) + .collect::, _>>()?; + self.slots_made += 1; + Ok(Seq { stores, at: 0 }) + } + fn swap_seq(&mut self, _gpu: &Gpu, seq: &mut Seq) -> Result<(), GpuError> { + if self.stores.len() != seq.stores.len() + || self + .stores + .iter() + .zip(&seq.stores) + .any(|(a, b)| a.bytes() != b.bytes()) + { + return Err(shape("parked sequence has another store layout")); + } + std::mem::swap(&mut self.stores, &mut seq.stores); + std::mem::swap(&mut self.at, &mut seq.at); + Ok(()) + } + fn seq_bytes(&self) -> usize { + self.store_bytes() + } +} + +impl Drop for Body { + fn drop(&mut self) { + self.shell.hybrid.stop_swap(); + } +} + +pub fn set_taps(m: &mut FlintModel, on: bool) -> Result<(), GpuError> { + m.set_mode(StepMode::Eager); + let (gpu, _, body) = m.body_parts(WHAT)?; + body.shell.set_taps(gpu, body.cfg.len(), on) +} + +fn refuse_store_count( + plan: &Plan<'_>, + card: usize, + one: u64, + slots: NonZeroUsize, +) -> Result<(), GpuError> { + let want = one.saturating_mul(slots.get() as u64); + let actual = plan + .cards + .get(card) + .ok_or_else(|| shape(format!("no card {card} in plan")))? + .kv_bytes; + if want != actual { + return Err(shape(format!( + "{slots} resident sequences need {want} store bytes; plan counts {actual}" + ))); + } + Ok(()) +} + +pub(crate) fn shape(detail: impl Into) -> GpuError { + GpuError::shape(WHAT, detail) +} diff --git a/crates/gpu-flint/src/lib.rs b/crates/gpu-flint/src/lib.rs new file mode 100644 index 000000000..57ceacb09 --- /dev/null +++ b/crates/gpu-flint/src/lib.rs @@ -0,0 +1,13 @@ +//! Flint's composition of the common head128 attention, shared routed block and host tier. +//! Device-linked consumers build this crate with `cargo oxide`. + +pub mod body; +pub mod prefill; +mod step; +pub mod tier; + +pub use body::{Body, FlintModel, Seq, StepInput, set_taps}; +pub use prefill::{prefill, set_prefill_group}; + +#[cfg(test)] +mod tests; diff --git a/crates/gpu-flint/src/prefill.rs b/crates/gpu-flint/src/prefill.rs new file mode 100644 index 000000000..a0e50c733 --- /dev/null +++ b/crates/gpu-flint/src/prefill.rs @@ -0,0 +1,128 @@ +//! Flint attention plugged into the common prompt walk and shared FFN. + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch}; +use bloomery_gpu::head::Head; +use bloomery_gpu::rope_table::RopeRows; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError}; +use bloomery_gpu_deepseek41::prompt_walk::{Batch, BatchAttention, UnitRows}; +use runtime::layer::{FfnKind, Layer}; + +use crate::body::{Body, FlintModel, shape}; + +/// The plan and the ring both hold one common prompt unit. +pub const T_MAX: usize = model::arch::flint::place::PROMPT_ROWS; +const _: () = assert!(T_MAX as u64 == runtime::stores::PROMPT_BATCH); + +/// Run a prompt through real attention and dense/routed blocks, then the final head. +pub fn prefill( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, + ids: &[u32], + at: u32, +) -> Result { + let to = + runtime::prompt::call_end(at, ids.len(), body.ctx).map_err(|e| shape(e.to_string()))?; + for &id in ids { + if id as usize >= body.embed.n_vocab() { + return Err(shape(format!( + "prompt token {id} past vocabulary {}", + body.embed.n_vocab() + ))); + } + } + // A failed call may already have overwritten ring rows. Reset, not a lower logical position, repairs that state. + body.at = body.at.max(to); + let Body { + shell, + cfg, + bindings, + stores, + k, + batch_attention, + rope, + units, + ffn, + embed, + hidden, + ctx, + .. + } = body; + let mut attention = Attention { + gpu, + w, + cfg, + bindings, + stores, + k, + scratch: batch_attention, + rope, + }; + let mut walk = Batch { + gpu, + w, + bufs: units, + attention: &mut attention, + ffn, + hybrid: &mut shell.hybrid, + embed, + head, + taps: shell.taps.as_deref_mut(), + embd: *hidden, + ctx: *ctx, + layers: cfg.len(), + }; + let mut cursor = at; + walk.prompt(ids, &mut cursor) +} + +struct Attention<'a> { + gpu: &'a Gpu, + w: &'a Weights, + cfg: &'a [Layer], + bindings: &'a [Binding], + stores: &'a mut [Cache], + k: &'a Kernels, + scratch: &'a mut Scratch, + rope: &'a RopeRows, +} + +impl BatchAttention for Attention<'_> { + fn attend( + &mut self, + l: usize, + _u: usize, + m: usize, + rows: UnitRows<'_>, + ) -> Result<(), GpuError> { + // SAFETY: the common walk stages consecutive positions and each count is position + 1. + unsafe { + self.bindings[l].enqueue( + self.gpu, + self.w, + self.k, + self.scratch, + &mut self.stores[l], + rows.x, + rows.x1, + rows.pos, + rows.n_keys, + m, + self.rope, + ) + } + } + fn ffn(&self, l: usize) -> FfnKind { + self.cfg[l].ffn + } +} + +/// Change prompt grouping between calls; allocation stays within the plan's reserve. +pub fn set_prefill_group(m: &mut FlintModel, group: usize) -> Result { + let (gpu, _, body) = m.body_parts("flint prompt group")?; + let made = body.units.set_group(gpu, body.hidden, group)?; + body.check_prompt_reserve()?; + Ok(made) +} diff --git a/crates/gpu-flint/src/step.rs b/crates/gpu-flint/src/step.rs new file mode 100644 index 000000000..a20320782 --- /dev/null +++ b/crates/gpu-flint/src/step.rs @@ -0,0 +1,183 @@ +//! One-token walk: common attention, then the dense or routed block, and the output head. + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch}; +use bloomery_gpu::head::Head; +use bloomery_gpu::host::StepLeg; +use bloomery_gpu::host::run::HostRun; +use bloomery_gpu::rope_table::RopeRows; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError}; +use bloomery_gpu_deepseek41::shared_routed; +use cuda_core::DeviceBuffer; +use runtime::layer::{FfnKind, Layer}; +use runtime::sched::{self, At, LayerProgram, Overlap, PortKind}; + +use crate::body::{Body, FfnBody, StepRows, shape}; + +pub(crate) fn enqueue( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, +) -> Result<(), GpuError> { + let Body { + shell, + cfg, + bindings, + stores, + k, + attention, + s, + rope, + ffn, + .. + } = body; + let mut port = StepLeg::new(gpu.stream(), &mut shell.hybrid); + let mut program = Step { + gpu, + w, + cfg, + bindings, + stores, + k, + attention, + s, + rope, + ffn, + taps: shell.taps.as_deref_mut(), + head, + }; + sched::walk( + Overlap { + units: 1, + cols: 1, + port: PortKind::Step, + }, + program.cfg.len(), + &mut port, + &mut program, + ) +} + +struct Step<'a> { + gpu: &'a Gpu, + w: &'a Weights, + cfg: &'a [Layer], + bindings: &'a [Binding], + stores: &'a mut [Cache], + k: &'a Kernels, + attention: &'a mut Scratch, + s: &'a mut StepRows, + rope: &'a RopeRows, + ffn: &'a mut FfnBody, + taps: Option<&'a mut [DeviceBuffer]>, + head: &'a mut Head, +} + +impl Step<'_> { + fn tap(&mut self, l: usize) -> Result<(), GpuError> { + let last = l + 1 == self.cfg.len(); + if let Some(taps) = self.taps.as_deref_mut() { + let out = if last { + &*self.head.input_mut() + } else { + &self.s.x + }; + taps[l].copy_from_device_async(out, self.gpu.stream())?; + } + Ok(()) + } +} + +impl<'a> LayerProgram for Step<'a> { + type Port = StepLeg<'a, HostRun>; + + fn begin(&mut self, unit: usize) -> Result<(), GpuError> { + one(unit) + } + + fn front(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + let l = at.layer; + // SAFETY: refresh stages the one position and its count, position + 1. + unsafe { + self.bindings[l].enqueue( + self.gpu, + self.w, + self.k, + self.attention, + &mut self.stores[l], + &self.s.x, + &mut self.s.x1, + &self.s.pos, + &self.s.n_keys, + 1, + self.rope, + )?; + } + match self.cfg[l].ffn { + FfnKind::Moe => { + shared_routed::front(self.gpu, self.w, self.ffn, &self.s.x1, (port.hybrid(), l)) + } + FfnKind::Dense => { + let dest = if l + 1 == self.cfg.len() { + self.head.input_mut() + } else { + &mut self.s.x + }; + shared_routed::dense( + self.gpu, + self.w, + self.ffn, + l, + (&self.s.x1, &mut self.s.xn, &mut self.s.delta), + dest, + )?; + self.tap(l) + } + } + } + + fn shadow(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + if !self.cfg[at.layer].host_leg() { + return Ok(()); + } + shared_routed::shadow( + self.gpu, + self.w, + self.ffn, + &self.s.x1, + (port.hybrid().boundary(), at.layer), + ) + } + + fn back(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + let l = at.layer; + if !self.cfg[l].host_leg() { + return Ok(()); + } + let dest = if l + 1 == self.cfg.len() { + self.head.input_mut() + } else { + &mut self.s.x + }; + shared_routed::back(self.gpu, self.ffn, port.hybrid(), l, dest)?; + self.tap(l)?; + port.hybrid().row_enqueued(l, at.unit) + } + + fn end(&mut self, unit: usize) -> Result<(), GpuError> { + one(unit)?; + self.head.enqueue(self.gpu, self.w) + } +} + +fn one(unit: usize) -> Result<(), GpuError> { + if unit == 0 { + Ok(()) + } else { + Err(shape(format!("step walk has one unit, not unit {unit}"))) + } +} diff --git a/crates/gpu-flint/src/tests.rs b/crates/gpu-flint/src/tests.rs new file mode 100644 index 000000000..5be7729eb --- /dev/null +++ b/crates/gpu-flint/src/tests.rs @@ -0,0 +1,252 @@ +//! Full-file consumer transitions, compared against an untouched resident sequence. + +use std::num::NonZeroUsize; + +use bloomery_gpu::GpuError; +use bloomery_gpu::host::served::prompt_pass; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::model::StepMode; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::PlanInputs; +use model::placement::{PlanLevers, workstation}; + +use crate::{Body, FlintModel}; + +const CTX: usize = 1536; +const PREFIX: usize = 512; +const WRITTEN: usize = 1025; +const SLOTS: usize = 3; +const WHAT: &str = "flint full-file transitions"; + +type Error = Box; + +struct Observation { + token: u32, + pos: u32, + logits: Vec, +} + +fn observe(m: &FlintModel, token: u32) -> Result { + let logits = m.logits()?; + assert!( + logits.iter().all(|x| x.is_finite()), + "{WHAT}: finite final logits" + ); + Ok(Observation { + token, + pos: m.pos(), + logits: logits.into_iter().map(f32::to_bits).collect(), + }) +} + +fn equal(label: &str, actual: &Observation, expected: &Observation) { + assert_eq!(actual.pos, expected.pos, "{label}: sequence position"); + assert_eq!(actual.token, expected.token, "{label}: next-token argmax"); + assert_eq!( + actual.logits.len(), + expected.logits.len(), + "{label}: vocabulary" + ); + let mismatch = actual + .logits + .iter() + .zip(&expected.logits) + .position(|(a, b)| a != b); + assert!( + mismatch.is_none(), + "{label}: final logit bits first differ at {mismatch:?}" + ); +} + +fn prompt(m: &mut FlintModel, ids: &[u32]) -> Result { + prompt_pass(m, |m| { + m.run_rows(ids.len(), WHAT, |gpu, w, body, head, at| { + crate::prefill::prefill(gpu, w, body, head, ids, at) + }) + })? + .ok_or(GpuError::State { + what: WHAT, + missing: "the prompt's final token", + }) +} + +fn stream(vocab: u32, seed: u32) -> Vec { + let mut state = seed; + (0..=WRITTEN) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + state % vocab + }) + .collect() +} + +/// Two decode observations straddle the same warmed context in every arm. The remainder wraps +/// the 1024-row window ring, while leaving a saved decode result exactly at its rollback boundary. +fn feed(m: &mut FlintModel, ids: &[u32]) -> Result<([Observation; 2], Observation), GpuError> { + prompt(m, &ids[..PREFIX])?; + let first = m.step(&ids[PREFIX..PREFIX + 1])?; + let first = observe(m, first)?; + let second = m.step(&ids[PREFIX + 1..PREFIX + 2])?; + let second = observe(m, second)?; + let last = prompt(m, &ids[PREFIX + 2..WRITTEN])?; + Ok(([first, second], observe(m, last)?)) +} + +#[test] +#[ignore = "needs a CUDA card and the complete Flint GGUF in BLOOMERY_MODEL"] +fn hw_full_file_capture_slots_reset_and_ring_boundary() -> Result<(), Error> { + let path = std::env::var_os("BLOOMERY_MODEL") + .ok_or("flint full-file transitions: BLOOMERY_MODEL must name the complete Flint GGUF")?; + let file = Split::open(path)?; + let inputs = PlanInputs::read(&file)?; + assert_eq!( + inputs.spec.layers.len(), + 49, + "the released full model, not a small fixture" + ); + assert_eq!( + inputs.spec.hidden, 2048, + "the released full model residual width" + ); + let ids = stream(inputs.spec.vocab, 0x4f1a_29c3); + let foreign = stream(inputs.spec.vocab, 0xe72d_810b); + assert_ne!( + &ids[..PREFIX], + &foreign[..PREFIX], + "the interleaved slot has a distinct history" + ); + let census = bloomery_gpu::census()?; + let [device] = census.as_slice() else { + return Err(format!( + "{WHAT}: exactly one CUDA device must be visible, found {}", + census.len() + ) + .into()); + }; + let machine = workstation::plan_on( + workstation::spec_of_device(device), + inputs.spec.layers.len(), + ); + let slots = NonZeroUsize::new(SLOTS).expect("three resident sequences"); + let plan = inputs.plan_with_slots(&machine, CTX as u64, &PlanLevers::default(), slots)?; + let host = HostCfg { + populate: false, + lock: false, + card_dontneed: false, + r8: false, + route_trace: false, + }; + let mut m = Body::open_placed_slots(file, &plan, &inputs, 0, host, Residency::Off, slots)?; + m.add_slots(SLOTS)?; + + // Slot 2 is a fresh sequence run without interleaving: its outputs are the control, not a guessed numerical band. + m.select_slot(2)?; + m.set_mode(StepMode::Eager); + let (control_decode, control_end) = feed(&mut m, &ids)?; + let control_next = m.step(&ids[WRITTEN..WRITTEN + 1])?; + let control_next = observe(&m, control_next)?; + + m.select_slot(0)?; + m.set_mode(StepMode::Graph); + let (captured_decode, captured_end) = feed(&mut m, &ids)?; + for (row, (actual, expected)) in captured_decode.iter().zip(&control_decode).enumerate() { + equal( + &format!("eager versus captured decode row {row}"), + actual, + expected, + ); + } + equal( + "eager versus captured after ring wrap", + &captured_end, + &control_end, + ); + assert!(m.has_capture(), "slot 0 actually captured a decode chain"); + let capture_nodes = m.step_graph_nodes()?.len(); + + // A second captured slot must not redirect the first slot's graph or its cache handles. + m.select_slot(1)?; + feed(&mut m, &foreign)?; + assert!(m.has_capture(), "slot 1 captured its own decode chain"); + m.select_slot(0)?; + assert_eq!( + m.pos(), + WRITTEN as u32, + "switching preserved slot 0's position" + ); + assert!( + m.has_capture(), + "slot 0's capture remained parked with its stores" + ); + assert_eq!( + m.step_graph_nodes()?.len(), + capture_nodes, + "slot 0's captured chain survived the switch" + ); + let resumed = m.step(&ids[WRITTEN..WRITTEN + 1])?; + equal( + "resumed slot versus untouched control", + &observe(&m, resumed)?, + &control_next, + ); + + // Reset after overwrite, then refeed through the retained capture, must reproduce a fresh sequence. + m.reset()?; + assert_eq!(m.pos(), 0, "reset rewound the selected sequence"); + assert!(m.has_capture(), "reset retains the reusable decode capture"); + let (refed_decode, refed_end) = feed(&mut m, &ids)?; + for (actual, expected) in refed_decode.iter().zip(&control_decode) { + equal( + "reset/refeed decode versus fresh sequence", + actual, + expected, + ); + } + equal( + "reset/refeed wrapped state versus fresh sequence", + &refed_end, + &control_end, + ); + + // At written end 1025, the ring holds positions 1..1025. Replaying position 513 + // needs 1..513, but position 512 needs overwritten position 0: depth 512, not 513. + let safe = WRITTEN as u32 - 512; + let lost = safe - 1; + assert!( + m.body(WHAT)?.cut_keeps(m.pos(), safe), + "depth 512 is retained" + ); + assert!( + !m.body(WHAT)?.cut_keeps(m.pos(), lost), + "depth 513 requires reset/refeed" + ); + m.rollback(safe)?; + assert_eq!( + m.pos(), + safe, + "retained rollback moved the logical position" + ); + let refusal = m.rollback(lost); + assert!( + matches!(&refusal, Err(GpuError::Shape { what, detail }) + if *what == "flint Body" && detail.contains("ring keys") && detail.contains("reset and refeed")), + "lost-window rollback must name the ring and recovery, got {refusal:?}" + ); + assert_eq!(m.pos(), safe, "refused rollback moved no sequence position"); + let replayed = m.step(&ids[safe as usize..safe as usize + 1])?; + equal( + "retained boundary replay", + &observe(&m, replayed)?, + &control_decode[1], + ); + + // Reset/refeed of slot 0 did not erase the parked control's position or continuation. + m.select_slot(2)?; + assert_eq!( + m.pos(), + (WRITTEN + 1) as u32, + "reset did not rewind a parked sequence" + ); + Ok(()) +} diff --git a/crates/gpu-flint/src/tier.rs b/crates/gpu-flint/src/tier.rs new file mode 100644 index 000000000..6286348fe --- /dev/null +++ b/crates/gpu-flint/src/tier.rs @@ -0,0 +1,65 @@ +//! Flint's checked description and host views for the common card-leg tier. + +use std::ops::Range; + +use bloomery_gpu::GpuError; +use bloomery_gpu::host::leg_tier::{self, LegFamily, LegWidths}; +use bloomery_gpu::host::node::NodeFamily; +use gguf::Split; +use model::ModelError; +use model::arch::flint::{host, place::PlanInputs}; +use model::moe::HostLayer; +use model::placement::ModelTensors; +use model::placement::accepts::Program; +use model::r8file::R8Source; +use models::LayerSpec; + +/// Maximum tokens per expert-tier launch; the placement reserves this capacity. +pub const CHUNK: usize = model::arch::flint::place::TIER_CHUNK as usize; + +struct Flint; + +impl LegFamily for Flint { + const WHAT: &'static str = "flint tier"; + const WORD: &'static str = "flint"; + const ROWS: usize = 1; + const CHUNK: usize = CHUNK; + type Inputs = PlanInputs; + + fn describe(file: &Split) -> Result { + PlanInputs::describe(file).map_err(|e| GpuError::plan(Self::WHAT, e)) + } + + fn into_model(inputs: PlanInputs) -> ModelTensors { + inputs.model + } + + fn layers(inputs: &PlanInputs) -> &[LayerSpec] { + &inputs.spec.layers + } + + fn widths(inputs: &PlanInputs) -> LegWidths { + LegWidths { + n_expert: 256, + n_used: 6, + n_embd: inputs.spec.hidden as usize, + ff: 512, + program: Program::FlintBody, + } + } + + fn expert_input(_inputs: &PlanInputs) -> bloomery_gpu::card_leg::ExpertInput { + bloomery_gpu::card_leg::ExpertInput::AmaxF16 + } + + fn routed( + src: R8Source<'_>, + inputs: &PlanInputs, + run: Range, + ) -> Result, ModelError> { + run.map(|l| host::layer(src, &inputs.spec, l)).collect() + } +} + +/// The tier service uses the same Program and host arithmetic as the stage card. +pub const NODE_FAMILY: NodeFamily = leg_tier::node_family::(); diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index 172d3c255..e2019422b 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -60,6 +60,7 @@ bloomery-gpu-mimo2 = { path = "../gpu-mimo2", optional = true } bloomery-gpu-solar2 = { path = "../gpu-solar2", optional = true } # The MiniMax-M3 program, reached only through the `minimaxm3` feature (its sparse-attention gate). bloomery-gpu-minimaxm3 = { path = "../gpu-minimaxm3", optional = true } +bloomery-gpu-flint = { path = "../gpu-flint", optional = true } # The EmbeddingGemma 2 encoder, reached only through the `gemma-embedding2` feature (its end-to-end # gate). bloomery-gpu-gemma-embedding2 = { path = "../gpu-gemma-embedding2", optional = true } @@ -152,6 +153,7 @@ glm5next = ["deepseek41", "dep:bloomery-gpu-glm5next", "bloomery-app/glm5next"] mimo2 = ["gpu", "dep:bloomery-gpu-mimo2", "bloomery-app/mimo2"] solar2 = ["gpu", "dep:bloomery-gpu-solar2", "bloomery-app/solar2"] minimaxm3 = ["gpu", "dep:bloomery-gpu-minimaxm3", "bloomery-app/minimaxm3"] +flint = ["gpu", "dep:bloomery-gpu-flint", "bloomery-app/flint"] gemma-embedding2 = ["gpu", "gemma4a", "dep:bloomery-gpu-vision", "dep:bloomery-gpu-gemma-embedding2", "dep:bloomery-serve"] gemma4a = ["gpu", "dep:bloomery-gpu-gemma4a", "dep:bloomery-vision"] @@ -171,7 +173,7 @@ required-features = ["deepseek41"] [[bin]] name = "bloomery-serve" path = "src/bin/bloomery_serve.rs" -required-features = ["glm5next", "clef", "mimo2", "solar2", "minimaxm3", "gemma-embedding2", "hf"] +required-features = ["glm5next", "clef", "mimo2", "solar2", "minimaxm3", "flint", "gemma-embedding2", "hf"] [[bin]] name = "bloomery-serve-ds41" diff --git a/crates/gpu-gates/src/bin/bloomery_serve.rs b/crates/gpu-gates/src/bin/bloomery_serve.rs index 1811c67ca..06f4749a9 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve.rs @@ -1,5 +1,5 @@ //! `bloomery-serve` — one server binary, one seat a model: `--model -//! ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide` picks the seat +//! ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide` picks the seat //! (`serve_seats`, the shared seat modules), every other flag the seat's own, //! parsed by the seat's own parser, so each seat's flag set is exactly its //! per-model binary's. The @@ -7,7 +7,7 @@ //! picks the seat, and a bare start or `--help` prints [`drive::MODELS`], //! one line a seat with a working `--hf` example. //! -//! bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide] +//! bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide] //! [-m PATH | --model-file PATH | --hf [:]] //! [--head [--head-config ]] //! @@ -40,7 +40,8 @@ //! glm5-next the glm seat, qwen4exp the qwen38 seat, qwen3moe and qwen35moe //! the qwen3 seat, mimo2 the mimo2 seat, solar-open2 the solar2 seat, minimax-m3 //! the minimaxm3 seat, -//! nemotron_h and nemotron_h_moe the nemotron seat, raon1 the raon1 seat, muse-spark the musespark seat; +//! nemotron_h and nemotron_h_moe the nemotron seat, raon1 the raon1 seat, muse-spark the musespark seat, +//! and flint the flint seat; //! a dense qwen35 file generates through `--model qwen3`. With no model //! word it keeps the decision selection rules: a file with no head is //! refused by name, saying what would seat it (`--head`, a row's `--hf` @@ -164,19 +165,20 @@ mod drive { use serve::preflight::Driver; use crate::serve_seats::{ - decide, ds41, glm, mimo2, minimaxm3, musespark, nemotron, qwen3, qwen38, raon1, solar2, + decide, ds41, flint, glm, mimo2, minimaxm3, musespark, nemotron, qwen3, qwen38, raon1, + solar2, }; const NAME: &str = "bloomery-serve"; - const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide] \ + const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide] \ [-m PATH | --model-file PATH | --hf [:]] [--head \ [--head-config ]] [--embd-model FILE | --embd-hf \ [:]] [--embd-ctx N] [--embd-cache-bytes N] [--embd-mmproj FILE | \ --embd-no-mmproj] [--embd-image-min-tokens N] [--embd-image-max-tokens N] \ [llama-server's placement flags: -dev, -sm, -mg, -ts, -ngl, \ --cpu-moe, -fit, -fitt, -devd] | --list-devices | --version"; @@ -196,6 +198,7 @@ models (the --model word is optional with a model file: the file's architecture nemotron Nemotron 3.5 Bolt -m raon1 Raon-1-mini -m musespark Muse Spark Open 816B -m + flint Flint -m decide a decision model by its head --hf bartowski/Cloudflare_clef-flash-GGUF:Q5_K_M decide a decision model in its file --hf ggml-org/lev-GGUF:Q4_K_M"; @@ -221,6 +224,7 @@ models (the --model word is optional with a model file: the file's architecture Nemotron, Raon1, MuseSpark, + Flint, } /// A `--model` word: a generative seat's, or the decide seat's. @@ -243,9 +247,10 @@ models (the --model word is optional with a model file: the file's architecture "nemotron" => Ok(Word::Seat(Model::Nemotron)), "raon1" => Ok(Word::Seat(Model::Raon1)), "musespark" => Ok(Word::Seat(Model::MuseSpark)), + "flint" => Ok(Word::Seat(Model::Flint)), serve::decide::WORD => Ok(Word::Decide), other => Err(format!( - "--model is ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark or {}, not {other}", + "--model is ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, flint or {}, not {other}", serve::decide::WORD ) .into()), @@ -262,7 +267,7 @@ models (the --model word is optional with a model file: the file's architecture impl Model { /// Every generative seat. - const ALL: [Model; 10] = [ + const ALL: [Model; 11] = [ Model::Ds41, Model::Qwen38, Model::Glm, @@ -273,6 +278,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron, Model::Raon1, Model::MuseSpark, + Model::Flint, ]; /// The seat's word, as errors and the usage name it. @@ -288,6 +294,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron => "nemotron", Model::Raon1 => "raon1", Model::MuseSpark => "musespark", + Model::Flint => "flint", } } @@ -310,6 +317,7 @@ models (the --model word is optional with a model file: the file's architecture ), Model::Raon1 => matches!(Arch::from_name(arch), Ok(Arch::Raon1)), Model::MuseSpark => matches!(Arch::from_name(arch), Ok(Arch::MuseSpark)), + Model::Flint => matches!(Arch::from_name(arch), Ok(Arch::Flint)), } } } @@ -327,7 +335,7 @@ models (the --model word is optional with a model file: the file's architecture } let v = it.next().ok_or_else(|| { format!( - "--model needs a value (ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark or {}): {USAGE}", + "--model needs a value (ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, flint or {}): {USAGE}", serve::decide::WORD ) })?; @@ -600,6 +608,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron => nemotron::run(&rest, &side), Model::Raon1 => raon1::run(&rest, &side), Model::MuseSpark => musespark::run(&rest, &side), + Model::Flint => flint::run(&rest, &side), }; ended(r) } diff --git a/crates/gpu-gates/src/bin/gate_flint_attn.rs b/crates/gpu-gates/src/bin/gate_flint_attn.rs index 486bb3ed6..0c2ad26d6 100644 --- a/crates/gpu-gates/src/bin/gate_flint_attn.rs +++ b/crates/gpu-gates/src/bin/gate_flint_attn.rs @@ -25,10 +25,12 @@ fn main() -> std::process::ExitCode { #[cfg(feature = "gpu")] mod gate { + use bloomery_gpu::attention_h128::Kernels as LoadedAttention; use bloomery_gpu::fault::{Fault, FaultSite}; use bloomery_gpu::flash_gqa::{GROUP, HEAD, window_segments}; use bloomery_gpu::flash_gqa_h128_swa::{H128SwaArgs, H128SwaKernels}; use bloomery_gpu::qk_rms::{HEAD as RMS_HEAD, QkRmsArgs, QkRmsKernels}; + use bloomery_gpu::rope_neox::NeoxArgs; use bloomery_gpu::{Gpu, GpuError, LAYER_NONE}; use bloomery_gpu_gates::rounding::butterfly; use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, verdict}; @@ -631,6 +633,315 @@ mod gate { Ok(ok) } + fn sample(pos: usize, head: usize, dim: usize, salt: usize) -> f32 { + ((pos * 17 + head * 13 + dim * 7 + salt) % 97) as f32 / 97.0 - 0.5 + } + + fn turn_at(values: &[f32], table: &[f32], pos: usize, rotate: bool) -> Vec { + let mut out = values.to_vec(); + if rotate { + for d in 0..HEAD / 2 { + let (x, y) = (values[d], values[d + HEAD / 2]); + let (c, s) = (table[pos * HEAD + 2 * d], table[pos * HEAD + 2 * d + 1]); + out[d] = x.mul_add(c, -(y * s)); + out[d + HEAD / 2] = x.mul_add(s, y * c); + } + } + out + } + + /// Append across a wrap before a causal multirow pass. The full logical planes are the + /// independent oracle; only the device ring maps absolute rows modulo its capacity. + fn loaded_case( + gpu: &Gpu, + loaded: &LoadedAttention, + rotate: bool, + window: usize, + rows: usize, + m: usize, + ) -> Result { + let stream = gpu.stream(); + let ctx = 3072usize; + let start = 2047usize; + let kv = 2; + let heads = kv * GROUP; + let gq_h: Vec = (0..HEAD).map(|d| 0.6 + d as f32 * 0.001).collect(); + let gk_h: Vec = (0..HEAD).map(|d| 0.8 - d as f32 * 0.001).collect(); + let mut table_h = vec![0.0f32; ctx * HEAD]; + for p in 0..ctx { + for d in 0..HEAD / 2 { + let angle = p as f64 * 10000.0f64.powf(-((2 * d) as f64) / HEAD as f64); + table_h[p * HEAD + 2 * d] = angle.cos() as f32; + table_h[p * HEAD + 2 * d + 1] = angle.sin() as f32; + } + } + let mut logical_k = vec![0u16; kv * ctx * HEAD]; + let mut logical_v = vec![0u16; kv * ctx * HEAD]; + for kh in 0..kv { + for p in 0..start + m { + let raw: Vec = (0..HEAD).map(|d| sample(p, kh, d, 11)).collect(); + let key = turn_at(&rms_head(&raw, &gk_h, EPS), &table_h, p, rotate); + for d in 0..HEAD { + logical_k[(kh * ctx + p) * HEAD + d] = f32_to_f16_bits(key[d]); + logical_v[(kh * ctx + p) * HEAD + d] = + f32_to_f16_bits(sample(p, kh, d, 29) * 4.0); + } + } + } + let guard = 2 * HEAD; + let mut cache_k_h = vec![SENTINEL; kv * rows * HEAD + guard]; + let mut cache_v_h = cache_k_h.clone(); + for kh in 0..kv { + for p in start.saturating_sub(rows)..start { + let physical = (kh * rows + p % rows) * HEAD; + let logical = (kh * ctx + p) * HEAD; + cache_k_h[physical..physical + HEAD] + .copy_from_slice(&logical_k[logical..logical + HEAD]); + cache_v_h[physical..physical + HEAD] + .copy_from_slice(&logical_v[logical..logical + HEAD]); + } + } + let mut expect_k = cache_k_h.clone(); + let mut expect_v = cache_v_h.clone(); + let mut q_h = Vec::new(); + let mut raw_k = Vec::new(); + let mut raw_v = Vec::new(); + let mut expect_q = Vec::new(); + for t in 0..m { + let p = start + t; + for h in 0..heads { + let raw: Vec = (0..HEAD).map(|d| sample(p, h, d, 3)).collect(); + expect_q.extend(turn_at(&rms_head(&raw, &gq_h, EPS), &table_h, p, rotate)); + q_h.extend(raw); + } + for kh in 0..kv { + raw_k.extend((0..HEAD).map(|d| sample(p, kh, d, 11))); + raw_v.extend((0..HEAD).map(|d| sample(p, kh, d, 29) * 4.0)); + let physical = (kh * rows + p % rows) * HEAD; + let logical = (kh * ctx + p) * HEAD; + expect_k[physical..physical + HEAD] + .copy_from_slice(&logical_k[logical..logical + HEAD]); + expect_v[physical..physical + HEAD] + .copy_from_slice(&logical_v[logical..logical + HEAD]); + } + } + let pos_h: Vec = (start..start + m).map(|p| p as u32).collect(); + let counts_h: Vec = pos_h.iter().map(|p| p + 1).collect(); + let mut q = DeviceBuffer::from_host(stream, &q_h)?; + let mut kd = DeviceBuffer::from_host(stream, &raw_k)?; + let v = DeviceBuffer::from_host(stream, &raw_v)?; + let gq = DeviceBuffer::from_host(stream, &gq_h)?; + let gk = DeviceBuffer::from_host(stream, &gk_h)?; + let table = DeviceBuffer::from_host(stream, &table_h)?; + let pos = DeviceBuffer::from_host(stream, &pos_h)?; + let counts = DeviceBuffer::from_host(stream, &counts_h)?; + let sinks_h: Vec = (0..heads).map(|h| 7.0 + h as f32 * 0.3).collect(); + let sinks = DeviceBuffer::from_host(stream, &sinks_h)?; + let mut ck = DeviceBuffer::from_host(stream, &cache_k_h)?; + let mut cv = DeviceBuffer::from_host(stream, &cache_v_h)?; + let segs = window_segments(window); + let mut pv = DeviceBuffer::zeroed(stream, m * heads * segs * HEAD)?; + let mut pm = DeviceBuffer::zeroed(stream, m * heads * segs * 2)?; + let mut out = DeviceBuffer::from_host(stream, &vec![12345.0f32; m * heads * HEAD + HEAD])?; + gpu.take_fault()?; + // SAFETY: consecutive positions, distinct modulo rows, and m <= rows-window+1 + // for rings. All readable historical keys above were initialized. + unsafe { + loaded.enqueue_append( + stream, + NeoxArgs { + q: &mut q, + k: &mut kd, + v: &v, + gq: &gq, + gk: &gk, + table: &table, + pos: &pos, + eps: EPS, + n_head: heads, + n_kv: kv, + ctx, + m, + fault: gpu.unlabelled_sink(), + cache_k: &mut ck, + cache_v: &mut cv, + }, + rows, + rotate, + )?; + } + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &counts, + scale: score_scale(), + n_kv: kv, + ctx, + m, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + )?; + stream.synchronize()?; + let got_q = q.to_host_vec(stream)?; + let got = out.to_host_vec(stream)?; + let cache_same = ck.to_host_vec(stream)? == expect_k && cv.to_host_vec(stream)? == expect_v; + let query_same = bits_equal(&got_q, &expect_q); + let mut error = 0.0f32; + let mut no_sink_gap = 0.0f32; + for t in 0..m { + let end = start + t + 1; + for (h, &sink) in sinks_h.iter().enumerate() { + let b = (t * heads + h) * HEAD; + let base = (h / GROUP) * ctx * HEAD; + let keys = &logical_k[base..base + ctx * HEAD]; + let values = &logical_v[base..base + ctx * HEAD]; + let reference = attend( + &expect_q[b..b + HEAD], + keys, + values, + window_first(end, window), + end, + score_scale(), + Some(sink), + ); + error = error.max(max_abs(&got[b..b + HEAD], &reference)); + if t == 0 && h == 0 { + let no_sink = attend( + &expect_q[b..b + HEAD], + keys, + values, + window_first(end, window), + end, + score_scale(), + None, + ); + no_sink_gap = max_abs(&reference, &no_sink); + } + } + } + let canary = got[m * heads * HEAD..].iter().all(|x| *x == 12345.0); + let clean = gpu.take_fault()?.is_none(); + let mut ok = clause( + "loaded attention logical/ring", + query_same && cache_same && canary && clean && error <= BAND, + &format!( + "rotate={rotate} window={window} rows={rows} m={m} q={query_same} cache={cache_same} canary={canary} clean={clean} max_abs={error:.3e}" + ), + ); + ok &= clause( + "loaded attention sink witness", + no_sink_gap > 1e-5, + &format!("max_abs without sink={no_sink_gap:.3e}"), + ); + if rows < ctx { + ok &= refused( + "ring unsafe append-then-attend chunk", + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &counts, + scale: score_scale(), + n_kv: kv, + ctx, + m: rows - window + 2, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + ), + "invalid ring geometry", + ); + let bad_counts = DeviceBuffer::from_host(stream, &[0u32, ctx as u32 + 1])?; + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &bad_counts, + scale: score_scale(), + n_kv: kv, + ctx, + m: 2, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + )?; + stream.synchronize()?; + let invalid = out.to_host_vec(stream)?; + ok &= clause( + "ring invalid logical count", + invalid[..2 * heads * HEAD].iter().all(|v| v.is_nan()) + && gpu.take_fault()? == Some(Fault::at(LAYER_NONE, FaultSite::KeyCount)), + "zero and count past logical ctx are NaN with key_count", + ); + let invalid_pos = DeviceBuffer::from_host(stream, &[ctx as u32])?; + let before_k = ck.to_host_vec(stream)?; + let before_v = cv.to_host_vec(stream)?; + // SAFETY: the out-of-range position is never appended. + unsafe { + loaded.enqueue_append( + stream, + NeoxArgs { + q: &mut q, + k: &mut kd, + v: &v, + gq: &gq, + gk: &gk, + table: &table, + pos: &invalid_pos, + eps: EPS, + n_head: heads, + n_kv: kv, + ctx, + m: 1, + fault: gpu.unlabelled_sink(), + cache_k: &mut ck, + cache_v: &mut cv, + }, + rows, + rotate, + )?; + } + stream.synchronize()?; + ok &= clause( + "ring invalid absolute position", + q.to_host_vec(stream)?[..heads * HEAD] + .iter() + .all(|v| v.is_nan()) + && ck.to_host_vec(stream)? == before_k + && cv.to_host_vec(stream)? == before_v + && gpu.take_fault()? == Some(Fault::at(LAYER_NONE, FaultSite::CachePos)), + "position at logical ctx is NaN and preserves all physical rows", + ); + } + Ok(ok) + } + pub fn run() -> Result<(), GateError> { bloomery_levers::at_main(&[])?; let gpu = Gpu::new()?; @@ -653,6 +964,13 @@ mod gate { ok &= flash_fault(&gpu, &flash, 9, 8, "flash count past ctx")?; ok &= flash_refusals(&gpu, &flash)?; ok &= rms_case(&gpu, &rms)?; + let loaded = LoadedAttention::load(&gpu)?; + for rotate in [false, true] { + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 1024, 3)?; + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 1024, 512)?; + ok &= loaded_case(&gpu, &loaded, rotate, 0, 3072, 3)?; + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 3072, 3)?; + } if ok { Ok(()) } else { Err(checks_failed()) } } } diff --git a/crates/gpu-gates/src/bin/gate_flint_router.rs b/crates/gpu-gates/src/bin/gate_flint_router.rs index fbaee13b9..c6052fd15 100644 --- a/crates/gpu-gates/src/bin/gate_flint_router.rs +++ b/crates/gpu-gates/src/bin/gate_flint_router.rs @@ -11,8 +11,8 @@ fn main() -> std::process::ExitCode { mod gate { use bloomery_gpu::gemm::Gemm32Kernels; use bloomery_gpu::route_body::sigmoid_floor_256_6::{RouterKernels, RouterOut}; - use bloomery_gpu::routed_block::{RouteIn, RoutedRows, RouterEntry}; use bloomery_gpu::route_core::sigmoid; + use bloomery_gpu::routed_block::{RouteIn, RoutedRows, RouterEntry}; use bloomery_gpu::{DeviceTensor, Fault, FaultSite, Gpu, LAYER_NONE}; use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, verdict}; use cuda_core::DeviceBuffer; @@ -54,9 +54,13 @@ mod gate { // Independent contract expression: do not reuse the device divisor. fn weights_of(scores: &[f32], ids: &[u32], scale: f32) -> Vec { - let sum = ids.iter().fold(0.0f64, |sum, &e| sum + f64::from(scores[e as usize])); + let sum = ids + .iter() + .fold(0.0f64, |sum, &e| sum + f64::from(scores[e as usize])); let div = (sum as f32).max(FLOOR); - ids.iter().map(|&e| scores[e as usize] / div * scale).collect() + ids.iter() + .map(|&e| scores[e as usize] / div * scale) + .collect() } fn matrix(logits: &[f32]) -> Vec { @@ -89,7 +93,12 @@ mod gate { pass } - fn valid_case(cx: &Cx, name: &str, logits: &[f32], bias: Option<&[f32]>) -> Result { + fn valid_case( + cx: &Cx, + name: &str, + logits: &[f32], + bias: Option<&[f32]>, + ) -> Result { let s = cx.gpu.stream(); let mut wh = matrix(logits); if name == "dense" { @@ -111,7 +120,12 @@ mod gate { } let x = DeviceBuffer::from_host(s, &xh)?; let b = bias.map(|b| DeviceBuffer::from_host(s, b)).transpose()?; - let input = RouteIn { w: &w, bias: b.as_ref(), scale: SCALE, fault: cx.gpu.unlabelled_sink() }; + let input = RouteIn { + w: &w, + bias: b.as_ref(), + scale: SCALE, + fault: cx.gpu.unlabelled_sink(), + }; let mut out = RouterOut::new(s)?; let mut single = Pick::default(); let mut first = Pick::default(); @@ -134,103 +148,190 @@ mod gate { let mut ids = DeviceBuffer::zeroed(s, USED * TOKENS)?; let mut weights = DeviceBuffer::zeroed(s, USED * TOKENS)?; cx.router.route_rows( - (s, &cx.gpu, &cx.wide), &x, &input, TOKENS, - RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, + (s, &cx.gpu, &cx.wide), + &x, + &input, + TOKENS, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, )?; s.synchronize()?; let batch = Pick { - scores: probs.to_host_vec(s)?, ids: ids.to_host_vec(s)?, weights: weights.to_host_vec(s)?, + scores: probs.to_host_vec(s)?, + ids: ids.to_host_vec(s)?, + weights: weights.to_host_vec(s)?, }; let clean = cx.gpu.take_fault()?.is_none(); let mut expected = true; for t in 0..TOKENS { let scores = &single.scores[t * EXPERTS..(t + 1) * EXPERTS]; - let keys: Vec = scores.iter().enumerate() - .map(|(e, &p)| bias.map_or(p, |b| p + b[e])).collect(); + let keys: Vec = scores + .iter() + .enumerate() + .map(|(e, &p)| bias.map_or(p, |b| p + b[e])) + .collect(); let want_ids = top_k(&keys, USED); expected &= single.ids[t * USED..(t + 1) * USED] == want_ids; if name != "dense" { - let planted_keys: Vec = logits.iter().enumerate().map(|(e, &logit)| { - let p = sigmoid(logit * factors[t]); - bias.map_or(p, |b| p + b[e]) - }).collect(); + let planted_keys: Vec = logits + .iter() + .enumerate() + .map(|(e, &logit)| { + let p = sigmoid(logit * factors[t]); + bias.map_or(p, |b| p + b[e]) + }) + .collect(); expected &= single.ids[t * USED..(t + 1) * USED] == top_k(&planted_keys, USED); } expected &= bits_equal( - &single.weights[t * USED..(t + 1) * USED], &weights_of(scores, &want_ids, SCALE), + &single.weights[t * USED..(t + 1) * USED], + &weights_of(scores, &want_ids, SCALE), ); } let mut ok = clause( &format!("{name} selection and unbiased floor weights"), expected && clean && tickets, - &format!("bias={} clean={clean} tickets_zero={tickets}", bias.is_some()), + &format!( + "bias={} clean={clean} tickets_zero={tickets}", + bias.is_some() + ), + ); + ok &= clause( + &format!("{name} token-major rows"), + single.same(&batch), + "bit_identical", ); - ok &= clause(&format!("{name} token-major rows"), single.same(&batch), "bit_identical"); if name.starts_with("tiny") { let sum: f32 = first.ids.iter().map(|&e| first.scores[e as usize]).sum(); - let legacy: Vec = first.ids.iter() - .map(|&e| first.scores[e as usize] / (sum + 1e-20) * SCALE).collect(); - ok &= clause("tiny floor active", sum > 0.0 && sum < FLOOR - && !bits_equal(&first.weights, &legacy) - && first.weights.iter().sum::() < SCALE, "not epsilon renormalization"); + let legacy: Vec = first + .ids + .iter() + .map(|&e| first.scores[e as usize] / (sum + 1e-20) * SCALE) + .collect(); + ok &= clause( + "tiny floor active", + sum > 0.0 + && sum < FLOOR + && !bits_equal(&first.weights, &legacy) + && first.weights.iter().sum::() < SCALE, + "not epsilon renormalization", + ); } if name == "zero" || name == "ties" { - ok &= clause(&format!("{name} descending-id ties"), - first.ids == [255, 254, 253, 252, 251, 250], "six distinct ids"); + ok &= clause( + &format!("{name} descending-id ties"), + first.ids == [255, 254, 253, 252, 251, 250], + "six distinct ids", + ); if name == "zero" { - ok &= clause("zero selected sum", first.weights.iter().all(|&w| w == 0.0), "finite zero weights"); + ok &= clause( + "zero selected sum", + first.weights.iter().all(|&w| w == 0.0), + "finite zero weights", + ); } } if let Some(bias) = bias { - let keys: Vec = first.scores.iter().zip(bias).map(|(&p, &b)| p + b).collect(); - ok &= clause("bias changes selection not weight source", + let keys: Vec = first + .scores + .iter() + .zip(bias) + .map(|(&p, &b)| p + b) + .collect(); + ok &= clause( + "bias changes selection not weight source", first.ids != top_k(&first.scores, USED) && !bits_equal(&first.weights, &weights_of(&keys, &first.ids, SCALE)), - "unbiased selected scores consumed"); + "unbiased selected scores consumed", + ); } else if name == "normal" { let top7 = top_k(&first.scores, 7); - ok &= clause("six not seven selected scores", - !bits_equal(&first.weights, &weights_of(&first.scores, &top7, SCALE)[..USED]), - "top-seven renormalization differs"); + ok &= clause( + "six not seven selected scores", + !bits_equal( + &first.weights, + &weights_of(&first.scores, &top7, SCALE)[..USED], + ), + "top-seven renormalization differs", + ); } let x1 = DeviceBuffer::from_host(s, &xh[..K])?; - let graph = cx.gpu.capture(|st| cx.router.route(st, &x1, &input, &mut out))?; + let graph = cx + .gpu + .capture(|st| cx.router.route(st, &x1, &input, &mut out))?; let mut replay = graph.node_count() == 1; for _ in 0..2 { graph.launch(s)?; s.synchronize()?; replay &= read_one(cx, &out)?.same(&first) && out.tickets(s)? == 0; } - ok &= clause(&format!("{name} single capture"), replay, "one node, two exact replays, zero tickets"); - let graph = cx.gpu.capture(|st| cx.router.route_rows( - (st, &cx.gpu, &cx.wide), &x, &input, TOKENS, - RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, - ))?; + ok &= clause( + &format!("{name} single capture"), + replay, + "one node, two exact replays, zero tickets", + ); + let graph = cx.gpu.capture(|st| { + cx.router.route_rows( + (st, &cx.gpu, &cx.wide), + &x, + &input, + TOKENS, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, + ) + })?; let mut replay = graph.node_count() == 3; for _ in 0..2 { graph.launch(s)?; s.synchronize()?; let now = Pick { - scores: probs.to_host_vec(s)?, ids: ids.to_host_vec(s)?, weights: weights.to_host_vec(s)?, + scores: probs.to_host_vec(s)?, + ids: ids.to_host_vec(s)?, + weights: weights.to_host_vec(s)?, }; replay &= now.same(&single); } replay &= cx.gpu.take_fault()?.is_none(); - ok &= clause(&format!("{name} rows capture"), replay, "three nodes, two exact replays, no fault"); + ok &= clause( + &format!("{name} rows capture"), + replay, + "three nodes, two exact replays, no fault", + ); Ok(ok) } - fn invalid_case(cx: &Cx, name: &str, logit: Option, bias: Option, scale: f32) -> Result { + fn invalid_case( + cx: &Cx, + name: &str, + logit: Option, + bias: Option, + scale: f32, + ) -> Result { let s = cx.gpu.stream(); let mut logits = vec![0.0; EXPERTS]; - if let Some(logit) = logit { logits[40] = logit; } + if let Some(logit) = logit { + logits[40] = logit; + } let w = DeviceTensor::upload(s, &matrix(&logits), EXPERTS, K)?; let mut bh = vec![0.0; EXPERTS]; - if let Some(bias) = bias { bh[40] = bias; } + if let Some(bias) = bias { + bh[40] = bias; + } let b = bias.map(|_| DeviceBuffer::from_host(s, &bh)).transpose()?; let x = DeviceBuffer::from_host(s, &activations(&[1.0, 1.0]))?; - let input = RouteIn { w: &w, bias: b.as_ref(), scale, fault: cx.gpu.unlabelled_sink() }; + let input = RouteIn { + w: &w, + bias: b.as_ref(), + scale, + fault: cx.gpu.unlabelled_sink(), + }; let mut out = RouterOut::new(s)?; cx.gpu.take_fault()?; cx.router.route(s, &x, &input, &mut out)?; @@ -242,15 +343,28 @@ mod gate { let mut ids = DeviceBuffer::zeroed(s, USED * 2)?; let mut weights = DeviceBuffer::zeroed(s, USED * 2)?; cx.router.route_rows( - (s, &cx.gpu, &cx.wide), &x, &input, 2, - RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, + (s, &cx.gpu, &cx.wide), + &x, + &input, + 2, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, )?; s.synchronize()?; let rows = cx.gpu.take_fault()?; let rows_poisoned = weights.to_host_vec(s)?.iter().all(|w| !w.is_finite()); let want = Some(Fault::at(LAYER_NONE, FaultSite::Router)); - Ok(clause(name, one == want && rows == want && one_poisoned && rows_poisoned && tickets == 0, - &format!("one={one:?} rows={rows:?} poisoned={} tickets={tickets}", one_poisoned && rows_poisoned))) + Ok(clause( + name, + one == want && rows == want && one_poisoned && rows_poisoned && tickets == 0, + &format!( + "one={one:?} rows={rows:?} poisoned={} tickets={tickets}", + one_poisoned && rows_poisoned + ), + )) } fn host_boundaries(cx: &Cx) -> Result { @@ -260,20 +374,68 @@ mod gate { let short_bias = DeviceBuffer::zeroed(s, EXPERTS - 1)?; let mut out = RouterOut::new(s)?; let sink = cx.gpu.unlabelled_sink(); - let short = cx.router.enqueue_router(s, &w, &x, Some(&short_bias), SCALE, &mut out, sink).is_err(); + let short = cx + .router + .enqueue_router(s, &w, &x, Some(&short_bias), SCALE, &mut out, sink) + .is_err(); let mut probs = DeviceBuffer::zeroed(s, EXPERTS)?; let mut ids = DeviceBuffer::zeroed(s, USED)?; let mut weights = DeviceBuffer::zeroed(s, USED)?; - let empty = cx.router.enqueue_router_rows(s, &cx.gpu, &cx.wide, &w, &x, None, - SCALE, 0, &mut probs, &mut ids, &mut weights, sink).is_err(); - let short_rows = cx.router.enqueue_router_rows(s, &cx.gpu, &cx.wide, &w, &x, None, - SCALE, 2, &mut probs, &mut ids, &mut weights, sink).is_err(); - let legacy = bloomery_gpu::route_body::sigmoid_256_6::RouterKernels::load(cx.gpu.context())?; + let empty = cx + .router + .enqueue_router_rows( + s, + &cx.gpu, + &cx.wide, + &w, + &x, + None, + SCALE, + 0, + &mut probs, + &mut ids, + &mut weights, + sink, + ) + .is_err(); + let short_rows = cx + .router + .enqueue_router_rows( + s, + &cx.gpu, + &cx.wide, + &w, + &x, + None, + SCALE, + 2, + &mut probs, + &mut ids, + &mut weights, + sink, + ) + .is_err(); + let legacy = + bloomery_gpu::route_body::sigmoid_256_6::RouterKernels::load(cx.gpu.context())?; let mut legacy_out = bloomery_gpu::route_body::sigmoid_256_6::RouterOut::new(s)?; - let missing = legacy.route(s, &x, - &RouteIn { w: &w, bias: None, scale: SCALE, fault: sink }, &mut legacy_out).is_err(); - Ok(clause("host boundaries", short && empty && short_rows && missing, - "short bias, empty/short rows, legacy missing bias refused")) + let missing = legacy + .route( + s, + &x, + &RouteIn { + w: &w, + bias: None, + scale: SCALE, + fault: sink, + }, + &mut legacy_out, + ) + .is_err(); + Ok(clause( + "host boundaries", + short && empty && short_rows && missing, + "short bias, empty/short rows, legacy missing bias refused", + )) } pub fn run() -> Result<(), GateError> { @@ -283,10 +445,16 @@ mod gate { let router = RouterKernels::load(gpu.context())?; let wide = Gemm32Kernels::load(gpu.context())?; let cx = Cx { gpu, router, wide }; - let normal: Vec = (0..EXPERTS).map(|e| if e < 16 { 2.0 - e as f32 * 0.25 } else { -20.0 }).collect(); - let tiny: Vec = (0..EXPERTS).map(|e| -30.0 - (e % 16) as f32 * 0.25).collect(); + let normal: Vec = (0..EXPERTS) + .map(|e| if e < 16 { 2.0 - e as f32 * 0.25 } else { -20.0 }) + .collect(); + let tiny: Vec = (0..EXPERTS) + .map(|e| -30.0 - (e % 16) as f32 * 0.25) + .collect(); let mut bias = vec![0.0; EXPERTS]; - for (e, b) in bias.iter_mut().enumerate().skip(240) { *b = 2.0 + e as f32 / 256.0; } + for (e, b) in bias.iter_mut().enumerate().skip(240) { + *b = 2.0 + e as f32 / 256.0; + } let mut ok = valid_case(&cx, "normal", &normal, None)?; ok &= valid_case(&cx, "dense", &normal, None)?; ok &= valid_case(&cx, "biased", &normal, Some(&bias))?; @@ -294,9 +462,19 @@ mod gate { ok &= valid_case(&cx, "tiny-biased", &tiny, Some(&bias))?; ok &= valid_case(&cx, "zero", &vec![-200.0; EXPERTS], None)?; ok &= valid_case(&cx, "ties", &vec![0.0; EXPERTS], None)?; - for (name, value) in [("NaN", f32::NAN), ("+infinity", f32::INFINITY), ("-infinity", f32::NEG_INFINITY)] { + for (name, value) in [ + ("NaN", f32::NAN), + ("+infinity", f32::INFINITY), + ("-infinity", f32::NEG_INFINITY), + ] { ok &= invalid_case(&cx, &format!("{name} logit"), Some(value), None, SCALE)?; - ok &= invalid_case(&cx, &format!("{name} selection bias"), None, Some(value), SCALE)?; + ok &= invalid_case( + &cx, + &format!("{name} selection bias"), + None, + Some(value), + SCALE, + )?; } ok &= invalid_case(&cx, "nonfinite weight scale", None, None, f32::INFINITY)?; ok &= host_boundaries(&cx)?; diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index 9bcd34bc7..df267bdaf 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -298,6 +298,7 @@ mod gate { host: levers.host(), residency: Residency::Off, group: 1, + slots: std::num::NonZeroUsize::MIN, }; let pass = bits(&file, &cfg, &inputs)?; println!("{NAME}: {}", tier::tally_line()); diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs index 495f265e3..0a91647cc 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs @@ -1,6 +1,6 @@ -//! The seat of a body that runs one chain on one card, fed in prompt batches: -//! the one owner of the open, the load record, the seat's calls and the think -//! tags, for the family whose body holds one sequence ([`solar2`](super::solar2)). +//! A common hybrid HTTP seat over one chain on one card: opening, planning, +//! resident sequence selection, load records and prompt calls. Its body's +//! sequence capability preserves named refusals for single-sequence families. //! //! A seat gives its family the facts the run needs — the architectures it //! accepts, its plan inputs, the words its refusals name and the records it @@ -13,8 +13,8 @@ use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, Instant}; -use app::arch::HybridCfg; -use app::{Keep, Loaded, OpenArgs, OpenLog, Prompt, Session, SessionError}; +use app::arch::{Batch, HybridCfg, SequenceSlots}; +use app::{Loaded, OpenArgs, OpenLog, Prompt, Session, SessionError}; use bloomery_gpu::GpuModel; use bloomery_gpu::model::{ChainBody, StepMode}; use bloomery_gpu_gates::GateError; @@ -40,7 +40,7 @@ use super::tiered::{self, Open, Tiered}; pub trait Family: Sized + 'static { /// The body the seat serves: one chain on one card, fed in prompt /// batches. - type Body: ChainBody + app::Open, Cfg = HybridCfg> + Prompt + Keep; + type Body: Batch>; /// The family's hyperparameters: what its plan inputs are read for. type H; @@ -64,6 +64,11 @@ pub trait Family: Sized + 'static { /// The file's plan inputs, its own reading. fn read(split: &Split) -> Result, PlacementError>; + /// Price the chosen prompt group before the common context probes. + fn configure(_inputs: &mut PlanInputs, _group: usize) -> Result<(), GateError> { + Ok(()) + } + /// The records the seat prints, registered once at `main`. fn register(); @@ -118,6 +123,7 @@ struct Log { opened: Duration, ctx: usize, group: usize, + slots: usize, host_experts: u64, card_experts: u64, graph_nodes: usize, @@ -160,7 +166,7 @@ impl OpenLog for Log { } impl Log { - /// The `load` record of `m`, the one slot ready. + /// The `load` record of `m`, with every planned slot ready. fn record(&self, m: &GpuModel) { if let Some((card, rate)) = &self.lane { record::lane(card, rate).eprint(); @@ -170,7 +176,7 @@ impl Log { .w("arch", ::arch().name()) .u("resident_bytes", m.resident_bytes()) .u("ctx", self.ctx) - .u("slots", 1) + .u("slots", self.slots) .u("layers", m.layers().len()) .u("host_experts", self.host_experts) .u("card_experts", self.card_experts) @@ -196,6 +202,7 @@ fn open_session(a: Open<()>) -> Result, GateError> { opened: Duration::ZERO, ctx: a.ctx, group: a.group, + slots: a.slots.get(), pin_main: a.pin_main, pinned, host_experts: 0, @@ -220,16 +227,26 @@ fn open_session(a: Open<()>) -> Result, GateError> { host: a.host, residency: a.residency, group: a.group, + slots: a.slots, }, }; - let s = Loaded::::open(file, args, &mut log)? + let mut s = Loaded::::open(file, args, &mut log)? .ok_or("the open stopped at its plan")? .ready(&mut log)?; + ::Sequences::add(&mut s, a.slots)?; log.record(s.model()); Ok(Hybrid { s, ctx: a.ctx }) } impl Seat for Hybrid { + fn slots(&self) -> usize { + ::Sequences::count(&self.s) + } + + fn select(&mut self, slot: usize) -> Result<(), GateError> { + Ok(::Sequences::select(&mut self.s, slot)?) + } + fn pos(&self) -> usize { self.s.model().pos() as usize } @@ -309,7 +326,8 @@ impl Tiered for Hybrid { const ALIAS: &'static str = F::ALIAS; const FAMILY: &'static str = F::FAMILY; const SERVES: &'static str = F::SERVES; - const RESIDENT: bool = false; + const RESIDENT: bool = <::Sequences as SequenceSlots>::RESIDENT; + const ONE_PASS: bool = false; fn accepts(arch: Arch) -> bool { F::accepts(arch) @@ -319,6 +337,10 @@ impl Tiered for Hybrid { F::read(split) } + fn configure(inputs: &mut PlanInputs, group: usize) -> Result<(), GateError> { + F::configure(inputs, group) + } + fn flags(_flag: &str, _value: &str, _out: &mut ()) -> Result { Ok(false) } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs index d477ce59d..5c2c206fb 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs @@ -37,6 +37,8 @@ mod drafted; // DSpark draft, its slot rule and the draft word (`served::ServedDraft`). #[cfg(feature = "deepseek41")] pub mod ds41; +#[cfg(all(feature = "deepseek41", feature = "flint"))] +pub mod flint; #[cfg(feature = "glm5next")] pub mod glm; #[cfg(feature = "deepseek41")] @@ -80,7 +82,7 @@ pub mod minimaxm3_folder; pub mod musespark; // The seat of a body that runs one chain on one card, fed in prompt batches // (`solar2`): its open, its load record, its calls and its think tags. -#[cfg(all(feature = "deepseek41", feature = "solar2"))] +#[cfg(all(feature = "deepseek41", any(feature = "solar2", feature = "flint")))] pub mod hybrid; // The nemotron seat sits behind `deepseek41` for the same reason: the server // surface it binds through is scoped to it, and it runs no V4.1 code. The diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs index a01293fef..a6dc756f2 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs @@ -179,6 +179,12 @@ pub trait Tiered: Seat + Sized + 'static { /// The file's plan inputs, its own `place::read`. fn read(split: &Split) -> Result, PlacementError>; + /// Price the selected prompt schedule before any context probe or allocation. + /// A family whose schedule has fixed buffers keeps its header-derived inputs. + fn configure(_inputs: &mut PlanInputs, _group: usize) -> Result<(), GateError> { + Ok(()) + } + /// One of the seat's own flags (`SeatFlags::parse`'s `own`): `Ok(true)` took it. fn flags(flag: &str, value: &str, out: &mut Self::Flags) -> Result; @@ -354,7 +360,7 @@ fn tier_experts( levers: &PlanLevers, ctx: Option, ) -> Result { - let machine = S::machine(offer, S::tier_batch(offer, inputs), inputs.model.layers)?; + let machine = S::machine(offer, S::tier_batch(offer, inputs)?, inputs.model.layers)?; let ctx = ctx.map_or(Ok(1), u64::try_from)?; let plan = inputs.plan(&machine, ctx, levers)?; Ok(plan.tier_n_l.iter().flatten().sum()) @@ -438,7 +444,8 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result { place::tier_batch(&inputs.hp, T_MAX as u64) } diff --git a/crates/gpu-minimaxm3/src/tier.rs b/crates/gpu-minimaxm3/src/tier.rs index b937c9291..ca916c671 100644 --- a/crates/gpu-minimaxm3/src/tier.rs +++ b/crates/gpu-minimaxm3/src/tier.rs @@ -20,8 +20,8 @@ use gguf::Split; use model::arch::minimaxm3::host as minimax_host; use model::arch::minimaxm3::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::{ModelTensors, PlacementError}; use model::placement::workstation::TierBatchBytes; +use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::{LayerSpec, Moe}; @@ -38,7 +38,6 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// The tier's prompt-batch bytes the plan reserves on the tier card and the host for the file /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. -#[must_use] pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(inputs, T_MAX as u64) } diff --git a/crates/gpu/src/arch/qwen3moe/tier_leg.rs b/crates/gpu/src/arch/qwen3moe/tier_leg.rs index f5b383b89..d0393f193 100644 --- a/crates/gpu/src/arch/qwen3moe/tier_leg.rs +++ b/crates/gpu/src/arch/qwen3moe/tier_leg.rs @@ -50,7 +50,6 @@ const _: () = assert!(model::arch::qwen35moe::place::LEG_TIER_CHUNK == LEG_CHUNK /// Threads per block of both join entries. const THREADS: u32 = 256; - /// The routed activation the tier runs: `silu(g)·u`. A clamped SwiGLU, and /// any other activation, is refused by name. fn qwen_act(moe: &Moe) -> Result { @@ -70,7 +69,6 @@ fn qwen_act(moe: &Moe) -> Result { } } - /// The host views of the routed layers `run`, the refusal naming the run as /// `word`'s. `n_layer` is the description's layer count, so a trunk layer is /// not refused for sitting past a next-token block. @@ -155,7 +153,6 @@ impl LegFamily for Qwen3 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - } /// Qwen3.6's facts for the common tier node service. The description's @@ -210,7 +207,6 @@ impl LegFamily for Qwen35 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - } /// Qwen3-30B's tier node service ([`leg_tier::node_family`]). diff --git a/crates/gpu/src/attention_h128.rs b/crates/gpu/src/attention_h128.rs new file mode 100644 index 000000000..ee8fbd1ca --- /dev/null +++ b/crates/gpu/src/attention_h128.rs @@ -0,0 +1,617 @@ +//! Loaded head-128 GQA: independent rotation and readable-window policies over f16 KV stores. +//! +//! KV retention belongs to the runtime's Store policy. This binding owns no logical cursor: +//! the caller supplies absolute positions and causal counts, and exchanges only cache planes +//! between slots. Ring appends are cut to `capacity - window + 1` rows before attention, so +//! a later row never overwrites a key still readable by the first row of the chunk. + +use crate::attention_h128_ring; +use crate::flash_gqa::{GROUP, HEAD, window_segments}; +use crate::flash_gqa_h128_swa::{H128SwaArgs, H128SwaKernels}; +use crate::gemm::{GEMM_MAX_SLOTS, Gemm32Kernels, GemmInput, GemmKernels, GemmRoute}; +use crate::qk_rms::{QkRmsArgs, QkRmsKernels}; +use crate::rope_neox::{NeoxArgs, RopeNeoxKernels}; +use crate::rope_table::RopeRows; +use crate::site::{self, SiteTy, SiteWeight, WideActs, WideForms, WideKernels}; +use crate::weights::Weights; +use crate::{Gpu, GpuError}; +use cuda_core::DeviceBuffer; +use gguf::Split; +use model::arch::models::{LayerSpec, Mixer, OutGate, RopeMode}; +use model::arch::names; +use model::placement::kernels::attention::{Forms, Head128Scratch}; + +const WHAT: &str = "attention_h128"; +const SITES: &[SiteTy] = &[ + SiteTy::Q3K, + SiteTy::Q4K, + SiteTy::Q5K, + SiteTy::Q6K, + SiteTy::Iq4Xs, + SiteTy::Q5_0, + SiteTy::Q8_0, + SiteTy::F32, + SiteTy::Mxfp8, + SiteTy::Fp8B128, + SiteTy::Bf16, +]; + +struct Projection { + name: String, + ty: SiteTy, + rows: usize, +} + +impl Projection { + fn load( + file: &Split, + w: &Weights, + name: String, + rows: usize, + cols: usize, + ) -> Result { + let ty = site::file_site(file, WHAT, &name, (rows, cols), SITES)?; + site::site(w, WHAT, &name, (rows, cols), ty)?; + Ok(Self { name, ty, rows }) + } + + #[allow( + clippy::too_many_arguments, + reason = "one typed projection's borrowed launch state (rust-quality R8)" + )] + fn enqueue( + &self, + gpu: &Gpu, + w: &Weights, + k: &WideKernels<'_>, + a: &WideActs, + x: &DeviceBuffer, + route: &GemmRoute, + m: usize, + y: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + site::gemm( + gpu, + k, + (self.ty, w, SiteWeight::Resident(&self.name)), + self.rows, + &a.input(x), + (route, GemmInput::PerSlot), + m, + y, + ) + } +} + +/// A layer's loaded sites, gains, sinks, and independent attention policies. +pub struct Binding { + layer: usize, + hidden: usize, + heads: usize, + kv: usize, + ctx: usize, + capacity: usize, + window: usize, + rotate: bool, + eps: f32, + norm: String, + qnorm: String, + knorm: String, + sinks: String, + q: Projection, + k: Projection, + v: Projection, + o: Projection, +} + +fn vector(file: &Split, w: &Weights, name: String, len: usize) -> Result { + let (_, t) = file + .find(&name) + .ok_or_else(|| GpuError::tensor(WHAT, &name, "a vector"))?; + if t.dims != [len as u64] { + return Err(GpuError::shape( + WHAT, + format!("{name} has shape {:?}; need [{len}]", t.dims), + )); + } + let mut values = vec![0.0; len]; + gguf::quant::dequant_row(t.ty, file.data(&name)?, &mut values) + .map_err(::model::ModelError::from)?; + if let Some(i) = values.iter().position(|x| !x.is_finite()) { + return Err(GpuError::shape(WHAT, format!("{name}[{i}] is not finite"))); + } + site::vector(w, WHAT, name, len) +} + +impl Binding { + /// Bind actual file and resident formats once. No architecture-name dispatch is involved. + #[allow( + clippy::too_many_arguments, + reason = "the loaded layer and its geometry (rust-quality R8)" + )] + pub fn load( + file: &Split, + w: &Weights, + layer: usize, + spec: &LayerSpec, + hidden: usize, + eps: f32, + ctx: usize, + ) -> Result { + let Mixer::Gqa(g) = &spec.mixer else { + return Err(GpuError::shape(WHAT, "a GQA layer is required")); + }; + let heads = g.heads as usize; + let kv = g.kv_heads as usize; + let rotate = match g.rope.mode { + RopeMode::None if g.rope.dims == 0 && g.rope.yarn.is_none() => false, + RopeMode::Neox + if g.rope.dims as usize == HEAD + && g.rope.yarn.is_none() + && g.rope.base.is_finite() + && g.rope.base > 0.0 => + { + true + } + _ => return Err(GpuError::shape(WHAT, "need NoPE or whole-head plain NeoX")), + }; + let window = g.window.map_or(0, |n| n as usize); + if hidden == 0 + || ctx == 0 + || u32::try_from(ctx).is_err() + || kv == 0 + || kv.checked_mul(GROUP) != Some(heads) + || g.head_dim as usize != HEAD + || g.value_dim as usize != HEAD + || !g.qk_norm + || !g.sinks + || g.o_norm + || g.out_gate != OutGate::None + || g.select.is_some() + || g.value_scale.is_some() + || !eps.is_finite() + || eps <= 0.0 + || g.window == Some(0) + { + return Err(GpuError::shape( + WHAT, + format!("unsupported head128 layer {layer}: {g:?}"), + )); + } + // The same capacity owner sizes the plan and the physical allocation. + let capacity = + runtime::stores::Rows::Ring.count(g.window.map(u64::from), ctx as u64) as usize; + let qwidth = heads + .checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "query width overflow"))?; + let kvwidth = kv + .checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "KV width overflow"))?; + Ok(Self { + layer, + hidden, + heads, + kv, + ctx, + capacity, + window, + rotate, + eps, + norm: vector(file, w, names::attn_norm(layer), hidden)?, + qnorm: vector(file, w, names::attn_q_norm(layer), HEAD)?, + knorm: vector(file, w, names::attn_k_norm(layer), HEAD)?, + sinks: vector(file, w, names::attn_sinks(layer), heads)?, + q: Projection::load(file, w, names::attn_q(layer), qwidth, hidden)?, + k: Projection::load(file, w, names::attn_k(layer), kvwidth, hidden)?, + v: Projection::load(file, w, names::attn_v(layer), kvwidth, hidden)?, + o: Projection::load(file, w, names::attn_output(layer), hidden, qwidth)?, + }) + } + + #[must_use] + pub fn capacity(&self) -> usize { + self.capacity + } + #[must_use] + pub fn window(&self) -> usize { + self.window + } + /// Maximum append-then-attend chunk. Plane rows have no overwrite hazard. + #[must_use] + pub fn chunk_rows(&self) -> usize { + if self.capacity == self.ctx { + self.ctx + } else { + self.capacity - self.window + 1 + } + } + #[must_use] + pub fn projection_types(&self) -> [SiteTy; 4] { + [self.q.ty, self.k.ty, self.v.ty, self.o.ty] + } + + /// Full attention sub-layer: RMS, Q/K/V, head RMS and optional NeoX, append, + /// sink-folded attention, O projection, residual. Every scratch cell read is written. + /// + /// # Safety + /// `pos[0..m]` are consecutive absolute positions and `n_keys[t] == pos[t] + 1`. + /// Cache rows for earlier readable positions are initialized. The runtime Store owner + /// has established retention before a rewind; a wrapped ring cannot restore overwritten keys. + #[allow( + clippy::too_many_arguments, + reason = "borrowed layer launch state (rust-quality R8)" + )] + pub unsafe fn enqueue( + &self, + gpu: &Gpu, + w: &Weights, + k: &Kernels, + s: &mut Scratch, + cache: &mut Cache, + x: &DeviceBuffer, + x1: &mut DeviceBuffer, + pos: &DeviceBuffer, + n_keys: &DeviceBuffer, + m: usize, + rope: &RopeRows, + ) -> Result<(), GpuError> { + if m == 0 + || m > s.rows + || m > self.chunk_rows() + || s.hidden != self.hidden + || s.heads != self.heads + || s.kv != self.kv + || cache.capacity != self.capacity + || cache.kv != self.kv + || x.len() < m * self.hidden + || x1.len() < m * self.hidden + || pos.len() < m + || n_keys.len() < m + || (self.rotate && (rope.width != HEAD || rope.rows() < self.ctx)) + { + return Err(GpuError::shape( + WHAT, + format!( + "layer {} incompatible scratch/cache/rows: m={m}", + self.layer + ), + )); + } + let stream = gpu.stream(); + let fault = gpu.layer_sink(self.layer)?; + let wide = WideKernels { + gemm: &k.gemm, + g32: Some(&k.g32), + }; + gpu.elem().enqueue_rms_norm( + stream, + x, + w.f32_buf(WHAT, &self.norm)?, + self.eps, + self.hidden, + m, + &mut s.xn, + )?; + k.gemm.enqueue_route_dense(stream, m, &mut s.route, fault)?; + s.input.quantize( + gpu, + &wide, + fault, + &s.xn, + m, + &[self.q.ty, self.k.ty, self.v.ty], + )?; + self.q + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.q)?; + self.k + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.k)?; + self.v + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.v)?; + let gq = w.f32_buf(WHAT, &self.qnorm)?; + let gk = w.f32_buf(WHAT, &self.knorm)?; + // SAFETY: positions and initialized retained rows are the caller's contract; + // chunk_rows prevents wrapped appends from destroying an earlier row's window. + unsafe { + k.enqueue_append( + stream, + NeoxArgs { + q: &mut s.q, + k: &mut s.k, + v: &s.v, + gq, + gk, + table: &rope.table, + pos, + eps: self.eps, + n_head: self.heads, + n_kv: self.kv, + ctx: self.ctx, + m, + fault, + cache_k: &mut cache.k, + cache_v: &mut cache.v, + }, + self.capacity, + self.rotate, + )?; + } + k.enqueue_attention( + stream, + H128SwaArgs { + q: &s.q, + kc: &cache.k, + vc: &cache.v, + n_keys, + scale: 1.0 / (HEAD as f32).sqrt(), + n_kv: self.kv, + ctx: self.ctx, + m, + window: self.window, + sinks: Some(w.f32_buf(WHAT, &self.sinks)?), + part_v: &mut s.part_v, + part_ms: &mut s.part_ms, + fault, + y: &mut s.attended, + }, + self.heads, + self.capacity, + )?; + s.output + .quantize(gpu, &wide, fault, &s.attended, m, &[self.o.ty])?; + self.o.enqueue( + gpu, + w, + &wide, + &s.output, + &s.attended, + &s.route, + m, + &mut s.delta, + )?; + gpu.elem() + .enqueue_add(stream, x, &s.delta, m * self.hidden, x1) + } +} + +/// Only the physical cache travels with a slot. Logical retention stays in runtime::Store. +pub struct Cache { + pub k: DeviceBuffer, + pub v: DeviceBuffer, + capacity: usize, + kv: usize, +} + +impl Cache { + pub fn new(gpu: &Gpu, binding: &Binding) -> Result { + let n = binding + .kv + .checked_mul(binding.capacity) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "cache allocation overflow"))?; + Ok(Self { + k: DeviceBuffer::zeroed(gpu.stream(), n)?, + v: DeviceBuffer::zeroed(gpu.stream(), n)?, + capacity: binding.capacity, + kv: binding.kv, + }) + } + #[must_use] + pub fn bytes(&self) -> usize { + self.k.num_bytes() + self.v.num_bytes() + } +} + +/// Loaded launch modules. Their module handles allocate no device scratch. +pub struct Kernels { + gemm: GemmKernels, + g32: Gemm32Kernels, + qk: QkRmsKernels, + rope: RopeNeoxKernels, + flash: H128SwaKernels, + ring: attention_h128_ring::Kernels, +} + +impl Kernels { + pub fn load(gpu: &Gpu) -> Result { + Ok(Self { + gemm: GemmKernels::load(gpu.context())?, + g32: Gemm32Kernels::load(gpu.context())?, + qk: QkRmsKernels::load(gpu.context())?, + rope: RopeNeoxKernels::load(gpu.context())?, + flash: H128SwaKernels::load(gpu.context())?, + ring: attention_h128_ring::Kernels::load(gpu.context())?, + }) + } + #[must_use] + pub fn bytes(&self) -> usize { + 0 + } + + /// Head RMS and append, with independent rotation and physical capacity. + /// # Safety + /// Valid absolute positions are distinct modulo `rows`. + pub unsafe fn enqueue_append( + &self, + stream: &cuda_core::CudaStream, + args: NeoxArgs<'_>, + rows: usize, + rotate: bool, + ) -> Result<(), GpuError> { + if rows != args.ctx { + // SAFETY: this method's caller guarantees distinct physical rows. + return unsafe { self.ring.append(stream, args, rows, rotate) }; + } + if rotate { + // SAFETY: distinct modulo the full plane means distinct plane positions. + unsafe { self.rope.enqueue_head_norm_neox_append(stream, args) } + } else { + let NeoxArgs { + q, + k, + v, + gq, + gk, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + .. + } = args; + self.qk.enqueue( + stream, + QkRmsArgs { + q, + k, + v, + gq, + gk, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + }, + ) + } + } + + pub fn enqueue_attention( + &self, + stream: &cuda_core::CudaStream, + args: H128SwaArgs<'_>, + heads: usize, + rows: usize, + ) -> Result<(), GpuError> { + if rows == args.ctx { + self.flash.enqueue(stream, args, heads) + } else { + self.ring.attention(stream, args, heads, rows) + } + } +} + +/// Reused step/prompt storage. No allocation occurs during enqueue. +pub struct Scratch { + rows: usize, + hidden: usize, + heads: usize, + kv: usize, + xn: DeviceBuffer, + q: DeviceBuffer, + k: DeviceBuffer, + v: DeviceBuffer, + attended: DeviceBuffer, + delta: DeviceBuffer, + part_v: DeviceBuffer, + part_ms: DeviceBuffer, + input: WideActs, + output: WideActs, + route: GemmRoute, +} + +fn layout( + bindings: &[Binding], + rows: usize, +) -> Result<(usize, usize, usize, usize, WideForms, WideForms), GpuError> { + let first = bindings + .first() + .ok_or_else(|| GpuError::shape(WHAT, "at least one binding required"))?; + if !(1..=GEMM_MAX_SLOTS).contains(&rows) + || bindings + .iter() + .any(|b| b.hidden != first.hidden || b.heads != first.heads || b.kv != first.kv) + { + return Err(GpuError::shape( + WHAT, + format!("scratch needs 1..={GEMM_MAX_SLOTS} rows and equal layer widths"), + )); + } + let mut input = WideForms::NONE; + let mut output = WideForms::NONE; + let mut segs = 1; + for b in bindings { + input = input.union(WideActs::wants_of(&[b.q.ty, b.k.ty, b.v.ty])); + output = output.union(WideActs::wants(b.o.ty)); + segs = segs.max(window_segments(b.window)); + } + // The f32 source buffers are already owned above; no duplicate form is needed. + input.f32 = false; + output.f32 = false; + Ok((first.hidden, first.heads, first.kv, segs, input, output)) +} + +impl Scratch { + pub fn new(gpu: &Gpu, bindings: &[Binding], rows: usize) -> Result { + let (hidden, heads, kv, segs, input, output) = layout(bindings, rows)?; + let _ = Self::planned_bytes(bindings, rows)?; + let stream = gpu.stream(); + Ok(Self { + rows, + hidden, + heads, + kv, + xn: DeviceBuffer::zeroed(stream, rows * hidden)?, + q: DeviceBuffer::zeroed(stream, rows * heads * HEAD)?, + k: DeviceBuffer::zeroed(stream, rows * kv * HEAD)?, + v: DeviceBuffer::zeroed(stream, rows * kv * HEAD)?, + attended: DeviceBuffer::zeroed(stream, rows * heads * HEAD)?, + delta: DeviceBuffer::zeroed(stream, rows * hidden)?, + part_v: DeviceBuffer::zeroed(stream, rows * heads * segs * HEAD)?, + part_ms: DeviceBuffer::zeroed(stream, rows * heads * segs * 2)?, + input: WideActs::new(stream, input, rows, hidden)?, + output: WideActs::new(stream, output, rows, heads * HEAD)?, + route: GemmRoute::new(stream, rows, 1)?, + }) + } + /// Exact device allocation, including typed activation forms and the dense route table. + pub fn planned_bytes(bindings: &[Binding], rows: usize) -> Result { + let (hidden, heads, kv, segs, input, output) = layout(bindings, rows)?; + let shape = Head128Scratch { + hidden: hidden as u64, + query: (heads * HEAD) as u64, + key_value: (kv * HEAD) as u64, + heads: heads as u64, + segments: segs as u64, + input: Forms { + q128: input.q128, + q32: input.q32, + }, + output: Forms { + q128: output.q128, + q32: output.q32, + }, + }; + shape + .bytes(rows as u64) + .and_then(|n| usize::try_from(n).ok()) + .ok_or_else(|| GpuError::shape(WHAT, "scratch allocation overflow")) + } + #[must_use] + pub fn bytes(&self) -> usize { + [ + &self.xn, + &self.q, + &self.k, + &self.v, + &self.attended, + &self.delta, + &self.part_v, + &self.part_ms, + ] + .iter() + .map(|b| b.num_bytes()) + .sum::() + + self.input.bytes() + + self.output.bytes() + + self.route.bytes() + } +} diff --git a/crates/gpu/src/attention_h128_ring.rs b/crates/gpu/src/attention_h128_ring.rs new file mode 100644 index 000000000..bda87f776 --- /dev/null +++ b/crates/gpu/src/attention_h128_ring.rs @@ -0,0 +1,880 @@ +//! Ring-addressed head-128 primitives. Logical positions index RoPE; only KV addresses wrap. + +use crate::GpuError; +use crate::elem::rms_scale_of_mean_checked; +use crate::fault::{FaultSink, FaultSite}; +use crate::flash::f32_to_f16_bits; +use crate::flash::{MERGE_BATCH, half_bits_to_f32, online_fold}; +use crate::flash_gqa::{ + GROUP, HEAD, KEY_TILE, SEG_KEYS, SEGMENTS, seg_span, window_cut, window_segments, +}; +use crate::flash_gqa_h128_swa::{H128SwaArgs, fold_tile}; +use crate::launch_u32; +use crate::rope_neox::NeoxArgs; +use crate::rope_neox::neox_pair; +use cuda_core::{CudaContext, CudaStream, LaunchConfig1D}; +use cuda_device::{ + DisjointSlice, SharedArray, kernel, launch_bounds, launch_contract, thread, warp, +}; +use cuda_host::cuda_module; +use std::sync::Arc; + +/// u32 words of one key row, and the padded stride the key tile is staged at. +const ROW_WORDS: usize = HEAD / 2; +const K_STRIDE: usize = ROW_WORDS + 1; +/// u64 words of one value row. +const ROW_QWORDS: usize = HEAD / 4; +const ILP: usize = 4; +const THREADS: usize = GROUP * 32; +const THREADS_U32: u32 = THREADS as u32; +const MERGE_THREADS: u32 = HEAD as u32; +const _: () = assert!(THREADS_U32 as usize == THREADS && MERGE_THREADS as usize == HEAD); +const _: () = assert!(ROW_QWORDS == 32 && ROW_WORDS.is_multiple_of(ILP) && K_STRIDE % 2 == 1); +// The launch's window: 513 positions is nine 64-key segments, and a row of +// 600 live keys starts at key 87. +const _: () = { + assert!(window_segments(0) == SEGMENTS && window_segments(513) == 9); + let (first, n_w) = window_cut(600, 513); + assert!(first == 87 && n_w == 513 && first + n_w == 600); +}; + +#[inline(always)] +fn live_keys(count: u32, ctx: usize) -> usize { + let c = count as usize; + if c > ctx { 0 } else { c } +} + +#[cuda_module] +mod append_kernels { + use super::*; + const THREADS: usize = 64; + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(64)] + #[launch_contract( + domain = 1, + block = (64, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + q.len() >= m * n_head * 128, + k.len() >= m * n_kv * 128, + v.len() >= m * n_kv * 128, + gq.len() >= 128, + gk.len() >= 128, + pos.len() >= m, + cache_k.len() >= n_kv * rows * 128, + cache_v.len() >= n_kv * rows * 128 + ) + )] + pub fn qk_rms_append_ring( + gq: &[f32], + gk: &[f32], + pos: &[u32], + v: &[f32], + eps: f32, + n_head: u32, + n_kv: u32, + ctx: u32, + rows: u32, + m: u32, + fault: FaultSink, + mut q: DisjointSlice, + mut k: DisjointSlice, + mut cache_k: DisjointSlice, + mut cache_v: DisjointSlice, + ) { + static mut WSUM: SharedArray = SharedArray::UNINIT; + + let heads = (n_head + n_kv) as usize; + let b = thread::blockIdx_x() as usize; + if b >= m as usize * heads { + return; // block-uniform + } + let t = b / heads; + let h = b - t * heads; + let tid = thread::threadIdx_x() as usize; + let is_q = h < n_head as usize; + let kh = h.wrapping_sub(n_head as usize); + let base = if is_q { + (t * n_head as usize + h) * HEAD + } else { + (t * n_kv as usize + kh) * HEAD + }; + // SAFETY: t < m <= pos.len() by the launch contract. + let p = unsafe { *pos.get_unchecked(t) } as usize; + + // SAFETY: base + tid + 64 is inside the head's buffer by the launch + // contract. One thread per value pair. + let (x0, x1) = unsafe { + if is_q { + ( + *q.get_unchecked_mut(base + tid), + *q.get_unchecked_mut(base + tid + THREADS), + ) + } else { + ( + *k.get_unchecked_mut(base + tid), + *k.get_unchecked_mut(base + tid + THREADS), + ) + } + }; + let mut acc = f64::from(x0 * x0) + f64::from(x1 * x1); + acc += warp::shuffle_xor_f64(acc, 16); + acc += warp::shuffle_xor_f64(acc, 8); + acc += warp::shuffle_xor_f64(acc, 4); + acc += warp::shuffle_xor_f64(acc, 2); + acc += warp::shuffle_xor_f64(acc, 1); + // SAFETY: WSUM is this block's own shared allocation. Two slots, one + // per warp, written before the barrier that publishes them. + let ws = unsafe { SharedArray::as_raw_mut_ptr(&raw mut WSUM) }; + if warp::lane_id() == 0 { + // SAFETY: tid / 32 < 2. One lane per warp writes its slot. + unsafe { *ws.add(tid / 32) = acc }; + } + thread::sync_threads(); + // SAFETY: both slots were written before the barrier above. + let sum = unsafe { *ws.add(0) + *ws.add(1) }; + let mean = (sum / HEAD as f64) as f32; + let scale = rms_scale_of_mean_checked(mean, eps, fault, tid == 0); + + if p >= ctx as usize { + if tid == 0 { + fault.raise(FaultSite::CachePos); + } + // SAFETY: the positions read above. This thread owns them. + unsafe { + if is_q { + *q.get_unchecked_mut(base + tid) = f32::NAN; + *q.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } else { + *k.get_unchecked_mut(base + tid) = f32::NAN; + *k.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } + } + return; // block-uniform: p is the token's + } + // SAFETY: tid + 64 < 128 <= the gain's length. + let (g0, g1) = unsafe { + let g = if is_q { gq } else { gk }; + (*g.get_unchecked(tid), *g.get_unchecked(tid + THREADS)) + }; + let n0 = (scale * g0) * x0; + let n1 = (scale * g1) * x1; + + if is_q { + // SAFETY: the positions read above. This thread owns them. + unsafe { + *q.get_unchecked_mut(base + tid) = n0; + *q.get_unchecked_mut(base + tid + THREADS) = n1; + } + return; + } + // SAFETY: the positions read above. This thread owns them. + unsafe { + *k.get_unchecked_mut(base + tid) = n0; + *k.get_unchecked_mut(base + tid + THREADS) = n1; + } + let row = (kh * rows as usize + p % rows as usize) * HEAD; + // SAFETY: kh < n_kv and p % rows < rows, so the row is inside both rings. + // base + tid + 64 is inside v. One thread per pair. The tokens of + // one launch hold distinct positions, so no two blocks write one row. + unsafe { + *cache_k.get_unchecked_mut(row + tid) = f32_to_f16_bits(n0); + *cache_k.get_unchecked_mut(row + tid + THREADS) = f32_to_f16_bits(n1); + *cache_v.get_unchecked_mut(row + tid) = f32_to_f16_bits(*v.get_unchecked(base + tid)); + *cache_v.get_unchecked_mut(row + tid + THREADS) = + f32_to_f16_bits(*v.get_unchecked(base + tid + THREADS)); + } + } + + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(64)] + #[launch_contract( + domain = 1, + block = (64, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + q.len() >= m * n_head * 128, + k.len() >= m * n_kv * 128, + v.len() >= m * n_kv * 128, + gq.len() >= 128, + gk.len() >= 128, + table.len() >= ctx * 128, + pos.len() >= m, + cache_k.len() >= n_kv * rows * 128, + cache_v.len() >= n_kv * rows * 128 + ) + )] + pub fn head_norm_neox_append_ring( + gq: &[f32], + gk: &[f32], + table: &[f32], + pos: &[u32], + v: &[f32], + eps: f32, + n_head: u32, + n_kv: u32, + ctx: u32, + rows: u32, + m: u32, + fault: FaultSink, + mut q: DisjointSlice, + mut k: DisjointSlice, + mut cache_k: DisjointSlice, + mut cache_v: DisjointSlice, + ) { + static mut WSUM: SharedArray = SharedArray::UNINIT; + + let heads = (n_head + n_kv) as usize; + let b = thread::blockIdx_x() as usize; + if b >= m as usize * heads { + return; // block-uniform + } + let t = b / heads; + let h = b - t * heads; + let tid = thread::threadIdx_x() as usize; + let is_q = h < n_head as usize; + let kh = h.wrapping_sub(n_head as usize); + let base = if is_q { + (t * n_head as usize + h) * HEAD + } else { + (t * n_kv as usize + kh) * HEAD + }; + // SAFETY: t < m <= pos.len() by the launch contract. + let p = unsafe { *pos.get_unchecked(t) } as usize; + + // SAFETY: base + tid + 64 < (t·n + h + 1)·128 <= m·n·128, inside the + // head's buffer by the launch contract; one thread per value pair. + let (x0, x1) = unsafe { + if is_q { + ( + *q.get_unchecked_mut(base + tid), + *q.get_unchecked_mut(base + tid + THREADS), + ) + } else { + ( + *k.get_unchecked_mut(base + tid), + *k.get_unchecked_mut(base + tid + THREADS), + ) + } + }; + let mut acc = f64::from(x0 * x0) + f64::from(x1 * x1); + acc += warp::shuffle_xor_f64(acc, 16); + acc += warp::shuffle_xor_f64(acc, 8); + acc += warp::shuffle_xor_f64(acc, 4); + acc += warp::shuffle_xor_f64(acc, 2); + acc += warp::shuffle_xor_f64(acc, 1); + // SAFETY: WSUM is this block's own shared allocation; the raw form is + // the only way to reach it without a reference to a `static mut`. + // Two slots, one per warp, written before the barrier that publishes + // them. + let ws = unsafe { SharedArray::as_raw_mut_ptr(&raw mut WSUM) }; + if warp::lane_id() == 0 { + // SAFETY: tid / 32 < 2; one lane per warp writes its slot. + unsafe { *ws.add(tid / 32) = acc }; + } + thread::sync_threads(); + // SAFETY: both slots were written before the barrier above. + let sum = unsafe { *ws.add(0) + *ws.add(1) }; + let mean = (sum / HEAD as f64) as f32; + let scale = rms_scale_of_mean_checked(mean, eps, fault, tid == 0); + + if p >= ctx as usize { + if tid == 0 { + fault.raise(FaultSite::CachePos); + } + // SAFETY: the positions read above; this thread owns them. + unsafe { + if is_q { + *q.get_unchecked_mut(base + tid) = f32::NAN; + *q.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } else { + *k.get_unchecked_mut(base + tid) = f32::NAN; + *k.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } + } + return; // block-uniform: p is the token's + } + // SAFETY: tid + 64 < 128 <= the gain's length; p < ctx and 2·tid + 1 + // < 128, so the table pair is inside row p of the table's ctx rows. + let (g0, g1, c, s) = unsafe { + let g = if is_q { gq } else { gk }; + ( + *g.get_unchecked(tid), + *g.get_unchecked(tid + THREADS), + *table.get_unchecked(p * HEAD + 2 * tid), + *table.get_unchecked(p * HEAD + 2 * tid + 1), + ) + }; + let n0 = (scale * g0) * x0; + let n1 = (scale * g1) * x1; + let (y0, y1) = neox_pair(n0, n1, c, s); + + if is_q { + // SAFETY: the positions read above; this thread owns them. + unsafe { + *q.get_unchecked_mut(base + tid) = y0; + *q.get_unchecked_mut(base + tid + THREADS) = y1; + } + return; + } + // SAFETY: the positions read above; this thread owns them. + unsafe { + *k.get_unchecked_mut(base + tid) = y0; + *k.get_unchecked_mut(base + tid + THREADS) = y1; + } + let row = (kh * rows as usize + p % rows as usize) * HEAD; + // SAFETY: kh < n_kv and p % rows < rows, so row + 127 < n_kv·rows·128 <= the + // planes' lengths; base + tid + 64 < m·n_kv·128 <= v.len(). One + // thread per pair of the row; the tokens of one launch hold distinct + // positions (the launch contract; the engine's embedding launch + // writes consecutive ones), so no two blocks write one row. + unsafe { + *cache_k.get_unchecked_mut(row + tid) = f32_to_f16_bits(y0); + *cache_k.get_unchecked_mut(row + tid + THREADS) = f32_to_f16_bits(y1); + *cache_v.get_unchecked_mut(row + tid) = f32_to_f16_bits(*v.get_unchecked(base + tid)); + *cache_v.get_unchecked_mut(row + tid + THREADS) = + f32_to_f16_bits(*v.get_unchecked(base + tid + THREADS)); + } + } +} + +#[cuda_module] +mod flash_kernels { + use super::*; + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + n_keys_buf.len() >= m, + segs >= 1, + q.len() >= m * n_kv * 8 * 128, + kc.len() >= n_kv * rows * 128, + vc.len() >= n_kv * rows * 128, + part_v.len() >= m * n_kv * 8 * segs * 128, + part_ms.len() >= m * n_kv * 8 * segs * 2 + ) + )] + pub fn gqa_flash_seg_h128_swa_ring( + q: &[f32], + kc: &[u16], + vc: &[u16], + n_keys_buf: &[u32], + scale: f32, + n_kv: u32, + ctx: u32, + segs: u32, + seg_keys: u32, + rows: u32, + m: u32, + window: u32, + mut part_v: DisjointSlice, + mut part_ms: DisjointSlice, + ) { + static mut QS: SharedArray = SharedArray::UNINIT; + static mut KS: SharedArray = SharedArray::UNINIT; + static mut VS: SharedArray = SharedArray::UNINIT; + static mut WS: SharedArray = SharedArray::UNINIT; + + let b = thread::blockIdx_x() as usize; + let nkv = n_kv as usize; + let n_seg = segs as usize; + let tokens = m as usize; + if b >= tokens * nkv * n_seg { + return; // block-uniform + } + let t = b % tokens; + let sk = b / tokens; + let seg = sk / nkv; + let kh = sk - seg * nkv; + let tid = thread::threadIdx_x() as usize; + let w = tid / 32; + let lane = warp::lane_id() as usize; + let n_head = nkv * GROUP; + let h = kh * GROUP + w; + let idx = (t * n_head + h) * n_seg + seg; + let ctx = ctx as usize; + // SAFETY: t < m <= n_keys_buf.len() by the launch contract. + let limit = live_keys(unsafe { *n_keys_buf.get_unchecked(t) }, ctx); + let (first, n_w) = window_cut(limit, window as usize); + let span = seg_span(n_w, n_seg, seg_keys as usize); + let lo = first + seg * span; + if lo >= first + n_w { + if lane == 0 { + // SAFETY: idx < m·n_kv·GROUP·segs, both slots inside part_ms. + unsafe { + *part_ms.get_unchecked_mut(2 * idx) = f32::NEG_INFINITY; + *part_ms.get_unchecked_mut(2 * idx + 1) = 0.0; + } + } + return; // block-uniform: the window cut is the block's + } + let hi = (lo + span).min(first + n_w); + + // SAFETY: each `static mut` is this block's own shared allocation. + // The raw form reaches it without a reference. Every index below is + // inside its array, and every write is ordered before its reads by a + // block barrier (or, for the weights, by the warp sync in `fold_tile`). + let (qs, ks, vs, ws) = unsafe { + ( + SharedArray::as_raw_mut_ptr(&raw mut QS), + SharedArray::as_raw_mut_ptr(&raw mut KS), + SharedArray::as_raw_mut_ptr(&raw mut VS), + SharedArray::as_raw_mut_ptr(&raw mut WS), + ) + }; + + let qb = (t * n_head + kh * GROUP) * HEAD; + let mut i = 0usize; + while i < 4 { + // SAFETY: qb + 4·tid + i is inside row t's group, and that row is + // inside q by the launch contract. The shared index is inside + // GROUP·HEAD. + unsafe { *qs.add(4 * tid + i) = *q.get_unchecked(qb + 4 * tid + i) }; + i += 1; + } + + let plane = kh * rows as usize * HEAD; + let k64 = kc.as_ptr() as *const u64; + let v64 = vc.as_ptr() as *const u64; + let mut mx = f32::NEG_INFINITY; + let mut s_sum = 0.0f32; + let mut acc = [0.0f32; 4]; + let mut t0 = lo; + while t0 < hi { + thread::sync_threads(); + let mut i = 0usize; + while i < 4 { + let e = tid + THREADS * i; + let key = e / ROW_QWORDS; + let word = e - key * ROW_QWORDS; + let (kw, vw) = if t0 + key < hi { + // SAFETY: t0 + key < hi <= limit <= ctx, so the row is + // mapped modulo rows inside key head kh's ring. A row is 32 u64 and the + // planes start 8-byte aligned (a device allocation). + unsafe { + let r = (plane + ((t0 + key) % rows as usize) * HEAD) / 4 + word; + (*k64.add(r), *v64.add(r)) + } + } else { + (0u64, 0u64) + }; + // SAFETY: key < 32 and word < 32, inside the staged tiles. + unsafe { + *ks.add(key * K_STRIDE + 2 * word) = kw as u32; + *ks.add(key * K_STRIDE + 2 * word + 1) = (kw >> 32) as u32; + *vs.add(key * ROW_QWORDS + word) = vw; + } + i += 1; + } + thread::sync_threads(); + + let live = t0 + lane < hi; + let mut a = [0.0f32; ILP]; + let mut wd = 0usize; + while wd < ROW_WORDS { + let mut p = 0usize; + while p < ILP { + // SAFETY: lane < 32 and wd + p < ROW_WORDS, inside the + // staged key tile. The query index is inside GROUP·HEAD. + let (kw, q0, q1) = unsafe { + ( + *ks.add(lane * K_STRIDE + wd + p), + *qs.add(w * HEAD + 2 * (wd + p)), + *qs.add(w * HEAD + 2 * (wd + p) + 1), + ) + }; + let k0 = half_bits_to_f32(kw as u16); + let k1 = half_bits_to_f32((kw >> 16) as u16); + a[p] = f32::mul_add(q0, k0, a[p]); + a[p] = f32::mul_add(q1, k1, a[p]); + p += 1; + } + wd += ILP; + } + let dot = (a[0] + a[1]) + (a[2] + a[3]); + let sc = if live { dot * scale } else { f32::NEG_INFINITY }; + + // SAFETY: the weight and value tiles are this block's, the value + // tile was published by the barrier above, and w < GROUP. + unsafe { fold_tile(sc, live, &mut mx, &mut s_sum, &mut acc, ws, vs, w, lane) }; + t0 += KEY_TILE; + } + + // SAFETY: idx is inside the partials. The four dims 4·lane .. +3 are + // this lane's alone, and lane 0 writes (max, sum). + unsafe { + let o = idx * HEAD + 4 * lane; + *part_v.get_unchecked_mut(o) = acc[0]; + *part_v.get_unchecked_mut(o + 1) = acc[1]; + *part_v.get_unchecked_mut(o + 2) = acc[2]; + *part_v.get_unchecked_mut(o + 3) = acc[3]; + if lane == 0 { + *part_ms.get_unchecked_mut(2 * idx) = mx; + *part_ms.get_unchecked_mut(2 * idx + 1) = s_sum; + } + } + } + + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + n_keys_buf.len() >= m, + segs >= 1, + sinks.len() >= n_head, + part_v.len() >= m * n_head * segs * 128, + part_ms.len() >= m * n_head * segs * 2, + y.len() >= m * n_head * 128 + ) + )] + pub fn gqa_flash_merge_h128_swa_ring( + part_v: &[f32], + part_ms: &[f32], + n_keys_buf: &[u32], + sinks: &[f32], + n_head: u32, + ctx: u32, + segs: u32, + seg_keys: u32, + m: u32, + window: u32, + fault: FaultSink, + mut y: DisjointSlice, + ) { + let b = thread::blockIdx_x() as usize; + let nh = n_head as usize; + if b >= m as usize * nh { + return; // block-uniform + } + let t = b / nh; + let h = b - t * nh; + let d = thread::threadIdx_x() as usize; + let n_seg = segs as usize; + // SAFETY: t < m <= n_keys_buf.len() by the launch contract. + let limit = live_keys(unsafe { *n_keys_buf.get_unchecked(t) }, ctx as usize); + if limit == 0 { + if d == 0 { + fault.raise(FaultSite::KeyCount); + } + // SAFETY: b·HEAD + d is inside y; thread d owns it. + unsafe { *y.get_unchecked_mut(b * HEAD + d) = f32::NAN }; + return; // block-uniform: the row is the block's + } + let (_, n_w) = window_cut(limit, window as usize); + let live = n_w.div_ceil(seg_span(n_w, n_seg, seg_keys as usize)); + let mut mx = f32::NEG_INFINITY; + let mut s = 0.0f32; + let mut acc = 0.0f32; + let mut j = 0usize; + while j < live { + let mut ms = [0.0f32; 2 * MERGE_BATCH]; + let mut vs = [0.0f32; MERGE_BATCH]; + for i in 0..MERGE_BATCH { + cuda_device::thread::__unroll_config::<0>(); + if j + i < live { + let idx = b * n_seg + j + i; + // SAFETY: idx is inside the row's segments, so both + // partials and the value dim are inside the buffers. + unsafe { + ms[2 * i] = *part_ms.get_unchecked(2 * idx); + ms[2 * i + 1] = *part_ms.get_unchecked(2 * idx + 1); + vs[i] = *part_v.get_unchecked(idx * HEAD + d); + } + } + } + for i in 0..MERGE_BATCH { + cuda_device::thread::__unroll_config::<0>(); + if j + i < live && ms[2 * i + 1] != 0.0 { + (mx, s, acc) = online_fold(mx, s, acc, ms[2 * i], ms[2 * i + 1], vs[i]); + } + } + j += MERGE_BATCH; + } + // SAFETY: h < n_head <= sinks.len() by the launch contract. + let sink = unsafe { *sinks.get_unchecked(h) }; + // The sink joins last, after every live segment. + let (_, s, acc) = online_fold(mx, s, acc, sink, 1.0, 0.0); + let out = if sink.is_nan() || sink == f32::INFINITY { + f32::NAN + } else { + acc * (1.0 / s) + }; + // SAFETY: b·HEAD + d is inside y; thread d owns it. + unsafe { *y.get_unchecked_mut(b * HEAD + d) = out }; + } +} + +/// New ring entries; legacy plane entries stay in their original modules. +pub(crate) struct Kernels { + append: append_kernels::LoadedModule, + flash: flash_kernels::LoadedModule, +} + +impl Kernels { + pub(crate) fn load(ctx: &Arc) -> Result { + // SAFETY: both embedded modules belong to this crate; every launch is checked below. + let append = unsafe { crate::shared_module!(append_kernels, ctx)? }; + // SAFETY: this crate owns this embedded module too. + let flash = unsafe { crate::shared_module!(flash_kernels, ctx)? }; + Ok(Self { append, flash }) + } + + /// # Safety + /// Valid positions are distinct modulo `rows`. + pub(crate) unsafe fn append( + &self, + stream: &CudaStream, + args: NeoxArgs<'_>, + rows: usize, + rotate: bool, + ) -> Result<(), GpuError> { + const WHAT: &str = "attention_h128::ring_append"; + let NeoxArgs { + q, + k, + v, + gq, + gk, + table, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + } = args; + if n_head == 0 || n_kv == 0 || ctx == 0 || rows == 0 || rows > ctx || m == 0 || m > rows { + return Err(GpuError::shape( + WHAT, + format!( + "need heads/kv >=1, 1<=rows<=ctx and 1<=m<=rows: heads={n_head} kv={n_kv} ctx={ctx} rows={rows} m={m}" + ), + )); + } + let plane = n_kv + .checked_mul(rows) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "cache shape overflow"))?; + let q_len = m + .checked_mul(n_head) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "query shape overflow"))?; + let kv_len = m + .checked_mul(n_kv) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "key shape overflow"))?; + for (name, got, need) in [ + ("q", q.len(), q_len), + ("k", k.len(), kv_len), + ("v", v.len(), kv_len), + ("gq", gq.len(), HEAD), + ("gk", gk.len(), HEAD), + ("pos", pos.len(), m), + ("cache_k", cache_k.len(), plane), + ("cache_v", cache_v.len(), plane), + ( + "table", + table.len(), + if rotate { + ctx.checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "table shape overflow"))? + } else { + 0 + }, + ), + ] { + if got < need { + return Err(GpuError::shape( + WHAT, + format!("{name}.len() {got} < {need}"), + )); + } + } + let grid = launch_u32( + WHAT, + "grid", + m.checked_mul( + n_head + .checked_add(n_kv) + .ok_or_else(|| GpuError::shape(WHAT, "head count overflow"))?, + ) + .ok_or_else(|| GpuError::shape(WHAT, "grid overflow"))?, + )?; + let nh = launch_u32(WHAT, "heads", n_head)?; + let nkv = launch_u32(WHAT, "kv", n_kv)?; + let ctx = launch_u32(WHAT, "ctx", ctx)?; + let rows = launch_u32(WHAT, "rows", rows)?; + let m = launch_u32(WHAT, "m", m)?; + if rotate { + let prep = self + .append + .prepare_head_norm_neox_append_ring(LaunchConfig1D::new(grid, 64, 0))?; + self.append.head_norm_neox_append_ring( + stream, &prep, gq, gk, table, pos, v, eps, nh, nkv, ctx, rows, m, fault, q, k, + cache_k, cache_v, + )?; + } else { + let prep = self + .append + .prepare_qk_rms_append_ring(LaunchConfig1D::new(grid, 64, 0))?; + self.append.qk_rms_append_ring( + stream, &prep, gq, gk, pos, v, eps, nh, nkv, ctx, rows, m, fault, q, k, cache_k, + cache_v, + )?; + } + Ok(()) + } + + pub(crate) fn attention( + &self, + stream: &CudaStream, + args: H128SwaArgs<'_>, + n_head: usize, + rows: usize, + ) -> Result<(), GpuError> { + const WHAT: &str = "attention_h128::ring_attention"; + let H128SwaArgs { + q, + kc, + vc, + n_keys, + scale, + n_kv, + ctx, + m, + window, + sinks, + part_v, + part_ms, + fault, + y, + } = args; + if n_kv == 0 + || ctx == 0 + || rows == 0 + || rows > ctx + || m == 0 + || n_kv.checked_mul(GROUP) != Some(n_head) + || !scale.is_finite() + || window == 0 + || window > rows + || m > rows - window + 1 + { + return Err(GpuError::shape( + WHAT, + format!( + "invalid ring geometry: heads={n_head} kv={n_kv} ctx={ctx} rows={rows} m={m} window={window} scale={scale}" + ), + )); + } + let sinks = sinks.ok_or_else(|| GpuError::shape(WHAT, "per-head sinks required"))?; + let segs = window_segments(window); + let heads = m + .checked_mul(n_head) + .ok_or_else(|| GpuError::shape(WHAT, "heads overflow"))?; + let partials = heads + .checked_mul(segs) + .ok_or_else(|| GpuError::shape(WHAT, "partials overflow"))?; + for (name, got, need) in [ + ("q", q.len(), heads.checked_mul(HEAD)), + ( + "kc", + kc.len(), + n_kv.checked_mul(rows).and_then(|n| n.checked_mul(HEAD)), + ), + ( + "vc", + vc.len(), + n_kv.checked_mul(rows).and_then(|n| n.checked_mul(HEAD)), + ), + ("n_keys", n_keys.len(), Some(m)), + ("sinks", sinks.len(), Some(n_head)), + ("part_v", part_v.len(), partials.checked_mul(HEAD)), + ("part_ms", part_ms.len(), partials.checked_mul(2)), + ("y", y.len(), heads.checked_mul(HEAD)), + ] { + let need = + need.ok_or_else(|| GpuError::shape(WHAT, format!("{name} shape overflow")))?; + if got < need { + return Err(GpuError::shape( + WHAT, + format!("{name}.len() {got} < {need}"), + )); + } + } + let grid = launch_u32( + WHAT, + "grid", + m.checked_mul(n_kv) + .and_then(|n| n.checked_mul(segs)) + .ok_or_else(|| GpuError::shape(WHAT, "grid overflow"))?, + )?; + let nkv = launch_u32(WHAT, "kv", n_kv)?; + let ctx = launch_u32(WHAT, "ctx", ctx)?; + let rows = launch_u32(WHAT, "rows", rows)?; + let ns = launch_u32(WHAT, "segments", segs)?; + let sk = launch_u32(WHAT, "segment keys", SEG_KEYS)?; + let mu = launch_u32(WHAT, "m", m)?; + let window = launch_u32(WHAT, "window", window)?; + let nh = launch_u32(WHAT, "heads", n_head)?; + let prep = self + .flash + .prepare_gqa_flash_seg_h128_swa_ring(LaunchConfig1D::new(grid, THREADS_U32, 0))?; + self.flash.gqa_flash_seg_h128_swa_ring( + stream, + &prep, + q, + kc, + vc, + n_keys, + scale, + nkv, + ctx, + ns, + sk, + rows, + mu, + window, + &mut *part_v, + &mut *part_ms, + )?; + let prep = self + .flash + .prepare_gqa_flash_merge_h128_swa_ring(LaunchConfig1D::new( + launch_u32(WHAT, "merge grid", heads)?, + MERGE_THREADS, + 0, + ))?; + self.flash.gqa_flash_merge_h128_swa_ring( + stream, &prep, part_v, part_ms, n_keys, sinks, nh, ctx, ns, sk, mu, window, fault, y, + )?; + Ok(()) + } +} diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 468a422bd..5811138e5 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -47,6 +47,7 @@ use model::placement::kernels::{self, DownForms}; use model::arch::place_hybrid::RingForms; use model::placement::kernels::DownAct; +use crate::card_leg::ExpertInput; use crate::card_leg::{ self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, }; @@ -86,6 +87,7 @@ struct CardKernels { leg: CardLeg, places: CardSlotKernels, arms: LegArms, + add_base: bool, } impl CardFamily { @@ -173,6 +175,12 @@ impl CardBatch { &self.bufs.pre } + /// The raw routed sum, before shared experts and residuals. + #[must_use] + pub fn acc(&self) -> &DeviceBuffer { + &self.bufs.acc + } + /// The buffers a tier layer's join reads and writes over the batch. pub fn join(&mut self) -> TierJoin<'_> { self.bufs.join(&self.sel) @@ -217,22 +225,6 @@ impl CardLayer { } } -/// The form the down stack of `layer`, of type `ty`, reads ([`kernels::down`]); -/// a type the table names no down entry for is refused by name (`what`). -fn down_act( - what: &'static str, - layer: &CardLayer, - ty: gguf::GgmlType, -) -> Result { - kernels::down(ty) - .map(|(_, act)| act) - .ok_or_else(|| GpuError::Tensor { - what, - name: layer.names[2].clone(), - need: "a down stack of a type the kernel table names a down entry for", - }) -} - /// The forms a load's leg buffers hold: the forms `downs` the card layers' /// downs read, or the 128-value [`DownAct::Q8Act`] on a load with no card /// layer (a tier card's join reads only the buffers' down rows). @@ -279,6 +271,7 @@ pub struct CardExperts { n: usize, /// The union of the resident layers' selected down activation forms. downs: DownForms, + expert_input: ExpertInput, /// The one-token step's buffers. step: CardBufs, /// The buffers of a pass of several rows, on a load that runs one. @@ -392,12 +385,14 @@ impl CardExperts { leg: CardLeg::load(gpu)?, places: CardSlotKernels::load(gpu.context())?, arms, + add_base: true, }, layers: per, tier: tier_k, ff, n, downs, + expert_input: ExpertInput::Direct, step: CardBufs::new(gpu.stream(), &family, (n, ff, 1), downs)?, pass: (rows > 1) .then(|| CardBufs::new(gpu.stream(), &family, (n, ff, rows), downs)) @@ -405,6 +400,21 @@ impl CardExperts { }) } + /// Select the loaded down-input policy before making prompt batches or captures. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + self.step.leg.set_expert_input(gpu, policy)?; + if let Some(pass) = &mut self.pass { + pass.leg.set_expert_input(gpu, policy)?; + } + self.expert_input = policy; + Ok(()) + } + + /// Leave only the raw routed sum when the caller owns residual association. + pub fn set_add_base(&mut self, add_base: bool) { + self.k.add_base = add_base; + } + /// Layer `l` computes experts on the card. #[must_use] pub fn has(&self, l: usize) -> bool { @@ -460,6 +470,12 @@ impl CardExperts { &self.step.pre } + /// The raw routed sum, before shared experts and residuals. + #[must_use] + pub fn acc(&self) -> &DeviceBuffer { + &self.step.acc + } + /// The buffers a tier layer's join reads and writes over the step's row, /// the slots' places `sel` as the handoff wrote them. pub fn step_join<'a>(&'a mut self, sel: &'a DeviceBuffer) -> TierJoin<'a> { @@ -475,6 +491,14 @@ impl CardExperts { }) } + /// The raw routed sum of the last pass. + pub fn pass_acc(&self) -> Result<&DeviceBuffer, GpuError> { + self.pass.as_ref().map(|p| &p.acc).ok_or(GpuError::State { + what: self.family.what, + missing: "the buffers of a pass of several rows", + }) + } + /// Device bytes of the step's buffers and a pass's. #[must_use] pub fn bytes(&self) -> usize { @@ -485,8 +509,10 @@ impl CardExperts { /// first-prompt only. pub fn batch(&self, gpu: &Gpu, cap: usize) -> Result { let stream = gpu.stream(); + let mut bufs = CardBufs::new(stream, &self.family, (self.n, self.ff, cap), self.downs)?; + bufs.leg.set_expert_input(gpu, self.expert_input)?; Ok(CardBatch { - bufs: CardBufs::new(stream, &self.family, (self.n, self.ff, cap), self.downs)?, + bufs, sel: DeviceBuffer::zeroed(stream, cap * self.family.n_used)?, ring: None, }) @@ -518,28 +544,31 @@ impl CardExperts { )); } let shape = (cb.bufs.cap(), self.family.n_used, self.n, self.ff); - let ring = RingGemm::new(gpu, shape, half_slots, RingForms::BLOCKS32)?; + let mut ring = RingGemm::new(gpu, shape, half_slots, RingForms::BLOCKS32)?; + ring.set_expert_input(gpu, self.expert_input)?; cb.ring = Some(ring); Ok(()) } - /// Device bytes of [`CardExperts::batch`]'s buffers for `cap` tokens - /// ([`CardBatch::bytes`]'s formula): the card leg's in the card layers' - /// down forms ([`kernels::leg_bytes`]), the card sum and the sum plus the - /// base, and the places. + /// The common pure descriptor of a prompt batch's leg and selected input protection. + #[must_use] + pub fn batch_layout(&self, cap: usize) -> model::placement::workstation::TierLegScratch { + model::placement::workstation::TierLegScratch { + n_embd: self.n as u64, + ff: self.ff as u64, + n_used: self.family.n_used as u64, + chunk: cap as u64, + downs: self.downs, + input_scales: self.expert_input == ExpertInput::AmaxF16, + } + } + + /// Device bytes of [`CardExperts::batch`] before an expert stream arms its ring. #[must_use] pub fn batch_bytes(&self, cap: usize) -> usize { - let shape = self.family.shape(self.n, self.ff, cap); - let leg = kernels::leg_bytes( - shape.cap as u64, - shape.slots_per_col as u64, - shape.n as u64, - shape.ff as u64, - self.downs, - ); - usize::try_from(leg).expect("the leg's bytes fit a usize") - + 2 * size_of::() * cap * self.n - + size_of::() * cap * self.family.n_used + let bytes = kernels::ffn::card_batch_bytes(self.batch_layout(cap)) + .expect("the card batch's dimensions have representable bytes"); + usize::try_from(bytes).expect("the card batch's bytes fit a usize") } /// Layer `l`'s card slots of the step (module doc, `m = 1`), the slots' @@ -795,6 +824,9 @@ impl CardExperts { self.k .leg .sum_rows(gpu, leg, &*sel, (l, io.m, n_card), sum)?; + if !self.k.add_base { + return Ok(()); + } gpu.elem() .enqueue_add(stream, acc, io.x1, io.m * self.n, pre) })(); @@ -838,7 +870,7 @@ impl CardKernels { acc: &mut bufs.acc, }); self.leg.enqueue(gpu, &mut bufs.leg, &run, sum)?; - if tiered { + if tiered || !self.add_base { return Ok(()); } let stream = gpu.stream(); @@ -992,11 +1024,12 @@ mod tests { vec![(0..CARD as u32).collect(); triples.len()], ) .expect("the tier set"); - let mut tier = LegTier::new( - &gpu, &specs, &set, &w, shape, FAMILY.program, - |moe| Ok(Act::SwigluClamp { limit: moe.act.swiglu_limit() }), - ) - .expect("the tier wrapper"); + let mut tier = LegTier::new(&gpu, &specs, &set, &w, shape, FAMILY.program, |moe| { + Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit(), + }) + }) + .expect("the tier wrapper"); assert_eq!(tier.block_bytes(), counted(shape.cap)); let host_x: Vec = (0..CAP * N).map(|i| (i % 71) as f32 / 71.0 - 0.5).collect(); let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); @@ -1163,8 +1196,15 @@ mod tests { assert!( matches!( LegTier::new( - &gpu, &specs, &set, &w, family.shape(N, FF, 2), program, - |moe| Ok(Act::SwigluClamp { limit: moe.act.swiglu_limit() }), + &gpu, + &specs, + &set, + &w, + family.shape(N, FF, 2), + program, + |moe| Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit() + }), ), Err(GpuError::Tensor { .. }), ), diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index d4e28c30c..a187dddad 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -69,6 +69,7 @@ //! down GEMM. use crate::card_slots::{CardAcc, CardSlotKernels, CardSlots}; +use crate::expert_input::{Protection, Slots}; use crate::fault::FaultSink; use crate::gemm::{ Gemm32Args, Gemm32Kernels, Gemm32Weight, GemmAct, GemmAct32, GemmArgs, GemmInput, GemmKernels, @@ -88,6 +89,7 @@ use crate::weights::{DevWeight, Weights}; use crate::{DeviceTensor, Gpu, GpuError, Q8Act}; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use gguf::GgmlType; +pub use model::ops::ExpertInput; use model::placement::accepts::{self, LegArms}; use model::placement::kernels::{self, DownAct, DownEntry, DownForms, GateUpEntry, GemmEntry}; use std::sync::Arc; @@ -154,6 +156,7 @@ pub struct LegBufs { h: DeviceBuffer, act_h: ActForms, down: DeviceBuffer, + protection: Option, } impl LegBufs { @@ -202,9 +205,22 @@ impl LegBufs { h: DeviceBuffer::zeroed(stream, slots * shape.ff)?, act_h, down: DeviceBuffer::zeroed(stream, slots * shape.n)?, + protection: None, }) } + /// Select the loaded down-input policy before capture or execution. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + match policy { + ExpertInput::Direct => self.protection = None, + ExpertInput::AmaxF16 if self.protection.is_none() => { + self.protection = Some(Protection::new(gpu, self.shape.slots())?); + } + ExpertInput::AmaxF16 => {} + } + Ok(()) + } + /// The shape the buffers were made for. #[must_use] pub fn shape(&self) -> LegShape { @@ -215,7 +231,11 @@ impl LegBufs { /// their shape and forms. #[must_use] pub fn bytes(&self) -> usize { - self.act_x.device_bytes() + self.h.num_bytes() + self.act_h.bytes() + self.down.num_bytes() + self.act_x.device_bytes() + + self.h.num_bytes() + + self.act_h.bytes() + + self.down.num_bytes() + + self.protection.as_ref().map_or(0, Protection::bytes) } /// The down rows of the last run: `n` values a slot, slot-major, the @@ -386,6 +406,29 @@ impl RouteBlock<'_> { n_slots: usize, (io, rule): (RouteIo<'_>, Act), fault: FaultSink, + ) -> Result<(), GpuError> { + self.enqueue_input( + gpu, + (gemm, g32), + (route, top_k), + n_slots, + (io, rule), + None, + fault, + ) + } + + /// The routed block under its loaded expert-input policy. + #[allow(clippy::too_many_arguments, reason = "route execution contract")] + pub(crate) fn enqueue_input( + &self, + gpu: &Gpu, + (gemm, g32): (Option<&GemmKernels>, &Gemm32Kernels), + (route, top_k): (&GemmRoute, usize), + n_slots: usize, + (io, rule): (RouteIo<'_>, Act), + mut protection: Option<&mut Protection>, + fault: FaultSink, ) -> Result<(), GpuError> { const WHAT_B: &str = "card_leg::RouteBlock"; let stream = gpu.stream(); @@ -461,12 +504,34 @@ impl RouteBlock<'_> { )); } } + let (ids, counts) = route.expert_slots(); + let selected = Slots { + ids, + counts, + routed: true, + n_card: 0, + len: n_slots, + }; + if let Some(p) = protection.as_deref_mut() { + p.kernels.normalize( + gpu, + (g, u, &mut p.scales), + &selected, + self.ff, + Some(rule), + fault, + )?; + } let per_slot = GemmInput::PerSlot; match self.down { RouteW::Grouped(ty, w) => { let act = h128.ok_or_else(|| lacks("q8_1 blocks of 128 (the down)"))?; let gemm = grouped_k()?; - gemm.enqueue_swiglu_act_quant(stream, g, u, rule, n_slots, act, fault)?; + if protection.is_some() { + gpu.enqueue_quantize_gemm(g, n_slots, act, fault)?; + } else { + gemm.enqueue_swiglu_act_quant(stream, g, u, rule, n_slots, act, fault)?; + } gemm.enqueue_gemm( stream, GemmArgs { @@ -478,15 +543,21 @@ impl RouteBlock<'_> { input: per_slot, y, }, - ) + )?; } RouteW::Blocks32(w) => { let act = h32.ok_or_else(|| lacks("q8 blocks of 32 (the down)"))?; - match rule { - Act::SiluMul => { - g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)? + if protection.is_some() { + g32.enqueue_quantize_gemm32(stream, g, n_slots, act, fault)?; + } else { + match rule { + Act::SiluMul => { + g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)? + } + _ => { + g32.enqueue_swiglu_act_quant32(stream, g, u, rule, n_slots, act, fault)? + } } - _ => g32.enqueue_swiglu_act_quant32(stream, g, u, rule, n_slots, act, fault)?, } g32.enqueue_gemm32( stream, @@ -498,9 +569,14 @@ impl RouteBlock<'_> { input: per_slot, y, }, - ) + )?; } } + if let Some(p) = protection { + p.kernels + .restore(gpu, y, &p.scales, &selected, self.hidden)?; + } + Ok(()) } } @@ -636,7 +712,6 @@ impl<'a> LegStacks<'a> { ) } - /// [`LegStacks::of`] for a body that loads every tensor in the format /// [`gguf::card::CardFormat::of`] names (`Weights::load`), whose downs may /// also be Q5_0 in the `pack_q5_0` layout ([`accepts::resident_down_runs`]). @@ -879,6 +954,7 @@ struct Scratch<'a> { act_x: &'a mut Q8Act, h: &'a mut DeviceBuffer, act_h: &'a mut ActForms, + protection: &'a mut Option, } impl CardLeg { @@ -924,6 +1000,7 @@ impl CardLeg { h, act_h, down, + protection, } = bufs; let shape = *shape; if sum.is_some() { @@ -934,6 +1011,7 @@ impl CardLeg { act_x, h, act_h, + protection, }; self.launch(gpu, scratch, run, down)?; let Some(sum) = sum else { @@ -966,6 +1044,7 @@ impl CardLeg { act_x, h, act_h, + protection, .. } = bufs; let scratch = Scratch { @@ -973,6 +1052,7 @@ impl CardLeg { act_x, h, act_h, + protection, }; self.launch(gpu, scratch, run, down) } @@ -1029,6 +1109,7 @@ impl CardLeg { act_x, h, act_h, + protection, } = scratch; if run.m == 0 || run.m > cap { return Err(GpuError::Shape { @@ -1114,6 +1195,23 @@ impl CardLeg { } _ => return Err(st.gate.unrun()), } + let selected = Slots { + ids: run.sel, + counts: run.sel, + routed: false, + n_card: run.n_card, + len: slots, + }; + if let Some(p) = protection { + p.kernels.normalize( + gpu, + (h, run.normed, &mut p.scales), + &selected, + ff, + None, + fault, + )?; + } let q = QuantSel { x: h, cols: 0..slots, @@ -1232,6 +1330,9 @@ impl CardLeg { } _ => return Err(st.down.unrun()), } + if let Some(p) = protection { + p.kernels.restore(gpu, down, &p.scales, &selected, n)?; + } Ok(()) } } diff --git a/crates/gpu/src/expert_input.rs b/crates/gpu/src/expert_input.rs new file mode 100644 index 000000000..47f3a10f1 --- /dev/null +++ b/crates/gpu/src/expert_input.rs @@ -0,0 +1,257 @@ +//! Per-slot protection of quantized expert down inputs and restoration before routing. + +use crate::fault::{FaultSink, FaultSite}; +use crate::kquant::{Act, act::apply_checked}; +use crate::{Gpu, GpuError, launch_u32}; +use cuda_core::{DeviceBuffer, LaunchConfig1D}; +use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread, warp}; +use cuda_host::cuda_module; + +/// Slot selection: resident `_sel` places, or the route's unique slot list and refused count. +pub(crate) struct Slots<'a> { + pub ids: &'a DeviceBuffer, + pub counts: &'a DeviceBuffer, + pub routed: bool, + pub n_card: usize, + pub len: usize, +} + +#[cuda_module] +mod expert_input_kernels { + use super::*; + + #[inline(always)] + unsafe fn slot( + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + len: u32, + i: usize, + ) -> Option { + if i >= len as usize { + return None; + } + if routed != 0 { + // SAFETY: the route owns two counters; its slot list has len entries. + if i >= len as usize - unsafe { *counts.get_unchecked(1) } as usize { + return None; + } + // SAFETY: i < len <= ids.len(). Route slots are unique and below len. + Some(unsafe { *ids.get_unchecked(i) } as usize) + } else { + // SAFETY: i < len <= ids.len(). + (unsafe { *ids.get_unchecked(i) } < n_card).then_some(i) + } + } + + #[allow(clippy::too_many_arguments, reason = "flat device ABI")] + #[kernel] + #[launch_bounds(32)] + #[launch_contract(domain = 1, block = (32, 1, 1), requires = ( + ids.len() >= slots, counts.len() >= 2 * routed, h.len() >= slots * width, + scales.len() >= slots, u.len() >= activate * slots * width + ))] + pub fn normalize( + mut h: DisjointSlice, + u: &[f32], + mut scales: DisjointSlice, + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + slots: u32, + width: u32, + activate: u32, + act: u32, + limit: f32, + fault: FaultSink, + ) { + let i = thread::index_1d().get() / 32; + // SAFETY: the launch contract bounds ids and counts; route invariants bound slot ids. + let Some(s) = (unsafe { slot(ids, counts, routed, n_card, slots, i) }) else { + return; + }; + let lane = warp::lane_id() as usize; + let base = s * width as usize; + let mut max = 0.0f32; + let mut bad = false; + let mut d = lane; + while d < width as usize { + // SAFETY: s < slots, d < width; each lane owns its strided cells. + let mut v = unsafe { *h.get_unchecked_mut(base + d) }; + if activate != 0 { + // SAFETY: u holds slots whole rows when activate is selected. + v = apply_checked(act, limit, v, unsafe { *u.get_unchecked(base + d) }, fault); + // SAFETY: this lane alone owns this cell. + unsafe { *h.get_unchecked_mut(base + d) = v }; + } + bad |= !v.is_finite(); + max = max.max(v.abs()); + d += 32; + } + let max = warp::reduce_max_f32(max); + let bad = warp::ballot(bad) != 0; + // Keep the pinned graph's log/multiply/ceil/clamp/multiply/exp order. + let e = if bad { + f32::NAN + } else { + (((max * (1.0 / 1024.0)).ln() * std::f32::consts::LOG2_E) + .ceil() + .clamp(0.0, 127.0) + * std::f32::consts::LN_2) + .exp() + }; + if lane == 0 { + // SAFETY: this warp uniquely owns slot s. + unsafe { *scales.get_unchecked_mut(s) = e }; + if bad { + fault.raise(FaultSite::QuantColumn); + } + } + d = lane; + while d < width as usize { + // SAFETY: the same lane owns this cell, and s < slots. + unsafe { + let v = *h.get_unchecked_mut(base + d); + *h.get_unchecked_mut(base + d) = v / e; + } + d += 32; + } + } + + #[allow(clippy::too_many_arguments, reason = "flat device ABI")] + #[kernel] + #[launch_bounds(32)] + #[launch_contract(domain = 1, block = (32, 1, 1), requires = ( + ids.len() >= slots, counts.len() >= 2 * routed, y.len() >= slots * width, scales.len() >= slots + ))] + pub fn restore( + mut y: DisjointSlice, + scales: &[f32], + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + slots: u32, + width: u32, + ) { + let i = thread::index_1d().get() / 32; + // SAFETY: launch bounds and route invariants are as in normalize. + let Some(s) = (unsafe { slot(ids, counts, routed, n_card, slots, i) }) else { + return; + }; + // SAFETY: s < slots <= scales.len(). + let e = unsafe { *scales.get_unchecked(s) }; + let mut d = warp::lane_id() as usize; + while d < width as usize { + // SAFETY: each lane owns its cells of the unique selected slot. + unsafe { + let v = *y.get_unchecked_mut(s * width as usize + d); + *y.get_unchecked_mut(s * width as usize + d) = v * e; + } + d += 32; + } + } +} + +pub(crate) struct ExpertInputKernels { + module: expert_input_kernels::LoadedModule, +} + +impl ExpertInputKernels { + pub fn load(gpu: &Gpu) -> Result { + // SAFETY: this crate owns the embedded module and enforces its launch contracts. + let module = unsafe { crate::shared_module!(expert_input_kernels, gpu.context())? }; + Ok(Self { module }) + } + + pub fn normalize( + &self, + gpu: &Gpu, + (h, u, scales): ( + &mut DeviceBuffer, + &DeviceBuffer, + &mut DeviceBuffer, + ), + slots: &Slots<'_>, + width: usize, + rule: Option, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "ExpertInput::normalize"; + let (act, limit) = rule.unwrap_or(Act::SiluMul).code(what)?; + let len = launch_u32(what, "slots", slots.len)?; + let width = launch_u32(what, "width", width)?; + let n_card = launch_u32(what, "n_card", slots.n_card)?; + let prep = self + .module + .prepare_normalize(LaunchConfig1D::new(len, 32, 0))?; + self.module.normalize( + gpu.stream(), + &prep, + h, + u, + scales, + slots.ids, + slots.counts, + u32::from(slots.routed), + n_card, + len, + width, + u32::from(rule.is_some()), + act, + limit, + fault, + )?; + Ok(()) + } + + pub fn restore( + &self, + gpu: &Gpu, + y: &mut DeviceBuffer, + scales: &DeviceBuffer, + slots: &Slots<'_>, + width: usize, + ) -> Result<(), GpuError> { + let what = "ExpertInput::restore"; + let len = launch_u32(what, "slots", slots.len)?; + let width = launch_u32(what, "width", width)?; + let n_card = launch_u32(what, "n_card", slots.n_card)?; + let prep = self + .module + .prepare_restore(LaunchConfig1D::new(len, 32, 0))?; + self.module.restore( + gpu.stream(), + &prep, + y, + scales, + slots.ids, + slots.counts, + u32::from(slots.routed), + n_card, + len, + width, + )?; + Ok(()) + } +} + +pub(crate) struct Protection { + pub kernels: ExpertInputKernels, + pub scales: DeviceBuffer, +} + +impl Protection { + pub fn new(gpu: &Gpu, slots: usize) -> Result { + Ok(Self { + kernels: ExpertInputKernels::load(gpu)?, + scales: DeviceBuffer::zeroed(gpu.stream(), slots)?, + }) + } + + pub fn bytes(&self) -> usize { + self.scales.num_bytes() + } +} diff --git a/crates/gpu/src/flash_gqa_h128_swa.rs b/crates/gpu/src/flash_gqa_h128_swa.rs index 68379d062..7d1d7c0a5 100644 --- a/crates/gpu/src/flash_gqa_h128_swa.rs +++ b/crates/gpu/src/flash_gqa_h128_swa.rs @@ -67,7 +67,7 @@ fn live_keys(count: u32, ctx: usize) -> usize { clippy::too_many_arguments, reason = "the tile step's state is the caller's registers, passed by reference" )] -unsafe fn fold_tile( +pub(crate) unsafe fn fold_tile( sc: f32, live: bool, mx: &mut f32, diff --git a/crates/gpu/src/gemm/route.rs b/crates/gpu/src/gemm/route.rs index 2abfceb70..49607235d 100644 --- a/crates/gpu/src/gemm/route.rs +++ b/crates/gpu/src/gemm/route.rs @@ -159,6 +159,11 @@ impl GemmRoute { self.n_experts } + /// The unique active-slot list and counters for per-slot expert operations. + pub(crate) fn expert_slots(&self) -> (&DeviceBuffer, &DeviceBuffer) { + (&self.cols, &self.n_tiles) + } + /// Device bytes of the table. #[must_use] pub fn bytes(&self) -> usize { diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 51df65d8d..502a05bca 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -281,8 +281,8 @@ mod handoff_kernels { unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; } - /// [`ds41_ffn_handoff`] and the row's f32 tier image, with the same - /// per-thread writes and following go as [`ds41_ffn_handoff_8_tier`]. + /// Six slots' f32 tier image, with the same per-thread writes and + /// following go as the four- and eight-slot tier entries. #[allow( clippy::too_many_arguments, reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" @@ -1630,8 +1630,10 @@ impl HandoffKernels { /// slot entries. The generated launch checks the input and output spans. fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result { let slots = HandoffSlots::of(lay.n_used); - if !matches!(slots, Ok(HandoffSlots::Four | HandoffSlots::Six | HandoffSlots::Eight)) - || tl.n_used != lay.n_used + if !matches!( + slots, + Ok(HandoffSlots::Four | HandoffSlots::Six | HandoffSlots::Eight) + ) || tl.n_used != lay.n_used || tl.x_len != lay.hidden || tl.q3_words != 0 || tl.d8_len != 0 diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 04572a78a..0ed10e8c0 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -40,7 +40,7 @@ use super::run::{HostRun, HostWidths}; use super::tier::{ TierAct, TierBlock, TierCard, TierExperts, TierInput, TierIo, TierOpen, TierSet, TierShape, }; -use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks}; +use crate::card_leg::{CardLeg, ExpertInput, LegBufs, LegRun, LegShape, LegStacks}; use crate::hybrid::{Hybrid, SlotMap}; use crate::kquant::Act; use crate::model::CardBytes; @@ -264,18 +264,10 @@ impl LegTier { arms, }) } -} -/// `names` read by `arms`, or by the card leg's own arms when the family -/// named none ([`LegStacks::of`]). -fn stacks_of<'a>( - arms: Option<(LegArms, (&'static str, &'static str))>, - w: &'a Weights, - names: [&'a str; 3], -) -> Result, GpuError> { - match arms { - None => LegStacks::of(w, names), - Some((arms, roles)) => LegStacks::of_arms(w, names, arms, roles), + /// Select the loaded policy shared by stage-card and tier-card experts. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + self.bufs.set_expert_input(gpu, policy) } } @@ -330,7 +322,7 @@ impl TierExperts for LegTier { .. } = *shape; let tl = layer_of(layers, cfg, layer, W)?; - let stacks = LegStacks::of_arms(weights, tl.names(), *arms, STACK_ROLES)?; + let stacks = LegStacks::of_arms(weights, tl.names(), arms, STACK_ROLES)?; for c0 in (0..io.cols).step_by(cap) { let cn = cap.min(io.cols - c0); let slots = cn * slots_per_col; @@ -365,7 +357,6 @@ impl TierExperts for LegTier { io: TierIo<'_>, ) -> Result<(), GpuError> { const W: &str = "LegTier::enqueue_layer"; - let arms = self.arms; let tl = layer_of(&self.layers, &self.cfg, layer, W)?; let TierInput::F32(x) = io.act else { return Err(GpuError::Shape { @@ -523,6 +514,11 @@ pub trait LegFamily { /// The file as the family's placement reads it. type Inputs; + /// The loaded down-input policy, shared with the stage card and host. + fn expert_input(_inputs: &Self::Inputs) -> ExpertInput { + ExpertInput::Direct + } + /// `file` read as the family's placement reads it; its refusals are /// the family's. fn describe(file: &Split) -> Result; @@ -569,7 +565,6 @@ pub trait LegFamily { limit: moe.act.swiglu_limit(), }) } - } /// The family `F`'s tier node service: the placement tensors of a file, the @@ -614,7 +609,7 @@ fn node_experts( let inputs = node_inputs::(file, open)?; let widths = F::widths(&inputs); let shape = widths.shape(F::CHUNK); - Ok(Box::new(LegTier::new( + let mut tier = LegTier::new( gpu, F::layers(&inputs), &open.set, @@ -622,7 +617,9 @@ fn node_experts( shape, widths.program, F::tier_act, - )?)) + )?; + tier.set_expert_input(gpu, F::expert_input(&inputs))?; + Ok(Box::new(tier)) } fn node_host(file: Arc, open: &NodeOpen) -> Result { @@ -640,8 +637,12 @@ fn node_host(file: Arc, open: &NodeOpen) -> Result(views) }, ) } diff --git a/crates/gpu/src/host/run.rs b/crates/gpu/src/host/run.rs index 405d5aade..01d434628 100644 --- a/crates/gpu/src/host/run.rs +++ b/crates/gpu/src/host/run.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use gguf::Split; use model::moe::{HostLayer, HostScratch, UNION_MAX_COLS, UnionScratch}; -use model::ops::PostNorm; +use model::ops::{ExpertInput, PostNorm}; use model::r8file::{R8Pair, R8Source}; use model::{Tensor2, Tensor2View}; @@ -174,6 +174,9 @@ impl HostRun { scratch.len() - 1 } }; + if view.expert_input() == ExpertInput::AmaxF16 { + scratch[scratch_of[l]].set_expert_input(ExpertInput::AmaxF16); + } layers[l] = Some(view); } Ok(HostRun { @@ -257,13 +260,15 @@ impl HostRun { .scratch .iter() .map(|s| { - if cols <= UNION_MAX_COLS { + let mut union = if cols <= UNION_MAX_COLS { UnionScratch::new_routed(embd, s.ff(), cols, n_used) } else { let groups = cols.div_ceil(UNION_MAX_COLS); let slots = groups * UNION_MAX_COLS * n_used; UnionScratch::new_tail(embd, s.ff(), groups, slots, n_used) - } + }?; + union.set_expert_input(s.expert_input()); + Ok::<_, model::ModelError>(union) }) .collect::>()?; Ok(()) diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index d478347e7..31a876adb 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -29,6 +29,8 @@ use cuda_host::cuda_module; use std::sync::{Arc, Mutex, PoisonError}; pub mod arch; +pub mod attention_h128; +mod attention_h128_ring; pub mod cand; pub mod capsync; pub mod card_experts; @@ -41,6 +43,7 @@ pub mod draft; pub mod draft_taps; pub mod dspark; pub mod elem; +mod expert_input; pub mod fault; pub mod flash; pub mod flash_gqa; diff --git a/crates/gpu/src/ring_gemm.rs b/crates/gpu/src/ring_gemm.rs index 008cedc73..f631a68dc 100644 --- a/crates/gpu/src/ring_gemm.rs +++ b/crates/gpu/src/ring_gemm.rs @@ -8,7 +8,9 @@ use cuda_core::DeviceBuffer; use model::arch::place_hybrid::{RingForms, ring_pass_bytes_in}; +use crate::card_leg::ExpertInput; use crate::card_leg::{RouteBlock, RouteIo}; +use crate::expert_input::Protection; use crate::gemm::{Gemm32Kernels, GemmAct, GemmAct32, GemmKernels, GemmRoute}; use crate::host::xstream::LandBatch; use crate::kquant::Act; @@ -50,6 +52,8 @@ pub(crate) struct RingGemm { h: Rows, route: GemmRoute, cap: usize, + slots: usize, + protection: Option, } impl RingGemm { @@ -72,6 +76,8 @@ impl RingGemm { h: Rows::new(gpu, slots, ff, forms.down32)?, route: GemmRoute::new(stream, slots, half_slots)?, cap, + slots, + protection: None, }; let want = ring_pass_bytes_in(cap, n_used, n, ff, half_slots, forms); if ring.bytes() != want { @@ -87,6 +93,21 @@ impl RingGemm { Ok(ring) } + pub(crate) fn set_expert_input( + &mut self, + gpu: &Gpu, + policy: ExpertInput, + ) -> Result<(), GpuError> { + match policy { + ExpertInput::Direct => self.protection = None, + ExpertInput::AmaxF16 if self.protection.is_none() => { + self.protection = Some(Protection::new(gpu, self.slots)?); + } + ExpertInput::AmaxF16 => {} + } + Ok(()) + } + /// Device bytes of the buffers. pub(crate) fn bytes(&self) -> usize { self.x.bytes() @@ -94,6 +115,7 @@ impl RingGemm { + self.u.num_bytes() + self.h.bytes() + self.route.bytes() + + self.protection.as_ref().map_or(0, Protection::bytes) } /// The slots of the half the buffers were made for. @@ -148,7 +170,13 @@ impl RingGemm { ) -> Result<(), GpuError> { let stream = gpu.stream(); let RingGemm { - x, g, u, h, route, .. + x, + g, + u, + h, + route, + protection, + .. } = self; batch.with_map(gpu.context(), n_expert, |row| { g32.enqueue_route_remap(stream, ids, row, n_slots, &mut *route, fault) @@ -164,7 +192,7 @@ impl RingGemm { Rows::Q128(a) => (Some(&mut **a), None), Rows::Blocks32(a) => (None, Some(&mut **a)), }; - blk.enqueue_rule( + blk.enqueue_input( gpu, (gemm, g32), (&*route, top_k), @@ -181,6 +209,7 @@ impl RingGemm { }, rule, ), + protection.as_mut(), fault, ) } diff --git a/crates/gpu/src/route_body.rs b/crates/gpu/src/route_body.rs index 6058f9154..253d668d7 100644 --- a/crates/gpu/src/route_body.rs +++ b/crates/gpu/src/route_body.rs @@ -68,9 +68,11 @@ pub unsafe fn publish_n( while e < E { // SAFETY: at + e < at + E, inside probs by this fn's contract. let p = unsafe { core::ptr::read_volatile(probs.add(at + e)) }; - // SAFETY: any supplied bias holds E entries by this fn's contract. let v = match bias { - Some(bias) => p + unsafe { *bias.get_unchecked(e) }, + Some(bias) => { + // SAFETY: e < E and any supplied bias holds E entries. + p + unsafe { *bias.get_unchecked(e) } + } None => p, }; if !v.is_finite() { @@ -107,7 +109,12 @@ pub unsafe fn publish_n( reason = "the selecting block's shared arrays, its lane and scale, the fault and the outputs" )] #[inline(always)] -pub unsafe fn select_n( +pub unsafe fn select_n< + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( score: *const f32, sel_v: *const f32, slot_p: *mut f32, @@ -238,7 +245,13 @@ pub unsafe fn select_n( +pub unsafe fn router_body< + S: Score, + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( w: &[f32], x: &[f32], bias: Option<&[f32]>, @@ -304,7 +317,9 @@ pub unsafe fn router_body(score, sel_v, slot_p, lane, scale, fault, ids, weights, 0) + select_n::( + score, sel_v, slot_p, lane, scale, fault, ids, weights, 0, + ) }; if lane == 0 { // SAFETY: `ticket` is done[0], inside `done` by this fn's @@ -331,7 +346,12 @@ pub unsafe fn router_body( +pub unsafe fn pick_body< + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( probs: &[f32], bias: Option<&[f32]>, n_expert: u32, @@ -982,11 +1002,23 @@ pub mod sigmoid_floor_256_6 { /// Both metadata bindings name the same optional-bias device entries. pub const ROW: RouterInst = shape::router_row_rule(RouterBody::SigmoidFloor, rules::SIGMOID_FLOOR, 256, 6); - pub const BIASED_ROW: RouterInst = - shape::router_row_rule(RouterBody::SigmoidFloor, rules::BIASED_SIGMOID_FLOOR, 256, 6); - const _: () = assert!(shape::router_row_is(ROW, rules::SIGMOID_FLOOR, false, (6, 6))); + pub const BIASED_ROW: RouterInst = shape::router_row_rule( + RouterBody::SigmoidFloor, + rules::BIASED_SIGMOID_FLOOR, + 256, + 6, + ); const _: () = assert!(shape::router_row_is( - BIASED_ROW, rules::BIASED_SIGMOID_FLOOR, false, (6, 6) + ROW, + rules::SIGMOID_FLOOR, + false, + (6, 6) + )); + const _: () = assert!(shape::router_row_is( + BIASED_ROW, + rules::BIASED_SIGMOID_FLOOR, + false, + (6, 6) )); const _: () = assert!(N_EXPERT.is_multiple_of(ROWS_PER_BLOCK)); const _: () = assert!(N_EXPERT.div_ceil(32) <= 32); @@ -1045,9 +1077,18 @@ pub mod sigmoid_floor_256_6 { // Shared arrays belong to this block and have the body's sizes. unsafe { router_body::( - w, x, if has_bias != 0 { Some(bias) } else { None }, - N_EXPERT as u32, k, scale, - &mut logits, &mut probs, &mut ids, &mut weights, &mut done, fault, + w, + x, + if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, + k, + scale, + &mut logits, + &mut probs, + &mut ids, + &mut weights, + &mut done, + fault, SharedArray::as_raw_mut_ptr(&raw mut LAST), SharedArray::as_raw_mut_ptr(&raw mut SCORE), SharedArray::as_raw_mut_ptr(&raw mut SEL_V), @@ -1091,8 +1132,14 @@ pub mod sigmoid_floor_256_6 { // block; shared arrays belong to this block at the body's sizes. unsafe { pick_body::( - probs, if has_bias != 0 { Some(bias) } else { None }, - N_EXPERT as u32, t, scale, &mut ids, &mut weights, fault, + probs, + if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, + t, + scale, + &mut ids, + &mut weights, + fault, SharedArray::as_raw_mut_ptr(&raw mut SCORE), SharedArray::as_raw_mut_ptr(&raw mut SEL_V), SharedArray::as_raw_mut_ptr(&raw mut SLOT_P), @@ -1140,19 +1187,26 @@ pub mod sigmoid_floor_256_6 { if w.rows() != N_EXPERT || k == 0 || !k.is_multiple_of(32) { return Err(GpuError::shape( what, - format!("router weight is {} x {k}, want 256 rows of a positive multiple of 32", w.rows()), + format!( + "router weight is {} x {k}, want 256 rows of a positive multiple of 32", + w.rows() + ), )); } Ok(k) } fn check_bias(what: &'static str, bias: Option<&DeviceBuffer>) -> Result<(), GpuError> { - if let Some(bias) = bias { - if bias.len() < N_EXPERT { - return Err(GpuError::shape( - what, format!("selection bias has {} entries, want at least 256", bias.len()), - )); - } + if let Some(bias) = bias + && bias.len() < N_EXPERT + { + return Err(GpuError::shape( + what, + format!( + "selection bias has {} entries, want at least 256", + bias.len() + ), + )); } Ok(()) } @@ -1188,18 +1242,43 @@ pub mod sigmoid_floor_256_6 { let k = weight_k(what, w)?; check_bias(what, bias)?; if x.len() < k { - return Err(GpuError::shape(what, format!("x.len() {} < k {k}", x.len()))); + return Err(GpuError::shape( + what, + format!("x.len() {} < k {k}", x.len()), + )); } let k = launch_u32(what, "k", k)?; - let prep = self.module.prepare_sigmoid_floor_256_6_router( - LaunchConfig1D::new((N_EXPERT / ROWS_PER_BLOCK) as u32, ROUTER_THREADS_U32, 0), - )?; - let RouterOut { logits, probs, ids, weights, done } = out; + let prep = self + .module + .prepare_sigmoid_floor_256_6_router(LaunchConfig1D::new( + (N_EXPERT / ROWS_PER_BLOCK) as u32, + ROUTER_THREADS_U32, + 0, + ))?; + let RouterOut { + logits, + probs, + ids, + weights, + done, + } = out; // With no bias the weight slice is only an ABI borrow: the // kernel takes None and never loads it as a selection bias. self.module.sigmoid_floor_256_6_router( - stream, &prep, w.buf(), x, bias.unwrap_or(w.buf()), u32::from(bias.is_some()), - k, scale, logits, probs, ids, weights, done, fault, + stream, + &prep, + w.buf(), + x, + bias.unwrap_or(w.buf()), + u32::from(bias.is_some()), + k, + scale, + logits, + probs, + ids, + weights, + done, + fault, )?; Ok(()) } @@ -1231,25 +1310,53 @@ pub mod sigmoid_floor_256_6 { let k = weight_k(what, w)?; check_bias(what, bias)?; let tt = launch_u32(what, "t", t)?; - let input = k.checked_mul(t).ok_or_else(|| GpuError::shape(what, "input size overflow"))?; - let scores = N_EXPERT.checked_mul(t).ok_or_else(|| GpuError::shape(what, "score size overflow"))?; - let slots = N_USED.checked_mul(t).ok_or_else(|| GpuError::shape(what, "slot size overflow"))?; - if t == 0 || x.len() < input || probs.len() < scores || ids.len() < slots || weights.len() < slots { + let input = k + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "input size overflow"))?; + let scores = N_EXPERT + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "score size overflow"))?; + let slots = N_USED + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "slot size overflow"))?; + if t == 0 + || x.len() < input + || probs.len() < scores + || ids.len() < slots + || weights.len() < slots + { return Err(GpuError::shape( what, - format!("{t} tokens of {k} (at least one): x {}, probs {}, ids {}, weights {}", - x.len(), probs.len(), ids.len(), weights.len()), + format!( + "{t} tokens of {k} (at least one): x {}, probs {}, ids {}, weights {}", + x.len(), + probs.len(), + ids.len(), + weights.len() + ), )); } // The tile GEMM uses the one-token row walk and warp sum tree. wide.enqueue_f32_tile(stream, w, x, t, probs)?; gpu.elem().enqueue_sigmoid_rows(stream, probs, scores)?; - let prep = self.module.prepare_sigmoid_floor_256_6_router_pick( - LaunchConfig1D::new(tt, ROUTER_THREADS_U32, 0), - )?; + let prep = self + .module + .prepare_sigmoid_floor_256_6_router_pick(LaunchConfig1D::new( + tt, + ROUTER_THREADS_U32, + 0, + ))?; self.module.sigmoid_floor_256_6_router_pick( - stream, &prep, probs, bias.unwrap_or(w.buf()), u32::from(bias.is_some()), - tt, scale, ids, weights, fault, + stream, + &prep, + probs, + bias.unwrap_or(w.buf()), + u32::from(bias.is_some()), + tt, + scale, + ids, + weights, + fault, )?; Ok(()) } @@ -1259,6 +1366,8 @@ pub mod sigmoid_floor_256_6 { const N_EXPERT: usize = N_EXPERT; const N_USED: usize = N_USED; const OPTIONAL_BIAS: bool = true; + const NORM: model::arch::models::RouterNorm = + model::arch::models::RouterNorm::SelectedSumFloor; type Out = RouterOut; fn load(ctx: &Arc) -> Result { @@ -1274,8 +1383,11 @@ pub mod sigmoid_floor_256_6 { &out.weights } fn out_bytes(out: &RouterOut) -> usize { - out.logits.num_bytes() + out.probs.num_bytes() + out.ids.num_bytes() - + out.weights.num_bytes() + out.done.num_bytes() + out.logits.num_bytes() + + out.probs.num_bytes() + + out.ids.num_bytes() + + out.weights.num_bytes() + + out.done.num_bytes() } fn route( &self, @@ -1295,8 +1407,18 @@ pub mod sigmoid_floor_256_6 { rows: RoutedRows<'_>, ) -> Result<(), GpuError> { self.enqueue_router_rows( - stream, gpu, wide, r.w, x, r.bias, r.scale, t, - rows.probs, rows.ids, rows.weights, r.fault, + stream, + gpu, + wide, + r.w, + x, + r.bias, + r.scale, + t, + rows.probs, + rows.ids, + rows.weights, + r.fault, ) } } diff --git a/crates/gpu/src/routed_block.rs b/crates/gpu/src/routed_block.rs index 7ad972410..146707c34 100644 --- a/crates/gpu/src/routed_block.rs +++ b/crates/gpu/src/routed_block.rs @@ -74,6 +74,8 @@ pub trait RouterEntry: Sized { const N_USED: usize; /// Whether an absent selection bias is a native binding, not a zero buffer. const OPTIONAL_BIAS: bool = false; + /// The selected-score normalization implemented by this compiled binding. + const NORM: model::arch::models::RouterNorm = model::arch::models::RouterNorm::SelectedSum; /// Where one token's launch leaves its results. type Out; diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index 696423f1d..446d76fa9 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -46,6 +46,66 @@ pub struct Available { /// Everything the programs run that a [`Need`] names. A need no row covers /// is an item of [`check`]; a row removed makes its needs items. pub const AVAILABLE: &[Available] = &[ + Available { + programs: &[Program::FlintBody], + at: "gpu/src/attention_h128.rs Binding::enqueue and attention_h128_ring.rs", + runs: |n| { + matches!( + n, + Need::Gqa { + head: 128, + value: 128, + group: 8, + window: None | Some(513), + sinks: true, + scaled: false, + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu/src/attention_h128_ring.rs (head128 RMS with optional full NeoX)", + runs: |n| { + matches!( + n, + Need::QkRope { + qk_norm: true, + head: 128, + mode: RopeMode::None, + dims: 0, + } | Need::QkRope { + qk_norm: true, + head: 128, + mode: RopeMode::Neox, + dims: 128, + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu/src/route_body.rs sigmoid_floor_256_6", + runs: |n| routes(n, &[RouterBody::SigmoidFloor]), + }, + Available { + programs: &[Program::FlintBody], + at: "gpu-deepseek41/src/shared_routed.rs (ungated shared expert)", + runs: |n| { + matches!( + n, + Need::Shared { + sigmoid_gate: false, + .. + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu-deepseek41/src/shared_routed.rs dense-leading gated blocks", + runs: |n| matches!(n, Need::DenseFfn { ff } if ff.is_multiple_of(256)), + }, Available { programs: &[Program::Deepseek41Chain], at: "gpu-deepseek41/src/attn.rs LATENT; chain/attn.rs rope dims", @@ -1500,6 +1560,46 @@ struct RowPin { /// and per-layer input projection (the host's rows), each `what` naming the /// types its row takes or the table row it follows. const ROW_PINS: &[RowPin] = &[ + RowPin { + program: Program::FlintBody, + role: Role::Attention, + matrices: true, + names: None, + what: "head128 attention projections (common dense sites)", + row: h128_projection, + }, + RowPin { + program: Program::FlintBody, + role: Role::DenseFfn, + matrices: true, + names: None, + what: "dense-leading gated blocks", + row: gated_block_site, + }, + RowPin { + program: Program::FlintBody, + role: Role::SharedExpert, + matrices: true, + names: None, + what: "shared gated blocks", + row: gated_block_site, + }, + RowPin { + program: Program::FlintBody, + role: Role::TokenEmbedding, + matrices: true, + names: None, + what: "common token embedding", + row: |ty| embed_of_widened(ty).is_some(), + }, + RowPin { + program: Program::FlintBody, + role: Role::Head, + matrices: true, + names: None, + what: "common output head", + row: |ty| head_of_widened(ty).is_some(), + }, // PIN(2026-10-22): Qwen38 dense pins admit MXFP8/FP8_B128 and Q8_0 indexers via BY_NAME; the old MXFP8 refusal fails the new coverage test (PR FAIL-first). RowPin { program: Program::Qwen38Body, @@ -1672,6 +1772,35 @@ const ROW_PINS: &[RowPin] = &[ }, ]; +fn h128_projection(ty: GgmlType) -> bool { + matches!( + ty, + GgmlType::Q3_K + | GgmlType::Q4_K + | GgmlType::Q5_K + | GgmlType::Q6_K + | GgmlType::IQ4_XS + | GgmlType::Q5_0 + | GgmlType::Q8_0 + | GgmlType::F32 + | GgmlType::MXFP8 + | GgmlType::FP8_B128 + | GgmlType::BF16 + ) +} + +fn gated_block_site(ty: GgmlType) -> bool { + matches!( + ty, + GgmlType::Q8_0 + | GgmlType::Q4_K + | GgmlType::Q5_K + | GgmlType::Q6_K + | GgmlType::IQ4_XS + | GgmlType::MXFP8 + ) +} + /// `Body35`'s projection types (`PROJ`): every attention, delta-rule and /// dense FFN matrix. const Q35_PROJ: &[GgmlType] = &[ @@ -1878,7 +2007,7 @@ pub fn check_with( // stack's row width ([`routed_unrun`], the one owner). A layer // with a stack the card experts do not read keeps all its // experts on the host. - Some(p @ (Program::Glm5nextBody | Program::MiniMaxM3Body)) => { + Some(p @ (Program::Glm5nextBody | Program::MiniMaxM3Body | Program::FlintBody)) => { if routed_unrun(t, |ty| leg_card_routed(p, ty)) { at(t.layer, Need::RoutedFormat(t.ty)); } diff --git a/crates/model/src/arch/flint/host.rs b/crates/model/src/arch/flint/host.rs new file mode 100644 index 000000000..714fd4eb2 --- /dev/null +++ b/crates/model/src/arch/flint/host.rs @@ -0,0 +1,83 @@ +//! Flint's validated routed block as the common host and NVMe expert tiers serve it. + +use models::{Act, Ffn, ModelSpec}; + +use crate::ModelError; +use crate::arch::names; +use crate::moe::{HostLayer, HostLayerSpec}; +use crate::ops::ExpertInput; +use crate::r8file::R8Source; + +/// Build the requested layer's routed stacks with Flint's unclamped SwiGLU and activated-value +/// scaling. Dense layers, layers outside the description and non-SwiGLU routed blocks are refused +/// before a tensor is opened. Load-time only. +pub fn layer(src: R8Source<'_>, spec: &ModelSpec, l: usize) -> Result { + let refuse = |detail: String| ModelError::Metadata { + key: format!("layer {l} routed block"), + detail, + }; + let block = spec + .layers + .get(l) + .ok_or_else(|| refuse(format!("outside the file's {} layers", spec.layers.len())))?; + let Ffn::Moe(moe) = &block.ffn else { + return Err(refuse("is dense, not a routed block".to_string())); + }; + if moe.act != (Act::SwiGlu { limit: None }) { + return Err(refuse(format!( + "activation {:?}; Flint's host experts require unclamped SwiGLU", + moe.act + ))); + } + let [gate, up, down] = names::routed_stacks(l); + let mut layer = HostLayer::build( + src, + &HostLayerSpec { + gate: &gate, + up: &up, + down: &down, + n_expert: moe.experts as usize, + embd: spec.hidden as usize, + ff: moe.expert_ff as usize, + swiglu_limit: 0.0, + }, + )?; + layer.set_expert_input(ExpertInput::AmaxF16); + Ok(layer) +} + +#[cfg(test)] +mod tests { + use super::super::tests::{keys, tensors}; + use super::*; + use crate::arch::synthetic::{V, header_shaped}; + + #[test] + fn refuses_dense_outside_and_non_swiglu_before_opening_experts() { + let path = header_shaped( + "flint-host-layer", + "flint", + &keys(), + &[("tokenizer.ggml.pre", V::Str("flint"))], + &tensors(), + ); + let split = gguf::Split::open(&path).unwrap(); + let mut spec = super::super::spec::read(&split).unwrap().spec; + let src = R8Source::rows(&split); + for (l, fragment) in [(0, "is dense"), (4, "outside the file's 4 layers")] { + let error = layer(src, &spec, l).err().expect("typed refusal"); + assert!(matches!(error, ModelError::Metadata { ref key, ref detail } + if key == &format!("layer {l} routed block") && detail.contains(fragment))); + } + let Ffn::Moe(moe) = &mut spec.layers[1].ffn else { + panic!("routed"); + }; + moe.act = Act::ReluSqr; + let error = layer(src, &spec, 1) + .err() + .expect("non-gated activation refused"); + assert!(matches!(error, ModelError::Metadata { ref detail, .. } + if detail.contains("require unclamped SwiGLU"))); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/model/src/arch/flint/mod.rs b/crates/model/src/arch/flint/mod.rs index 5297cb5f4..afee36297 100644 --- a/crates/model/src/arch/flint/mod.rs +++ b/crates/model/src/arch/flint/mod.rs @@ -1,12 +1,11 @@ -//! Flint's header description and exact tensor roles. Full and sliding attention -//! independently select NoPE or NeoX rotation; every layer has head-local q/k -//! norms and attention sinks. Routed blocks carry one ungated shared MLP whose -//! width is explicit, not a count of physical modules. -//! -//! This is a reader, not an executable program. Coverage retains `Need::Program` -//! and the missing numerical/site bindings; no fixture or serve seat is registered. +//! Flint's validated description, placement inputs and routed host adapter. Full and sliding +//! attention independently select NoPE or NeoX rotation; every layer has head-local q/k norms and +//! attention sinks. Routed blocks carry one ungated shared MLP whose width is explicit, not a +//! count of physical modules. Executable admission belongs to the common program/site registry. +pub mod host; mod hparams; +pub mod place; mod roles; pub mod spec; diff --git a/crates/model/src/arch/flint/place.rs b/crates/model/src/arch/flint/place.rs new file mode 100644 index 000000000..53a5006a7 --- /dev/null +++ b/crates/model/src/arch/flint/place.rs @@ -0,0 +1,478 @@ +//! Flint's validated description attached to the common hybrid placement and buffer owners. + +use gguf::Split; +use models::{Ffn, Mixer, ModelSpec}; + +use super::spec; +use crate::arch::{names, place_hybrid}; +use crate::placement::accepts::{Program, leg_down_forms}; +use crate::placement::kernels::attention::{Forms, Head128Scratch}; +use crate::placement::kernels::ffn::{PromptWalk, RoutedBatch, WideBlock, WideForms}; +use crate::placement::workstation::{TierBatchBytes, TierLegScratch}; +use crate::placement::{ModelTensors, PlacementError, kernels, workstation}; + +pub use place_hybrid::{KvLayout, PlaceError}; + +/// No second hyperparameter reader: the validated description owns Flint's facts. +pub type PlanInputs = place_hybrid::PlanInputs<()>; + +/// The common host union's prompt capacity, also the window ring's batch allowance. +pub const PROMPT_ROWS: usize = crate::moe::UNION_MAX_COLS; +/// Tokens served by one run of the common expert tier leg. +pub const TIER_CHUNK: u64 = 8; + +fn overflow() -> PlacementError { + PlacementError::Metadata { + key: "flint prompt reserve".into(), + detail: "buffer bytes overflow u64".into(), + } +} + +fn reads(model: &ModelTensors, name: &str) -> Result { + let tensor = model + .tensors + .iter() + .find(|t| t.name == name) + .ok_or_else(|| PlacementError::Tensor { + name: name.into(), + detail: "is not in the validated description".into(), + })?; + kernels::dense(tensor.ty) + .map(|site| site.reads()) + .ok_or_else(|| PlacementError::Tensor { + name: name.into(), + detail: format!("type {:?} has no dense projection site", tensor.ty), + }) +} + +/// The actual Q/K/V and output forms, widest rows and maximum partial segments of the shared +/// head128 attention scratch. NoPE changes no buffer width; windowing changes stores, not the +/// maximum flash launch scratch a full layer needs. +pub fn attention_scratch( + spec: &ModelSpec, + model: &ModelTensors, +) -> Result { + let mut scratch = Head128Scratch { + hidden: u64::from(spec.hidden), + query: 0, + key_value: 0, + heads: 0, + segments: 0, + input: Forms::default(), + output: Forms::default(), + }; + for (l, layer) in spec.layers.iter().enumerate() { + let Mixer::Gqa(g) = &layer.mixer else { + return Err(PlacementError::Metadata { + key: format!("layer {l} attention"), + detail: "Flint requires GQA".into(), + }); + }; + scratch.query = scratch + .query + .max(u64::from(g.heads) * u64::from(g.head_dim)); + scratch.key_value = scratch + .key_value + .max(u64::from(g.kv_heads) * u64::from(g.head_dim)); + scratch.heads = scratch.heads.max(u64::from(g.heads)); + scratch.segments = scratch + .segments + .max(g.window.map_or(80, |w| u64::from(w).div_ceil(64))); + for name in [names::attn_q(l), names::attn_k(l), names::attn_v(l)] { + scratch.input = scratch.input.with(reads(model, &name)?); + } + scratch.output = scratch.output.with(reads(model, &names::attn_output(l))?); + } + Ok(scratch) +} + +fn block_forms( + model: &ModelTensors, + names: [String; 3], + block: &mut WideBlock, +) -> Result<(), PlacementError> { + block.input = block + .input + .with(reads(model, &names[0])?) + .with(reads(model, &names[1])?); + block.down = block.down.with(reads(model, &names[2])?); + Ok(()) +} + +/// The common routed/shared batch with selected dense/shared forms. `card` prices the allocation +/// made when either resident card serves experts; a plan reserves that candidate before assigning +/// expert ids. `tiered` adds only the prompt join's three slot words. +pub fn routed_batch( + spec: &ModelSpec, + model: &ModelTensors, + rows: u64, + card: bool, + tiered: bool, +) -> Result { + let empty = WideBlock { + hidden: u64::from(spec.hidden), + ff: 0, + input: WideForms::default(), + down: WideForms::default(), + }; + let mut shared = empty; + let mut dense = empty; + let mut experts = 0; + let mut used = 0; + let mut expert_ff = 0; + for (l, layer) in spec.layers.iter().enumerate() { + match &layer.ffn { + Ffn::Dense { ff, .. } => { + dense.ff = dense.ff.max(u64::from(*ff)); + block_forms( + model, + [names::ffn_gate(l), names::ffn_up(l), names::ffn_down(l)], + &mut dense, + )?; + } + Ffn::Moe(moe) => { + experts = experts.max(u64::from(moe.experts)); + used = used.max(u64::from(moe.top_k)); + expert_ff = expert_ff.max(u64::from(moe.expert_ff)); + let sh = moe + .shared + .as_ref() + .ok_or_else(|| PlacementError::Metadata { + key: format!("layer {l} shared block"), + detail: "Flint requires one ungated shared MLP".into(), + })?; + shared.ff = shared.ff.max(u64::from(sh.ff)); + block_forms( + model, + [ + names::ffn_gate_shexp(l), + names::ffn_up_shexp(l), + names::ffn_down_shexp(l), + ], + &mut shared, + )?; + } + } + } + Ok(RoutedBatch { + hidden: u64::from(spec.hidden), + experts, + used, + shared, + dense: (dense.ff > 0).then_some(dense), + card: card.then_some(TierLegScratch { + n_embd: u64::from(spec.hidden), + ff: expert_ff, + n_used: used, + chunk: rows, + downs: leg_down_forms(Program::FlintBody, model), + input_scales: true, + }), + tiered, + }) +} + +/// Prompt reserve from the same descriptors the attention, FFN and walk allocate by. It includes +/// the candidate card batch, not a quantization padding estimate or a per-layer copy of scratch. +pub fn prompt_reserve_bytes( + spec: &ModelSpec, + model: &ModelTensors, + rows: u64, +) -> Result { + let attention = attention_scratch(spec, model)? + .bytes(rows) + .ok_or_else(overflow)?; + let walk = PromptWalk { + hidden: u64::from(spec.hidden), + sets: 1, + } + .bytes(rows) + .ok_or_else(overflow)?; + let ffn = routed_batch(spec, model, rows, true, false)? + .bytes(rows) + .ok_or_else(overflow)?; + attention + .checked_add(walk) + .and_then(|bytes| bytes.checked_add(ffn)) + .ok_or_else(overflow) +} + +/// Price the selected prompt group before borrowing the inputs for a plan. Reconfiguration is +/// idempotent: only unit rows multiply; attention, FFN and the host-sum upload stay shared. +pub fn configure_group(inputs: &mut PlanInputs, group: usize) -> Result<(), PlacementError> { + runtime::prompt::check_group(group, bloomery_levers::PREFILL_GROUP_MAX as usize).map_err( + |error| PlacementError::Metadata { + key: runtime::prompt::GROUP_LEVER.into(), + detail: error.to_string(), + }, + )?; + let rows = PROMPT_ROWS as u64; + let hidden = u64::from(inputs.spec.hidden); + let one = PromptWalk { hidden, sets: 1 } + .bytes(rows) + .ok_or_else(overflow)?; + let selected = PromptWalk { + hidden, + sets: runtime::prompt::group_sets(group) as u64, + } + .bytes(rows) + .ok_or_else(overflow)?; + let base = prompt_reserve_bytes(&inputs.spec, &inputs.model, rows)?; + inputs.prompt_reserve = base + .checked_sub(one) + .and_then(|bytes| bytes.checked_add(selected)) + .ok_or_else(overflow)?; + Ok(()) +} + +/// The real expert tier's staging, protected leg scratch and host exchanges for one prompt unit. +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let batch = routed_batch(&inputs.spec, &inputs.model, TIER_CHUNK, true, false)?; + let leg = batch.card.ok_or_else(overflow)?; + workstation::tier_leg_batch_bytes( + leg.n_embd, + leg.ff, + leg.n_used, + TIER_CHUNK, + PROMPT_ROWS as u64, + leg.downs, + true, + ) +} + +impl PlanInputs { + /// Describe once, then require the genuine program and its numerical/site bindings. + pub fn read(split: &Split) -> Result { + Self::describe(split)?.runnable() + } + + /// The validated file, common windowed stores, actual prompt reserve and tier loader contract. + /// The common host-run loader serves NVMe through the same residency arena that placement split. + pub fn describe(split: &Split) -> Result { + let read = spec::read(split)?; + let reserve = prompt_reserve_bytes(&read.spec, &read.tensors, PROMPT_ROWS as u64)?; + let batch = routed_batch(&read.spec, &read.tensors, PROMPT_ROWS as u64, false, true)?; + let tier = + kernels::ffn::tier_places_bytes(PROMPT_ROWS as u64, batch.used).ok_or_else(overflow)?; + Ok(Self::new("Flint", (), read.tensors, read.spec)? + .with_prompt_reserve(reserve) + .with_tier_reserve(tier) + .with_nvme_tier()) + } +} + +#[cfg(test)] +mod tests { + use super::super::tests::{check, keys, tensors}; + use super::*; + use crate::arch::synthetic::{V, header_shaped}; + + #[test] + fn description_uses_real_prompt_and_tier_terms_without_admitting_unsupported_shapes() { + let path = header_shaped( + "flint-place-description", + "flint", + &keys(), + &[("tokenizer.ggml.pre", V::Str("flint"))], + &tensors(), + ); + let split = Split::open(&path).unwrap(); + let mut inputs = PlanInputs::describe(&split).unwrap(); + assert_eq!( + inputs.prompt_reserve, + attention_scratch(&inputs.spec, &inputs.model) + .unwrap() + .bytes(PROMPT_ROWS as u64) + .unwrap() + + PromptWalk { + hidden: 16, + sets: 1 + } + .bytes(PROMPT_ROWS as u64) + .unwrap() + + routed_batch(&inputs.spec, &inputs.model, PROMPT_ROWS as u64, true, false) + .unwrap() + .bytes(PROMPT_ROWS as u64) + .unwrap() + ); + assert_eq!(inputs.tier_reserve, Some(12 * PROMPT_ROWS as u64 * 2)); + let tier = tier_batch(&inputs).unwrap(); + let leg = tier.leg.unwrap(); + assert!(leg.input_scales); + assert_eq!(leg.chunk, TIER_CHUNK); + assert_eq!(tier.scratch, leg.bytes(leg.downs).unwrap()); + assert!(matches!( + PlanInputs::read(&split), + Err(PlacementError::Unimplemented(_)) + )); + let one = inputs.prompt_reserve; + configure_group(&mut inputs, 2).unwrap(); + let grouped = inputs.prompt_reserve; + assert_eq!(grouped - one, 2 * PROMPT_ROWS as u64 * (8 * 16 + 8)); + configure_group(&mut inputs, 2).unwrap(); + assert_eq!(inputs.prompt_reserve, grouped); + configure_group(&mut inputs, 1).unwrap(); + assert_eq!(inputs.prompt_reserve, one); + assert!(configure_group(&mut inputs, 0).is_err()); + assert_eq!(inputs.prompt_reserve, one); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn released_window_geometry_counts_each_resident_sequence() { + let read = check("place-window-slots", &keys(), &tensors()).unwrap(); + let mut full = read.spec.layers[0].mixer.clone(); + let Mixer::Gqa(g) = &mut full else { + panic!("GQA"); + }; + g.heads = 32; + g.kv_heads = 4; + g.head_dim = 128; + g.value_dim = 128; + g.window = None; + let mut window = full.clone(); + let Mixer::Gqa(g) = &mut window else { + panic!("GQA"); + }; + g.window = Some(513); + let mixers: Vec<_> = (0..49) + .map(|l| if l < 14 { full.clone() } else { window.clone() }) + .collect(); + let kv = KvLayout::of(&mixers).unwrap(); + for ctx in [1, 512, 513, 1023, 1024, 1025, 4096] { + let expected = 2048 * (14 * ctx + 35 * ctx.min(1024)); + assert_eq!(kv.bytes(0..49, ctx), expected); + assert_eq!(kv.rows_at(14, ctx), ctx.min(1024)); + assert_eq!(kv.rows_at(0, ctx), ctx); + for slots in [1, 3] { + assert_eq!(kv.seq_terms().bytes(ctx, slots), slots * expected); + } + } + } +} + +#[cfg(test)] +mod planning_tests { + use super::super::tests::{check, keys, tensors}; + use super::*; + use crate::arch::synthetic::V; + use crate::placement::{PlanLevers, Role}; + + #[test] + fn selected_projection_forms_are_a_union_not_blanket_padding() { + let mut read = check("place-selected-forms", &keys(), &tensors()).unwrap(); + let plain = attention_scratch(&read.spec, &read.tensors).unwrap(); + assert_eq!(plain.input, Forms::default()); + assert_eq!(plain.output, Forms::default()); + for (name, ty) in [ + (names::attn_q(0), gguf::GgmlType::Q4_K), + (names::attn_k(1), gguf::GgmlType::Q8_0), + (names::attn_output(2), gguf::GgmlType::Q6_K), + ] { + read.tensors + .tensors + .iter_mut() + .find(|t| t.name == name) + .unwrap() + .ty = ty; + } + let mixed = attention_scratch(&read.spec, &read.tensors).unwrap(); + assert_eq!( + mixed.input, + Forms { + q128: true, + q32: true + } + ); + assert_eq!( + mixed.output, + Forms { + q128: true, + q32: false + } + ); + let rows = PROMPT_ROWS as u64; + assert_eq!( + mixed.bytes(rows).unwrap() - plain.bytes(rows).unwrap(), + mixed.input.bytes(rows, mixed.hidden).unwrap() + + mixed.output.bytes(rows, mixed.query).unwrap() + ); + } + + #[test] + fn nvme_loader_opt_in_preserves_unserved_loader_refusal_and_slot_kv() { + let mut keys = keys(); + for (key, value) in &mut keys { + *value = match *key { + "block_count" => V::U32(8), + "expert_count" => V::U32(256), + "expert_used_count" => V::U32(6), + "attention.sliding_window_pattern" => { + V::Bools((0..8).map(|l| l % 2 == 1).collect()) + } + "attention.rope_pattern" => V::Bools((0..8).map(|l| l % 2 == 0).collect()), + _ => continue, + }; + } + let mut ts = tensors(); + let template: Vec<_> = ts + .iter() + .filter(|(n, _)| n.starts_with("blk.1.")) + .cloned() + .collect(); + for l in 4..8 { + ts.extend( + template + .iter() + .map(|(n, dims)| (n.replacen("blk.1.", &format!("blk.{l}."), 1), dims.clone())), + ); + } + for (name, dims) in &mut ts { + if name.contains("_exps.weight") { + dims[2] = 256; + } else if name.ends_with("ffn_gate_inp.weight") { + dims[1] = 256; + } + } + let read = check("place-paging", &keys, &ts).unwrap(); + let mut inputs = PlanInputs::new("Flint", (), read.tensors, read.spec).unwrap(); + for tensor in &mut inputs.model.tensors { + if tensor.role == Role::RoutedExperts { + tensor.dims[1] *= 4096; + tensor.file_bytes *= 4096; + } + } + inputs.prompt_reserve = + prompt_reserve_bytes(&inputs.spec, &inputs.model, PROMPT_ROWS as u64).unwrap(); + inputs.room = (u64::MAX, workstation::HostRead::Given); + let machine = workstation::plan_gate(8); + let levers = PlanLevers::default(); + let (need, held) = { + let ample = inputs.plan(&machine, 2048, &levers).unwrap(); + ( + workstation::HostNeed::of(&le, 0).bytes(), + ample.host.expert_bytes, + ) + }; + inputs.room.0 = need - held / 8; + assert!(matches!(inputs.plan(&machine, 2048, &levers), + Err(PlaceError::NoNvmeTier { paged, .. }) if paged > 0)); + let inputs = inputs.with_nvme_tier(); + for slots in [1, 3] { + let plan = inputs + .plan_with_slots( + &machine, + 2048, + &levers, + std::num::NonZeroUsize::new(slots).unwrap(), + ) + .unwrap(); + assert!(plan.host.nvme_expert_bytes > 0); + assert_eq!(plan.host.expert_bytes + plan.host.nvme_expert_bytes, held); + assert_eq!( + plan.cards.iter().map(|c| c.kv_bytes).sum::(), + slots as u64 * inputs.kv.bytes(0..8, 2048) + ); + } + } +} diff --git a/crates/model/src/arch/flint/tests.rs b/crates/model/src/arch/flint/tests.rs index 698806d85..0ec55ee4c 100644 --- a/crates/model/src/arch/flint/tests.rs +++ b/crates/model/src/arch/flint/tests.rs @@ -3,7 +3,7 @@ use crate::arch::synthetic::{V, header_shaped}; use crate::placement::PlacementError; use models::{Ffn, Mixer, Need, RopeMode}; -fn keys() -> Vec<(&'static str, V)> { +pub(super) fn keys() -> Vec<(&'static str, V)> { vec![ ("block_count", V::U32(4)), ("embedding_length", V::U32(16)), @@ -36,7 +36,7 @@ fn keys() -> Vec<(&'static str, V)> { ] } -fn tensors() -> Vec<(String, Vec)> { +pub(super) fn tensors() -> Vec<(String, Vec)> { let mut out = vec![ ("token_embd.weight".into(), vec![16, 2]), ("output.weight".into(), vec![16, 2]), @@ -76,7 +76,7 @@ fn tensors() -> Vec<(String, Vec)> { out } -fn check( +pub(super) fn check( tag: &str, keys: &[(&str, V)], tensors: &[(String, Vec)], @@ -162,12 +162,6 @@ fn masks_are_independent_and_nope_still_requires_head_norm() { .count(), 4 ); - assert!(crate::arch::coverage::program_of(r.spec.arch).is_none()); - assert!( - crate::arch::coverage::check(&r.spec, &r.tensors) - .iter() - .any(|item| item.feature == Need::Program("flint").to_string()) - ); } #[test] diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index d497ca063..670d34f98 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -146,7 +146,6 @@ fn down_forms(downs: impl Iterator) -> DownForms { .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)) } - /// Tokens a chunk of the tier's prompt-batch service runs at most: the /// stage card's chunk of the card experts' launches, which the body binds /// to its own constant. @@ -166,6 +165,7 @@ pub fn tier_batch(inputs: &PlanInputs) -> Result TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, crate::placement::accepts::leg_down_forms(Program::Glm5nextBody, &inputs.model), + false, ) } @@ -421,7 +421,6 @@ pub enum PlaceError { /// A plan of no resident sequence: a load serves at least one. #[error("a plan of {slots} resident sequences: a load serves at least one")] Slots { slots: usize }, - } impl PlanRefusal for PlaceError { @@ -692,7 +691,6 @@ impl PlanInputs { } Ok(plan) } - } /// Card bytes the NextN walk's arena may hold beside the plan's tensors @@ -1886,7 +1884,7 @@ mod tests { use GgmlType::{IQ3_S, IQ4_XS, NVFP4, Q4_K, Q5_K, Q6_K, Q8_0}; let q8 = DownForms::of(DownAct::Q8Act); let b32 = DownForms::of(DownAct::Q8Blocks32); - let base = tier_batch(&inputs()); + let base = tier_batch(&inputs()).expect("the described tier geometry"); let hp = inputs().hp; let leg = |forms| { leg_bytes( @@ -1899,7 +1897,6 @@ mod tests { }; assert_ne!(leg(q8), leg(b32)); for (layers, forms) in [ - (vec![], q8), (vec![[Q4_K, Q4_K, Q5_K]], q8), (vec![[Q5_K, Q5_K, Q6_K]], q8), (vec![[IQ3_S, IQ3_S, IQ4_XS]], q8), @@ -1914,7 +1911,7 @@ mod tests { q8.with(DownAct::Q8Blocks32), ), ] { - let batch = tier_batch(&routed_types(&layers)); + let batch = tier_batch(&routed_types(&layers)).expect("the selected tier geometry"); assert_eq!(batch.scratch, leg(forms), "{layers:?}"); assert_eq!(batch.staging, base.staging, "{layers:?}"); assert_eq!(batch.host, base.host, "{layers:?}"); diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 1b994b4ae..4832225ce 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -121,7 +121,6 @@ pub const TIER_CHUNK: u64 = 16; /// plan's rule carries reads (the rule's down arm is the leg's `Q8Act` downs), and the host's rows /// and places ([`workstation::tier_leg_batch_bytes`]). The plan reserves them on the tier card and /// the host, and the load checks its allocations against them. -#[must_use] pub fn tier_batch(hp: &Hparams, cols: u64) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, @@ -130,6 +129,7 @@ pub fn tier_batch(hp: &Hparams, cols: u64) -> Result Result { let hp = &inputs.hp; workstation::tier_leg_batch_bytes( @@ -144,6 +143,7 @@ pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result usize { /// placed pass's rows ([`pass_rows_max`]): the experts run in the latent, so /// the tier's rows and its staging are the latent's width, through the card /// leg in chunks of [`TIER_CHUNK`]. -#[must_use] pub fn tier_batch(hp: &Hparams) -> Result { let ff = hp.expert_width(); workstation::tier_leg_batch_bytes( @@ -86,6 +85,7 @@ pub fn tier_batch(hp: &Hparams) -> Result { TIER_CHUNK, pass_rows_max(hp.n_used) as u64, crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act), + false, ) } diff --git a/crates/model/src/arch/names.rs b/crates/model/src/arch/names.rs index 3179eb093..bd531bdf8 100644 --- a/crates/model/src/arch/names.rs +++ b/crates/model/src/arch/names.rs @@ -37,6 +37,12 @@ pub fn attn_norm(l: usize) -> String { blk(l, "attn_norm.weight") } +/// `blk.{l}.attn_sinks.weight` — one learned sink logit per query head. +#[must_use] +pub fn attn_sinks(l: usize) -> String { + blk(l, "attn_sinks.weight") +} + /// `blk.{l}.attn_q.weight` — the query projection, every head. #[must_use] pub fn attn_q(l: usize) -> String { diff --git a/crates/model/src/arch/place_hybrid.rs b/crates/model/src/arch/place_hybrid.rs index 10ca6c063..44831ac72 100644 --- a/crates/model/src/arch/place_hybrid.rs +++ b/crates/model/src/arch/place_hybrid.rs @@ -9,9 +9,9 @@ //! forces: the keys it reads, the layers it refuses, the engine's name in the refusals that name it. //! //! The plan refuses by name a machine that hangs an expert tier card where the family's body serves -//! none ([`PlanInputs::tier_reserve`]), a context of no position and a host room that cannot hold -//! the host experts: the bodies this plans serve no NVMe expert tier, so a plan that pages one -//! ([`PlaceError::NoNvmeTier`]) would load experts nothing serves. +//! none ([`PlanInputs::tier_reserve`]), a context of no position and, unless the loader explicitly +//! serves the common NVMe tier ([`PlanInputs::with_nvme_tier`]), a host room that pages any expert +//! ([`PlaceError::NoNvmeTier`]). //! //! A body that makes buffers beside the plan's terms on its card (the prompt batch's rows) states //! their bytes in [`PlanInputs::prompt_reserve`]: the expert rule spreads within the card's budget @@ -72,7 +72,10 @@ pub const fn wide_bytes(cols: usize, k: usize) -> usize { /// (a [`GEMM_TILE_COLS`] column tile each, and the expert's), the two counts and the all-zero ids. #[must_use] pub const fn route_bytes(cols: usize) -> usize { - 4 * (cols + 2 * (cols / GEMM_TILE_COLS + 1) + 2 + cols) + match kernels::ffn::dense_route_bytes(cols as u64) { + Some(bytes) => bytes as usize, + None => panic!("route_bytes: route table bytes overflow u64"), + } } /// The forms a ring pass's route GEMMs read their rows in, by the stacks' file types: the gate's and @@ -196,7 +199,10 @@ pub fn stream_ring_bytes( /// the copy of the slot's route weight, three four-byte words. #[must_use] pub const fn tier_places_bytes(rows: usize, n_used: usize) -> u64 { - 3 * 4 * (rows * n_used) as u64 + match kernels::ffn::tier_places_bytes(rows as u64, n_used as u64) { + Some(bytes) => bytes, + None => panic!("tier_places_bytes: slot bytes overflow u64"), + } } /// What a plan of a hybrid file is made from, read from its headers. `H` is the family's own @@ -213,9 +219,9 @@ pub struct PlanInputs { pub kv: KvLayout, /// The host's available bytes when the inputs were read, with the reading that decided /// ([`workstation::host_available_read`]): the routed experts the room cannot hold follow it to - /// the NVMe tier ([`placement::expert_nvme_tier`]), which these bodies do not serve, so such a - /// plan is refused ([`PlaceError::NoNvmeTier`]). A gate that plans an arm by a room of its own - /// sets it, read [`HostRead::Given`]. + /// the NVMe tier ([`placement::expert_nvme_tier`]). A body without that loader contract refuses + /// a plan that pages experts ([`PlaceError::NoNvmeTier`]). A gate that plans an arm by a room of + /// its own sets it, read [`HostRead::Given`]. /// /// The effective room is [`Plan::plan_at`]'s: the reading's plan room, /// capped by the machine's host room and unified pool. @@ -233,6 +239,9 @@ pub struct PlanInputs { /// keeps `bytes` more than [`PlanInputs::prompt_reserve`] on a machine that names a tier card, /// the buffers only a tiered load makes. pub tier_reserve: Option, + /// Whether the loader serves the common host-run NVMe arena. Opt-in only: a plan cannot + /// promise paging merely because the placement can split the expert stacks. + nvme_tier: bool, /// The engine the refusals that name one name. engine: &'static str, } @@ -357,6 +366,7 @@ impl PlanInputs { arena, prompt_reserve: 0, tier_reserve: None, + nvme_tier: false, engine, }) } @@ -391,6 +401,16 @@ impl PlanInputs { } } + /// `self` for a loader that serves the common host-run NVMe arena, with the same residency + /// identity and split as the plan. Existing loaders remain non-paging unless they opt in. + #[must_use] + pub fn with_nvme_tier(self) -> PlanInputs { + PlanInputs { + nvme_tier: true, + ..self + } + } + /// `self` with each delta-rule layer's state `lanes` lanes ([`KvLayout::with_lanes`]): the /// stores of a load that verifies `lanes` rows a pass. #[must_use] @@ -457,8 +477,8 @@ impl PlanInputs { /// /// The host experts are held to the room the inputs name ([`PlanInputs::room`]): a room under /// the host need runs the common NVMe tier split ([`placement::expert_nvme_tier`], which refuses - /// a room under the tier's floor by name), and a plan that comes out paging any routed expert is - /// refused ([`PlaceError::NoNvmeTier`]), for the body serves no such tier. + /// a room under the tier's floor by name). A plan paging any routed expert is refused + /// ([`PlaceError::NoNvmeTier`]) unless the loader serves it ([`PlanInputs::with_nvme_tier`]). pub fn plan_with_slots<'a>( &'a self, machine: &'a Machine, @@ -485,7 +505,7 @@ impl PlanInputs { let need = HostNeed::of(&plan, 0).bytes(); let room = plan.plan_at(&self.room); placement::expert_nvme_tier(&mut plan, room, self.arena)?; - if plan.host.nvme_expert_bytes > 0 { + if plan.host.nvme_expert_bytes > 0 && !self.nvme_tier { return Err(PlaceError::NoNvmeTier { paged: plan.host.nvme_expert_bytes, need, diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index f4bd99284..70a47490a 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1471,7 +1471,6 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// card's staging, one chunk of the leg's scratch and the host's rows, for /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. -#[must_use] pub fn leg_tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, @@ -1481,6 +1480,7 @@ pub fn leg_tier_batch(hp: &Hparams) -> Result { crate::moe::UNION_MAX_COLS as u64, crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act) .with(crate::placement::kernels::DownAct::Q8Blocks32), + false, ) } diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 4926c5462..a48fc5e6d 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -45,7 +45,6 @@ pub const TIER_CHUNK: u64 = 8; /// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. -#[must_use] pub fn tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, @@ -55,6 +54,7 @@ pub fn tier_batch(hp: &Hparams) -> Result { crate::moe::UNION_MAX_COLS as u64, crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act) .with(crate::placement::kernels::DownAct::Q8Blocks32), + false, ) } diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 90e91e2dc..6ce4aaadd 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -54,6 +54,7 @@ pub fn tier_batch(inputs: &PlanInputs) -> Result TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, leg_down_forms(Program::Solar2Body, &inputs.model), + false, ) } @@ -137,6 +138,7 @@ pub fn prompt_reserve_bytes( n_used: experts.n_used as u64, chunk: r as u64, downs, + input_scales: false, } .bytes(downs)?; ((4 * f32s + acts + wide + 2 * route_bytes(r)) as u64) diff --git a/crates/model/src/moe.rs b/crates/model/src/moe.rs index 44725a667..312edf96b 100644 --- a/crates/model/src/moe.rs +++ b/crates/model/src/moe.rs @@ -22,7 +22,7 @@ //! the weight type's and nothing else. `crate::ops::matmul_q` is that single owner. use std::cell::RefCell; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering::Relaxed}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, Ordering::Relaxed}; use std::sync::{Mutex, PoisonError}; use std::time::Instant; @@ -30,9 +30,9 @@ use gguf::{GgmlType, Gguf, Split, TensorInfo}; use crate::ModelError; use crate::ops::{ - self, DEFER_MAX_COLS, ExpertForm, ExpertStack, GroupClaims, GroupInput, Lanes, PostNorm, - R8Stack, RowLayout, ShardTensor, StackTier, Tensor2, Tensor2View, UnionCall, UnionPlanView, - UnionSlabs, UnionStack, WarmSpan, Weight, defers, matmul_q, matmul_q_group, + self, DEFER_MAX_COLS, ExpertForm, ExpertInput, ExpertStack, GroupClaims, GroupInput, Lanes, + PostNorm, R8Stack, RowLayout, ShardTensor, StackTier, Tensor2, Tensor2View, UnionCall, + UnionPlanView, UnionSlabs, UnionStack, WarmSpan, Weight, defers, matmul_q, matmul_q_group, matmul_q_group_into_on, matmul_q_group_swiglu, matmul_q_group_swiglu_into_on, }; use crate::profile; @@ -875,6 +875,8 @@ pub struct HostScratch { pars: Vec, /// The down outputs, `{embd, 1}` each. downs: Vec, + /// Per-expert activation scales, restored before weighting the down outputs. + scales: Vec, /// The call's gate and up matrices, interleaved; empty between calls. gu_w: Vec>, /// The call's down matrices; empty between calls. @@ -910,6 +912,7 @@ impl HostScratch { gate_up: blocks(2 * n_used, ff), pars: blocks(n_used, ff), downs: blocks(n_used, embd), + scales: Vec::new(), gu_w: Vec::with_capacity(2 * n_used), down_w: Vec::with_capacity(n_used), xs: Vec::with_capacity(2 * n_used), @@ -918,6 +921,23 @@ impl HostScratch { }) } + /// Allocate protection scales at load; direct quantization needs none. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + match policy { + ExpertInput::Direct => self.scales = Vec::new(), + ExpertInput::AmaxF16 => self.scales.resize(self.n_used, 1.0), + } + } + + /// The policy this scratch has load-allocated scale storage for. + pub fn expert_input(&self) -> ExpertInput { + if self.scales.is_empty() { + ExpertInput::Direct + } else { + ExpertInput::AmaxF16 + } + } + fn fits(&self, embd: usize, ff: usize) -> bool { (self.embd, self.ff) == (embd, ff) } @@ -961,6 +981,7 @@ impl HostScratch { #[derive(Clone, Copy)] struct LegRun<'n> { form: ExpertForm, + input: ExpertInput, lanes: Lanes, post: Option<&'n PostNorm>, } @@ -971,6 +992,7 @@ impl LegRun<'_> { fn flat() -> LegRun<'static> { LegRun { form: ExpertForm::Gated { limit: 0.0 }, + input: ExpertInput::Direct, lanes: Lanes::Flat, post: None, } @@ -1042,11 +1064,18 @@ fn serve_resolved( s: &mut HostScratch, ) -> Result<(), ModelError> { let n = experts.len(); + if run.input == ExpertInput::AmaxF16 && s.scales.len() < n { + return Err(ModelError::Metadata { + key: "host expert input".to_string(), + detail: "AmaxF16 requires HostScratch::set_expert_input at load".to_string(), + }); + } let lvl = profile::level(); let HostScratch { gate_up, pars, downs, + scales, xs, srcs, .. @@ -1064,33 +1093,55 @@ fn serve_resolved( *xs = relist(xl); let gt = gt?; let mut sl: Vec> = relist(std::mem::take(srcs)); - match run.form { - ExpertForm::Gated { limit } => { - for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { - sl.push(GroupInput::SwigluClamp(gate, up, limit)); - } + let dt = if run.input == ExpertInput::AmaxF16 { + for i in 0..n { + let (gate, up) = match run.form { + ExpertForm::Gated { .. } | ExpertForm::GatedOai { .. } => { + (gate_up[2 * i].col(0), gate_up[2 * i + 1].col(0)) + } + ExpertForm::Relu2 => (&[][..], gate_up[i].col(0)), + }; + run.form.combine_into(gate, up, pars[i].col_mut(0)); + scales[i] = run.input.protect(pars[i].col_mut(0)); } - ExpertForm::GatedOai { alpha, limit } => { - for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { - sl.push(GroupInput::SwigluOai(gate, up, alpha, limit)); + sl.extend(pars[..n].iter().map(GroupInput::Ready)); + matmul_q_group_swiglu_into_on(run.lanes, "host_down", down, &sl, &mut downs[..n], &mut []) + } else { + match run.form { + ExpertForm::Gated { limit } => { + for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { + sl.push(GroupInput::SwigluClamp(gate, up, limit)); + } } - } - ExpertForm::Relu2 => { - for up in &gate_up[..n] { - sl.push(GroupInput::Relu2(up)); + ExpertForm::GatedOai { alpha, limit } => { + for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { + sl.push(GroupInput::SwigluOai(gate, up, alpha, limit)); + } + } + ExpertForm::Relu2 => { + for up in &gate_up[..n] { + sl.push(GroupInput::Relu2(up)); + } } } - } - let dt = matmul_q_group_swiglu_into_on( - run.lanes, - "host_down", - down, - &sl, - &mut downs[..n], - &mut pars[..n], - ); + matmul_q_group_swiglu_into_on( + run.lanes, + "host_down", + down, + &sl, + &mut downs[..n], + &mut pars[..n], + ) + }; *srcs = relist(sl); let dt = dt?; + if run.input == ExpertInput::AmaxF16 { + for (d, &scale) in downs[..n].iter_mut().zip(&scales[..n]) { + for v in d.col_mut(0) { + *v *= scale; + } + } + } if let ExpertForm::Relu2 = run.form { // Back to back from the top: expert `i`'s up goes to slot `2 * i + 1`, // where `HostScratch::up` reads it, and no block moves twice. @@ -1430,6 +1481,8 @@ pub struct UnionScratch { qc: Vec, /// Slot `q`'s down output at `store[q·embd..(q + 1)·embd]`. store: Vec, + /// Per-slot activation scales; claims publish them before the down reads. + scales: Vec, plan: UnionPlan, /// A narrow call's claim state per distinct expert: /// `min(max_cols, DEFER_MAX_COLS) · per_list`, every expert such a call @@ -1550,6 +1603,7 @@ impl UnionScratch { gu: vec![0.0; 2 * slots * ff], qc: vec![0; slots * c_col], store: vec![0.0; slots * embd], + scales: Vec::new(), plan: UnionPlan::with_room(cols, per_list), claims: (0..cols.min(DEFER_MAX_COLS) * per_list) .map(|_| AtomicU8::new(0)) @@ -1559,6 +1613,16 @@ impl UnionScratch { } } + /// Allocate protection scales at load; direct quantization needs none. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + self.scales = match policy { + ExpertInput::Direct => Vec::new(), + ExpertInput::AmaxF16 => (0..self.max_slots) + .map(|_| AtomicU32::new(1.0f32.to_bits())) + .collect(), + }; + } + /// The most columns a call through this scratch takes. pub fn max_cols(&self) -> usize { self.max_cols @@ -1866,6 +1930,7 @@ fn serve_batched<'w>( gu, qc, store, + scales, passes, batches, .. @@ -1886,7 +1951,8 @@ fn serve_batched<'w>( }; let whole = UnionCall::new(plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let mut xqs = whole.quantize_batched(xq, xq_up, passes); slot.clear(); slot.resize(plan.n(), 0); @@ -1927,7 +1993,8 @@ fn serve_batched<'w>( } let batch_call = UnionCall::new(sub.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; batch_call.run_batch(&mut xqs, gu, qc, &mut store[base * embd..], None, passes)?; base += sub.cols.len(); drop(pin); @@ -1982,6 +2049,7 @@ fn serve_step_split<'w>( gu, qc, store, + scales, passes, batches, claims, @@ -1998,7 +2066,8 @@ fn serve_step_split<'w>( let (order, ids) = (&*order, &*ids); let whole = UnionCall::new(plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let deferred = defers(x.ne1()); let mut xqs = if deferred { whole.x_rooms(xq, xq_up) @@ -2032,7 +2101,8 @@ fn serve_step_split<'w>( } let call = UnionCall::new(sub.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let claim = deferred.then(|| GroupClaims { x: g == 0, states: &mut claims[..], @@ -2069,7 +2139,8 @@ fn serve_planned( } let call = UnionCall::new(s.plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, &s.scales)?; let slabs = UnionSlabs { xq: &mut s.xq, xq_up: &mut s.xq_up, @@ -2637,6 +2708,7 @@ pub struct HostLayer { /// made for it ([`HostScratch::new`]). ff: usize, form: ExpertForm, + input: ExpertInput, tier: Option, /// The norm each expert's down row takes before its weight /// ([`HostLayer::set_post_norm`]); `None`, the plain weighted sum. @@ -2798,11 +2870,23 @@ impl HostLayer { n_expert, ff, form, + input: ExpertInput::Direct, tier: None, post: None, }) } + /// Select the activated expert input policy for every host serving path. + /// This is a load-time setting; the default preserves direct quantization. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + self.input = policy; + } + + /// The loaded activated-expert input policy. + pub fn expert_input(&self) -> ExpertInput { + self.input + } + /// Give the layer the norm each routed expert's down row takes before /// its router weight ([`PostNorm`]): from then on every call that serves /// the layer — the one-column leg, the union and the step union, from the @@ -3091,6 +3175,7 @@ impl HostLayer { }; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3158,6 +3243,7 @@ impl HostLayer { tier.lend(&mut stacks); let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3213,6 +3299,7 @@ impl HostLayer { )?; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3284,6 +3371,7 @@ impl HostLayer { let mut stacks = self.stacks_of(split)?; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; diff --git a/crates/model/src/ops.rs b/crates/model/src/ops.rs index 298985a72..7fbd55856 100644 --- a/crates/model/src/ops.rs +++ b/crates/model/src/ops.rs @@ -5116,6 +5116,52 @@ pub enum ExpertForm { Relu2, } +/// How an activated routed expert enters its down projection. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ExpertInput { + /// Quantize the activation unchanged. + #[default] + Direct, + /// Rescale each expert/token activation for F16 activation-block sums, + /// then restore the down output before applying its router weight. + AmaxF16, +} + +impl ExpertInput { + /// Divide one activated expert/token column by the oracle's scale. + /// The returned factor restores its down output. + pub(crate) fn protect(self, h: &mut [f32]) -> f32 { + match self { + ExpertInput::Direct => 1.0, + ExpertInput::AmaxF16 => { + let mut amax = 0.0f32; + for &v in h.iter() { + assert!(v.is_finite(), "expert input: non-finite activation {v}"); + amax = amax.max(v.abs()); + } + let exponent = ((amax / 1024.0).ln() * std::f32::consts::LOG2_E) + .ceil() + .clamp(0.0, 127.0); + let scale = (exponent * std::f32::consts::LN_2).exp(); + for v in h { + *v /= scale; + } + scale + } + } + } +} + +impl ExpertForm { + pub(crate) fn combine_into(self, gate: &[f32], up: &[f32], out: &mut [f32]) { + match self { + ExpertForm::Gated { limit } => swiglu_into(gate, up, Some(limit), out), + ExpertForm::GatedOai { alpha, limit } => swiglu_oai_into(gate, up, alpha, limit, out), + ExpertForm::Relu2 => relu2_into(up, out), + } + } +} + /// A layer's norm of each routed expert's down row before the row's router /// weight applies: the row scaled by `gain`, then divided by the root of its /// mean square plus `eps` (the card's `gain_rms_norm`). An architecture whose @@ -5375,6 +5421,7 @@ struct Combines<'c> { gu: &'c [f32], ff: usize, form: ExpertForm, + scales: Option<&'c [std::sync::atomic::AtomicU32]>, ty: GgmlType, qc: SharedQuantCols, } @@ -5413,6 +5460,10 @@ impl Combines<'_> { relu2_into(up, par); } } + if let Some(scales) = self.scales { + let scale = ExpertInput::AmaxF16.protect(par); + scales[q].store(scale.to_bits(), std::sync::atomic::Ordering::Relaxed); + } // SAFETY: the caller owns slot `q` of `qc`. unsafe { self.qc.quantize_into(self.ty, self.ff, par, q) }; }); @@ -5824,6 +5875,7 @@ pub(crate) struct UnionCall<'c, 'w> { /// The norm each down takes before its weight in the sums, when the /// layer has one. post: Option<&'c PostNorm>, + scales: Option<&'c [std::sync::atomic::AtomicU32]>, } impl<'c, 'w> UnionCall<'c, 'w> { @@ -5861,6 +5913,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { down, spread: Lanes::Flat, post: None, + scales: None, }) } @@ -5877,6 +5930,28 @@ impl<'c, 'w> UnionCall<'c, 'w> { UnionCall { post, ..self } } + /// Apply the loaded activation policy, using load-allocated slot scales. + pub(crate) fn expert_input( + self, + input: ExpertInput, + scales: &'c [std::sync::atomic::AtomicU32], + ) -> Result { + let scales = match input { + ExpertInput::Direct => None, + ExpertInput::AmaxF16 => { + if scales.len() < self.plan.slots() { + return Err(crate::ModelError::Metadata { + key: "host union expert input".to_string(), + detail: "AmaxF16 requires UnionScratch::set_expert_input at load" + .to_string(), + }); + } + Some(scales) + } + }; + Ok(UnionCall { scales, ..self }) + } + /// The cells the gate/up pass writes: two columns a slot gated — the /// gate's and the up's — one a slot for a relu² layer, the up's alone. fn gu_cells(&self) -> usize { @@ -6215,6 +6290,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { cb: self.down.cb, ptr: qc.as_mut_ptr(), }, + scales: self.scales, } } @@ -6299,6 +6375,14 @@ impl<'c, 'w> UnionCall<'c, 'w> { let mut pacc = profile::CallAcc::new(); let t0 = if lvl > 0 { Some(Instant::now()) } else { None }; run_row_pool(&set, lvl, &mut pacc, true, self.spread)?; + if let Some(scales) = self.scales { + for (row, scale) in store.chunks_exact_mut(self.down.n).zip(scales) { + let scale = f32::from_bits(scale.load(std::sync::atomic::Ordering::Relaxed)); + for v in row { + *v *= scale; + } + } + } let claim_ns = claims.claim_ns.into_inner(); if let Some(t0) = t0 { let wall = t0.elapsed().as_nanos() as u64; @@ -6757,6 +6841,35 @@ mod tests { }; use gguf::{GgmlType, Gguf}; + #[test] + fn expert_range_protection_preserves_values_and_bounds_large_activated_inputs() { + use super::ExpertInput; + for peak in [0.0f32, 1024.0, 1024.125, 2048.0, 2048.25, 1.0e8] { + let original = [peak, -peak * 0.5, peak * 0.25]; + let mut direct = original; + assert_eq!(ExpertInput::Direct.protect(&mut direct), 1.0); + assert_eq!(direct, original); + let mut protected = original; + let scale = ExpertInput::AmaxF16.protect(&mut protected); + assert!( + protected + .iter() + .all(|v| v.abs() <= 1024.0 * (1.0 + 4.0 * f32::EPSILON)) + ); + if peak <= 1024.0 { + assert_eq!(scale, 1.0); + assert_eq!(protected, original); + } else { + assert!(scale > 1.0); + for (actual, expected) in protected.into_iter().zip(original) { + assert!( + (actual * scale - expected).abs() <= expected.abs() * 2.0 * f32::EPSILON + ); + } + } + } + } + /// A GGUF v3 file holding one `[k, 2]` tensor `w` of type `ty` over zero bytes — /// the least the strict reader opens. fn one_tensor_file(ty: GgmlType, k: usize) -> std::path::PathBuf { diff --git a/crates/model/tests/flint_meta.rs b/crates/model/tests/flint_meta.rs index bde81addf..04887c059 100644 --- a/crates/model/tests/flint_meta.rs +++ b/crates/model/tests/flint_meta.rs @@ -106,30 +106,46 @@ fn hw_flint_header() { .filter(|(n, _)| matches!(n, Need::Gqa { sinks: true, .. })) .count(); assert_eq!((norms, rotary, sinks), (49, 35, 49)); - let coverage = model::arch::coverage::check(s, &r.tensors); - for (need, layer) in &needs { + for (need, _) in &needs { if let Need::Router(shape) = need { let binding = model::arch::models::shape::select_router(*shape).unwrap(); - assert_eq!(binding.body, model::arch::models::shape::RouterBody::SigmoidFloor); + assert_eq!( + binding.body, + model::arch::models::shape::RouterBody::SigmoidFloor + ); assert_eq!(binding.rule, shape.rule); - assert_eq!((binding.experts(), binding.top_k_min, binding.top_k_max), (256, 6, 6)); - assert!(coverage.iter().any(|u| { - u.layer == layer.map(|l| l as usize) && u.feature == need.to_string() - })); + assert_eq!( + (binding.experts(), binding.top_k_min, binding.top_k_max), + (256, 6, 6) + ); } } - assert!( - coverage - .iter() - .any(|u| u.feature == Need::Program("flint").to_string()) - ); - assert!(model::arch::coverage::program_of(s.arch).is_none()); let bytes: u64 = r.tensors.tensors.iter().map(|t| t.file_bytes).sum(); assert_eq!(bytes, 25_049_218_176); + let mut inputs = model::arch::flint::place::PlanInputs::read(&split) + .expect("the validated header has genuine program and site bindings"); + inputs.room = (u64::MAX, model::placement::workstation::HostRead::Given); + let machine = model::placement::workstation::plan_gate(s.layers.len()); + let levers = model::placement::PlanLevers::default(); + for ctx in [512, 1024, 4096] { + for slots in [1, 3] { + let plan = inputs + .plan_with_slots( + &machine, + ctx, + &levers, + std::num::NonZeroUsize::new(slots).unwrap(), + ) + .expect("the real mixed-quant header plans with resident slots"); + let cache = slots as u64 * 2048 * (14 * ctx + 35 * ctx.min(1024)); + assert_eq!(plan.cards.iter().map(|c| c.kv_bytes).sum::(), cache); + assert!(inputs.prompt_batch_bytes(&machine).unwrap() > 0); + } + } + let tier = model::arch::flint::place::tier_batch(&inputs).unwrap(); + assert!(tier.leg.unwrap().input_scales); + assert!(tier.card().unwrap() > 0); println!( "flint header_sha={HEADER_SHA} tensors=779 layers=49 dense=2 routed=47 norms={norms} rotary={rotary} sinks={sinks} payload_bytes={bytes}" ); - for item in coverage { - println!("unimplemented layer={:?} {}", item.layer, item.feature); - } } diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index 972d82125..71b1e998f 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -529,7 +529,10 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); let machine = tiered( - Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.hp).expect("tier batch bytes"), + Q36_LAYERS, + stage, + tier, + q35::leg_tier_batch(&inputs.hp).expect("tier batch bytes"), ); let plan = inputs.plan_rule( &machine, @@ -546,7 +549,10 @@ fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place fn qwen3_30b_q8_bp_tier_experts() { let inputs = qwen3_inputs(); let machine = tiered( - Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp).expect("tier batch bytes"), + Q3_LAYERS, + A6000, + RTX_3090, + q3::tier_batch(&inputs.hp).expect("tier batch bytes"), ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) diff --git a/crates/models/src/shape.rs b/crates/models/src/shape.rs index 6fd7f1592..8680797c4 100644 --- a/crates/models/src/shape.rs +++ b/crates/models/src/shape.rs @@ -934,8 +934,8 @@ pub fn gqa_row(head: u32, group: u32) -> Option { #[cfg(test)] mod tests { use super::{ - AttnShape, GQA, GroupRule, LANE_PICKS, MoeShape, ROUTERS, Refusal, RouterInst, RouterRule, - Shape, gqa_row_v, router_row, router_row_is, router_row_k, rules, select_gqa, + AttnShape, GQA, GroupRule, LANE_PICKS, MoeShape, ROUTERS, Refusal, RouterBody, RouterInst, + RouterRule, Shape, gqa_row_v, router_row, router_row_is, router_row_k, rules, select_gqa, select_router, }; use crate::{Act, Moe, Router, RouterNorm, Score, Shared}; @@ -1022,11 +1022,13 @@ mod tests { #[test] fn floor_top_six_serves_both_bias_modes_and_only_its_shape() { for bias in [false, true] { - let rule = RouterRule { bias, ..rules::SIGMOID_FLOOR }; + let rule = RouterRule { + bias, + ..rules::SIGMOID_FLOOR + }; let row = select_router(moe(rule, 256, 6)).unwrap(); assert_eq!(row.body, RouterBody::SigmoidFloor); assert_eq!(row.rule, rule); - assert_eq!(row, router_row_rule(RouterBody::SigmoidFloor, rule, 256, 6)); for top_k in [0, 5, 7, 8, 257] { assert_eq!( select_router(moe(rule, 256, top_k)).unwrap_err().why, @@ -1038,7 +1040,11 @@ mod tests { Refusal::RouterWidth { ref served } if served == &[256] )); for (score, gated) in [(Score::Softmax, false), (Score::Sigmoid, true)] { - let wrong = RouterRule { score, gated, ..rule }; + let wrong = RouterRule { + score, + gated, + ..rule + }; assert_eq!( select_router(moe(wrong, 256, 6)).unwrap_err().why, Refusal::NoRouterRule @@ -1048,10 +1054,16 @@ mod tests { let legacy = select_router(moe(rules::BIASED_SIGMOID, 256, 6)).unwrap(); assert_eq!(legacy.body, RouterBody::BiasedSigmoid); assert_eq!( - select_router(moe(RouterRule { - norm: RouterNorm::SelectedSum, - ..rules::SIGMOID_FLOOR - }, 256, 6)).unwrap_err().why, + select_router(moe( + RouterRule { + norm: RouterNorm::SelectedSum, + ..rules::SIGMOID_FLOOR + }, + 256, + 6 + )) + .unwrap_err() + .why, Refusal::NoRouterRule ); } @@ -1315,9 +1327,7 @@ mod tests { assert_eq!(MoeShape::of(&m).rule, rules::SOFTMAX_NORM); } - /// Rows are distinct instances: no two rows run one rule at one width - /// and one pick count, and one body holds two rows at a width only by - /// disjoint pick ranges. + /// No two rows select the same rule, expert width and pick count. #[test] fn no_two_router_rows_overlap() { for (i, a) in ROUTERS.iter().enumerate() { @@ -1325,12 +1335,6 @@ mod tests { let picks = a.top_k_min <= b.top_k_max && b.top_k_min <= a.top_k_max; let overlap = a.experts == b.experts && (a.runs(b.rule) || b.runs(a.rule)) && picks; assert!(!overlap, "{} and {}", a.at, b.at); - assert!( - !((a.body, a.experts) == (b.body, b.experts) && picks), - "{} and {}", - a.at, - b.at - ); } assert!(a.top_k_min >= 1 && a.top_k_min <= a.top_k_max, "{}", a.at); assert!(a.per_lane() >= 1 && a.per_lane() <= 32, "{}", a.at); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index daa05c3ec..856aee47b 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -67,6 +67,8 @@ pub enum Program { /// gate·up and down `_sel` entries ([`MUSESPARK_LEG`]) and the rest on the /// host tier. MuseSparkBody, + /// The head128 sink/SWA body with floor-normalized top-six experts. + FlintBody, } /// The program that runs `arch`'s layers; `None` when none does. @@ -85,7 +87,7 @@ pub fn program_of(arch: Arch) -> Option { Arch::NemotronH => Some(Program::NemotronHBody), Arch::NemotronHMoe => Some(Program::NemotronHMoeBody), Arch::GemmaEmbedding | Arch::GemmaEmbedding2 => Some(Program::GemmaEmbedding2Body), - Arch::Flint => None, + Arch::Flint => Some(Program::FlintBody), Arch::MuseSpark => Some(Program::MuseSparkBody), } } @@ -110,6 +112,7 @@ impl Program { Self::GemmaEmbedding2Body => "gemma-embedding2", Self::Raon1Body => "raon1", Self::MuseSparkBody => "musespark", + Self::FlintBody => "flint", } } } @@ -404,7 +407,7 @@ pub struct LegArms { #[must_use] pub fn leg_arms(p: Program) -> Option { match p { - Program::Qwen3moeBody | Program::Qwen35Body => Some(LegArms { + Program::Qwen3moeBody | Program::Qwen35Body | Program::FlintBody => Some(LegArms { gate_up: prompt_leg_gate_up, down: prompt_leg_down, }), @@ -500,7 +503,7 @@ pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> ker #[must_use] pub fn routed_rule(p: Program) -> RoutedRule { match p { - Program::Qwen3moeBody | Program::Qwen35Body => RoutedRule::Leg { + Program::Qwen3moeBody | Program::Qwen35Body | Program::FlintBody => RoutedRule::Leg { gate_up: |ty| prompt_leg_gate_up(ty).then(|| leg_card_format(ty)), down: |ty| prompt_leg_down(ty).then(|| leg_card_format(ty)), }, @@ -578,7 +581,7 @@ mod tests { use super::*; use crate::placement::{ModelTensor, ModelTensors, Role}; - const PROGRAMS: [Program; 12] = [ + const PROGRAMS: [Program; 13] = [ Program::Deepseek41Chain, Program::Qwen3moeBody, Program::Qwen35Body, @@ -591,6 +594,7 @@ mod tests { Program::NemotronHMoeBody, Program::Raon1Body, Program::MuseSparkBody, + Program::FlintBody, ]; /// Every type the file format names. diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index f2d85d534..ac0f34f34 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -16,6 +16,9 @@ pub use gguf::card::{ }; use models::Role; +pub mod attention; +pub mod ffn; + /// The common card gate·up `_sel` entry that runs a stack of type `ty`. #[must_use] pub const fn gate_up(ty: GgmlType) -> Option { diff --git a/crates/placement/src/kernels/attention.rs b/crates/placement/src/kernels/attention.rs new file mode 100644 index 000000000..c896272fd --- /dev/null +++ b/crates/placement/src/kernels/attention.rs @@ -0,0 +1,79 @@ +//! The buffers of the common head128 projection/attention scratch. + +use super::{SiteAct, try_q8_act_bytes}; + +/// Quantized wide forms beside the caller-owned f32 input rows. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Forms { + pub q128: bool, + pub q32: bool, +} + +impl Forms { + /// Include the activation form a selected projection reads. + #[must_use] + pub const fn with(self, form: SiteAct) -> Self { + Self { + q128: self.q128 || matches!(form, SiteAct::Q8x128), + q32: self.q32 || matches!(form, SiteAct::Q8x32 | SiteAct::Q8Blocks32), + } + } + + /// Bytes of these forms over `rows` columns of `width` values. + #[must_use] + pub fn bytes(self, rows: u64, width: u64) -> Option { + let q128 = if self.q128 { + try_q8_act_bytes(rows, width)? + } else { + 0 + }; + let q32 = if self.q32 { + wide32_bytes(rows, width)? + } else { + 0 + }; + q128.checked_add(q32) + } +} + +/// Bytes of wide q8 blocks of 32: twenty words per padded 64-value step. +#[must_use] +pub const fn wide32_bytes(rows: u64, width: u64) -> Option { + match rows.checked_mul(width.div_ceil(64)) { + Some(steps) => steps.checked_mul(80), + None => None, + } +} + +/// The loaded projection widths, partial segments and actual selected forms. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Head128Scratch { + pub hidden: u64, + pub query: u64, + pub key_value: u64, + pub heads: u64, + pub segments: u64, + pub input: Forms, + pub output: Forms, +} + +impl Head128Scratch { + /// Checked bytes for Q/K/V, normalized/residual rows, attention/partials, + /// selected wide forms and the one-matrix route table. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let partial = self.heads.checked_mul(self.segments)?.checked_mul(130)?; + let values = self + .hidden + .checked_mul(2)? + .checked_add(self.query.checked_mul(2)?)? + .checked_add(self.key_value.checked_mul(2)?)? + .checked_add(partial)?; + let floats = values.checked_mul(rows)?.checked_mul(4)?; + let route = super::ffn::dense_route_bytes(rows)?; + floats + .checked_add(self.input.bytes(rows, self.hidden)?)? + .checked_add(self.output.bytes(rows, self.query)?)? + .checked_add(route) + } +} diff --git a/crates/placement/src/kernels/ffn.rs b/crates/placement/src/kernels/ffn.rs new file mode 100644 index 000000000..0087d0343 --- /dev/null +++ b/crates/placement/src/kernels/ffn.rs @@ -0,0 +1,234 @@ +//! Buffer geometry of the common routed/shared FFN and prompt walk. + +use super::SiteAct; +use super::attention::Forms; +use crate::placement::workstation::TierLegScratch; + +/// The first unit size whose shared and dense blocks use the wide path. +pub const GATED_FROM: u64 = 9; + +/// Wide quantized forms and an owned f32 row, when a selected down reads one. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct WideForms { + pub quant: Forms, + pub f32: bool, +} + +impl WideForms { + /// Include a selected site's activation form. + #[must_use] + pub const fn with(self, reads: SiteAct) -> Self { + Self { + quant: self.quant.with(reads), + f32: self.f32 || matches!(reads, SiteAct::F32), + } + } + + /// Input f32 rows are borrowed from the caller. + #[must_use] + pub const fn without_f32(self) -> Self { + Self { f32: false, ..self } + } + + /// Checked allocation bytes of these forms. + #[must_use] + pub fn bytes(self, rows: u64, width: u64) -> Option { + let floats = if self.f32 { + rows.checked_mul(width)?.checked_mul(4)? + } else { + 0 + }; + self.quant.bytes(rows, width)?.checked_add(floats) + } +} + +/// One shared or dense block's wide scratch, with the actual selected read forms. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WideBlock { + pub hidden: u64, + pub ff: u64, + pub input: WideForms, + pub down: WideForms, +} + +impl WideBlock { + /// Route table, borrowed-input forms, owned activation forms, and gate/up outputs. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + dense_route_bytes(rows)? + .checked_add(self.input.without_f32().bytes(rows, self.hidden)?)? + .checked_add(self.down.bytes(rows, self.ff)?)? + .checked_add(rows.checked_mul(self.ff)?.checked_mul(8)?) + } +} + +/// A single-expert wide route table: slot ids, tiles, counts and zero ids. +#[must_use] +pub const fn dense_route_bytes(rows: u64) -> Option { + let bytes = (2 * rows as u128 + 2 * (rows as u128 / 64 + 1) + 2) * 4; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } +} + +/// The common card batch beside its leg: accumulation/base rows and selected places. +#[must_use] +pub fn card_batch_bytes(leg: TierLegScratch) -> Option { + leg.bytes(leg.downs) + .ok()? + .checked_add(leg.chunk.checked_mul(leg.n_embd)?.checked_mul(8)?)? + .checked_add(leg.chunk.checked_mul(leg.n_used)?.checked_mul(4)?) +} + +/// Places, join places and join weights of a prompt unit on an expert-tier load. +#[must_use] +pub const fn tier_places_bytes(rows: u64, used: u64) -> Option { + match rows.checked_mul(used) { + Some(slots) => slots.checked_mul(12), + None => None, + } +} + +/// Exact buffers shared by all units of the routed/shared prompt walk. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RoutedBatch { + pub hidden: u64, + pub experts: u64, + pub used: u64, + pub shared: WideBlock, + pub dense: Option, + pub card: Option, + pub tiered: bool, +} + +impl RoutedBatch { + /// Norm/router/shared/base rows, narrow activation rows, optional dense/card/wide/tier buffers. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let dense_ff = self.dense.map_or(0, |d| d.ff); + let mut values = self + .hidden + .checked_mul(3)? + .checked_add(self.experts)? + .checked_add(self.used.checked_mul(2)?)? + .checked_add(self.shared.ff.max(dense_ff))?; + if self.dense.is_some() { + values = values.checked_add(self.hidden.checked_mul(2)?)?; + } + let mut bytes = rows.checked_mul(values)?.checked_mul(4)?; + if rows >= GATED_FROM { + bytes = bytes.checked_add(self.shared.bytes(rows)?)?; + if let Some(dense) = self.dense { + bytes = bytes.checked_add(dense.bytes(rows)?)?; + } + } + if let Some(card) = self.card { + if card.chunk != rows { + return None; + } + bytes = bytes.checked_add(card_batch_bytes(card)?)?; + } + if self.tiered { + bytes = bytes.checked_add(tier_places_bytes(rows, self.used)?)?; + } + Some(bytes) + } +} + +/// Prompt walk units and the shared host-sum upload buffer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PromptWalk { + pub hidden: u64, + pub sets: u64, +} + +impl PromptWalk { + /// Each unit owns x/x1 plus positions/key counts; all units share one host-sum row set. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let unit = self.hidden.checked_mul(8)?.checked_add(8)?; + self.sets + .checked_mul(unit)? + .checked_add(self.hidden.checked_mul(4)?)? + .checked_mul(rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kernels::{DownAct, DownForms}; + + #[test] + fn wide_threshold_selected_forms_and_protection_are_independent() { + let wide = WideBlock { + hidden: 256, + ff: 256, + input: WideForms::default().with(SiteAct::F32), + down: WideForms::default().with(SiteAct::F32), + }; + let batch = RoutedBatch { + hidden: 256, + experts: 8, + used: 2, + shared: wide, + dense: None, + card: None, + tiered: false, + }; + assert_eq!(batch.bytes(8), Some(8 * 4 * (3 * 256 + 8 + 4 + 256))); + assert_eq!( + batch.bytes(9), + Some(9 * 4 * (3 * 256 + 8 + 4 + 256) + dense_route_bytes(9).unwrap() + 9 * 256 * 12) + ); + let leg = TierLegScratch { + n_embd: 256, + ff: 256, + n_used: 2, + chunk: 9, + downs: DownForms::of(DownAct::Q8Act), + input_scales: false, + }; + let direct = card_batch_bytes(leg).unwrap(); + let scaled = card_batch_bytes(TierLegScratch { + input_scales: true, + ..leg + }) + .unwrap(); + assert_eq!(scaled - direct, 4 * 9 * 2); + let q32 = WideBlock { + input: WideForms::default().with(SiteAct::Q8x32), + ..wide + }; + assert_eq!(q32.bytes(9).unwrap() - wide.bytes(9).unwrap(), 9 * 4 * 80); + assert_eq!(batch.bytes(u64::MAX), None); + } + + #[test] + fn walk_groups_only_duplicate_unit_rows() { + let one = PromptWalk { + hidden: 256, + sets: 1, + } + .bytes(512) + .unwrap(); + let two = PromptWalk { + hidden: 256, + sets: 2, + } + .bytes(512) + .unwrap(); + assert_eq!(one, 512 * (12 * 256 + 8)); + assert_eq!(two - one, 512 * (8 * 256 + 8)); + assert_eq!( + PromptWalk { + hidden: u64::MAX, + sets: 1 + } + .bytes(512), + None + ); + } +} diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 68156e9c9..0a7830a70 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -6083,7 +6083,7 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, tier_leg_batch_bytes}; let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); - let batch = tier_leg_batch_bytes(256, 256, 1, 2, 3, both).expect("tier bytes"); + let batch = tier_leg_batch_bytes(256, 256, 1, 2, 3, both, false).expect("tier bytes"); let leg = batch.leg.expect("a tier leg descriptor"); let rule = routed_rule(Program::Qwen35Body); for (downs, first) in [ @@ -6159,7 +6159,7 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, TierLegScratch, tier_leg_batch_bytes}; let forms = DownForms::of(DownAct::Q8Act); - let batch = tier_leg_batch_bytes(256, 256, 1, 1, 1, forms).expect("valid batch"); + let batch = tier_leg_batch_bytes(256, 256, 1, 1, 1, forms, false).expect("valid batch"); let good = batch.leg.expect("a leg descriptor"); let mut model = legged(&[[GgmlType::Q5_K; 3]]); model.experts_used = 1; @@ -6218,13 +6218,15 @@ mod tests { ); assert!( matches!( - tier_leg_batch_bytes(leg.n_embd, leg.ff, leg.n_used, leg.chunk, 1, leg.downs), + tier_leg_batch_bytes( + leg.n_embd, leg.ff, leg.n_used, leg.chunk, 1, leg.downs, leg.input_scales, + ), Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" ), "{leg:?}" ); } - assert!(tier_leg_batch_bytes(256, 256, 8, 1, 1 << 61, forms).is_err()); + assert!(tier_leg_batch_bytes(256, 256, 8, 1, 1 << 61, forms, false).is_err()); let inconsistent = workstation::TierBatchBytes { scratch: 0, ..batch diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 2f6f72386..7614702c4 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -437,6 +437,8 @@ pub struct TierLegScratch { pub n_used: u64, pub chunk: u64, pub downs: DownForms, + /// One f32 input-restoration scale per slot of the leg's chunk. + pub input_scales: bool, } impl TierLegScratch { @@ -460,7 +462,16 @@ impl TierLegScratch { { return Err(invalid()); } - try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs).ok_or_else(invalid) + let counted = || { + let bytes = try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs)?; + let scales = if self.input_scales { + 4u64.checked_mul(self.chunk)?.checked_mul(self.n_used)? + } else { + 0 + }; + bytes.checked_add(scales) + }; + counted().ok_or_else(invalid) } } @@ -568,6 +579,7 @@ pub fn tier_leg_batch_bytes( chunk: u64, cols: u64, downs: DownForms, + input_scales: bool, ) -> Result { let leg = TierLegScratch { n_embd, @@ -575,6 +587,7 @@ pub fn tier_leg_batch_bytes( n_used, chunk, downs, + input_scales, }; let scratch = leg.bytes(downs)?; let counted = || -> Option<(u64, u64)> { @@ -1680,6 +1693,39 @@ mod tests { plan_on, plan_room, plan_tiers, spec_of_device, tier_batch_bytes, unified_of, }; + #[test] + fn tier_input_scales_cost_one_f32_per_chunk_slot() { + use crate::kernels::{DownAct, DownForms}; + + let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + for chunk in [1, 8, 16] { + for downs in [DownForms::NONE, DownForms::of(DownAct::Q8Act), both] { + let direct = super::tier_leg_batch_bytes(256, 256, 6, chunk, 32, downs, false) + .expect("direct descriptor"); + let scaled = super::tier_leg_batch_bytes(256, 256, 6, chunk, 32, downs, true) + .expect("scaled descriptor"); + assert_eq!(scaled.staging, direct.staging); + assert_eq!(scaled.host, direct.host); + assert_eq!(scaled.scratch, direct.scratch + 4 * chunk * 6); + assert_eq!( + scaled.card().expect("scaled card"), + direct.card().expect("direct card") + 4 * chunk * 6, + ); + let leg = scaled.leg.expect("scaled leg"); + assert!(leg.input_scales); + assert_eq!( + leg.bytes(DownForms::NONE).expect("empty selected union"), + direct + .leg + .expect("direct leg") + .bytes(DownForms::NONE) + .expect("empty union") + + 4 * chunk * 6, + ); + } + } + } + /// One plan room: `tools/ref/plan-room.sh --room`, the room /// `records-refresh` makes V4.1's plans at, prints [`PLAN_ROOM`], and /// the workstation's host leaves a load that room diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 64ad6ee9b..e76c7c082 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,9 +2,9 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader** and a real common -256/top6 floor-router binding; it still has no Flint body, app session or serve seat. -No GPU, inference, performance or weight-content validation is claimed. +that snapshot. The integration based on **`e0c82e1779a8e754c53f4442800e6c3d61271a48`** +adds an actual `gpu-flint` body, application session and shared HTTP seat, not just architecture +admission. Device execution, GPU numerical acceptance and performance remain unverified. ## Host reader status @@ -15,15 +15,15 @@ It reuses the common metadata validators, tensor stems, classification and dimen GemmaEmbedding2's sliding mask: missing keys, scalars, wrong lengths and non-bool items are refused. Present `expert_weights_scale` uses `arch::finite_positive`; only an absent key defaults to 1, with that default recorded. -The common needs walk emits q/k normalization even on NoPE layers. `program_of(Flint)` remains -`None`; missing program, head128 sinks/SWA and NoPE normalization remain coverage refusals. -`Router.norm` and `RouterRule.norm` distinguish no normalization, the existing selected-sum -rule and Flint's selected-sum floor. The real floor-router binding is selectable in both bias -modes; selecting that operation does not supply a whole program or activated-value scaling. +The common needs walk emits q/k normalization even on NoPE layers. `program_of(Flint)` selects +`FlintBody`, whose coverage names the operations its loader actually binds: head128/group8 +sink attention, independent NoPE/NeoX and SWA masks, and the real floor router. +`Router.norm` and `RouterRule.norm` keep the old selected-sum rule distinct from Flint's floor. +The activated-value protection and explicit residual association are loaded FFN policies. Fault formatting uses `bloomery_device::fault`'s shared layout/decoder and the -`gpu::arch::step_order` registry. Flint's registry row uses neutral code order; -it does not describe a runnable step. +`gpu::arch::step_order` registry. The body delegates fault checks and captured execution to the +common chain runtime; the architecture registry's neutral order is not numerical evidence. Run `BLOOMERY_BOX_ENV="BLOOMERY_MODEL=/absolute/path/flint-header.gguf" BLOOMERY_BOX_LOCAL=1 just gate-flint-meta`. Set `BLOOMERY_ENV_FILE` to the local toolchain environment file if it is not `~/bloomery-env.sh`. @@ -45,16 +45,14 @@ remaining acceptance rows. This header gate is not an end-to-end model gate. ### Runtime handoff and host policy The design, common template support, tokenizer/four-EOG contract and reader (#676–#680) are merged. -The existing header recipe was exercised on the exact 5,470,528-byte prefix above: four reader tests and -the pinned-header test passed, still reporting the missing program and operation bindings. -No full weights or GPU were needed for that check. +The pinned-header consumer now also checks actual Program coverage and planning on the complete +local GGUF: 779 tensors, 49 layers, 2 dense/47 routed, 49 head norms, 35 rotations and 49 sinks. +This reads the header, not tensor contents; it does not prove CUDA execution. -The maintainer [assigned the model port and offered card runs](https://github.com/midagedev/bloomery/issues/632#issuecomment-6352066886), -and named a forthcoming head128 sink/SWA attention, QK RMS and sigmoid256/top6 train. -Those complete attention contracts are not in this main snapshot or the inspected public heads of -#726 (`1723485602c1`) and #747 (`f883e84f8d2c`). Existing ring and sink-merge ingredients do not -establish the combined attention operation. Obtain the train's fetchable ref before duplicating -those primitives. +The maintainer [assigned the model port and offered card runs](https://github.com/midagedev/bloomery/issues/632#issuecomment-6352066886). +The later public QK-RMS, sink/SWA and sigmoid train supplies reusable primitives. The new common +`attention_h128` binding combines those with actual ring append/attention kernels; it is not an +identity-RoPE substitute or a padded-head adaptation of another family. The common `route_body::sigmoid_floor_256_6` binding now supplies Flint's fixed `max(sum_selected, 2^-14)` rule. `RouterKernels::enqueue_router` and `enqueue_router_rows` @@ -108,10 +106,9 @@ Its nonnegative constant-weight case derives a scalar floating-operation bound; it is not a whole-model accuracy pin. The hardware case has not been run here. Existing program-specific admission remains unchanged: in particular the legacy -Qwen projection/shared dispatch does not gain this site automatically. Flint -still has no Program, attention/router binding, body or serve seat. Run the new -consumer gate and the affected regressions on NVIDIA before claiming device -support; reconcile full-model coverage only with the actual body integration. +Qwen projection/shared dispatch does not gain this site automatically. Flint's implemented +Program binds the real ordinary dense/shared operations. Run the new consumer gate and affected +regressions on NVIDIA before claiming device support; header coverage is not arithmetic proof. ### Selected expert arms and tier scratch Common card and tier wrappers resolve `LegArms` from their selected `Program`, rather than assuming @@ -133,18 +130,22 @@ family's activation contract. `LegTier` receives its activation converter at loa selected `Program` arms replace the redundant family arm hook. Qwen, Muse, MiMo and MiniMax sizing callers propagate checked tier descriptors rather than bypassing their validation. Share-budget floors use the same selected reserve calculation. +`TierLegScratch::input_scales` names an optional physical f32 scale per chunk +slot. Its checked byte formula includes `4 * chunk * n_used` when enabled, +including an empty selected-down union; legacy descriptors leave it disabled. +The same descriptor prices the loader's scale allocation without reserve padding. A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an -exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. +exact-head NVIDIA run. The integration's body/app/seat now consume these selected forms. ### Six-slot F32 tier handoff implementation `HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry beside the existing four- and eight-slot entries. Both host and tier layouts must have the same supported slot count and activation width; compressed tier images remain refused. The entry uses the existing copy/place/fault helpers and the existing -subsequent go/publication protocol; no Flint router or Program is registered. +subsequent go/publication protocol; the integration's configured routed block consumes it. The pure layout contract passed locally after an oxide build. `hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes`, owned by @@ -165,6 +166,37 @@ the new entry in the four existing architecture/capability tables. Offline no-JIT scans cannot supply that missing column, so no placeholder row is added. +### Integrated runtime consumer and NVIDIA handoff + +`crates/gpu-flint` loads the validated `ModelSpec`, actual selected projection/expert forms and +host/NVMe source through the common hosted loader. It implements the common chain, tier, +rollback and resident-slot interfaces. `app::flint` and `bloomery-serve --model flint` use the +existing hybrid/tiered seats, tokenizer, original chat template and stop handling. + +Head128 attention keeps NoPE and NeoX independent of the readable window, stores f16 KV, and +uses `Rows::Ring.count` for capacity and checked chunking. For the released model, per-slot KV is +`2048 * (14*C + 35*min(C,1024))` bytes. Slot state retains high-water positions: at a filled +1024-row ring a 512-token cut is retained, whereas 513 tokens require reset/refeed. + +The routed/shared block loads `(routed + shared) + residual` and the reference's activated-input +protection: `exp(ceil(log(amax / 1024) * log2(e)).clamp(0,127) * log(2))`. The down output is +restored before route weighting; this is not an activation clamp. Legacy models keep their +previous residual association and direct expert inputs without scale buffers or added launches. +Prompt and tier reservations include the actual optional scales and selected buffer unions. + +Run the owning model-free gates `gate-gpu-flint-attn` and `gate-gpu-flint-router`, then the +full-file `gate-gpu-flint-lib` with the complete GGUF passed through `BLOOMERY_BOX_ENV`. +Its ignored hardware case compares eager/captured logits, interleaved resident slots, +reset/refeed and retained/refused ring-boundary transitions. The test must be executed, not +counted as a pass because it compiles or remains ignored. + +NVIDIA acceptance also requires the affected shared expert/residency/tier regressions, exact-head +oracle comparisons and the existing per-architecture spill/JIT/resource checks. Ten new entries +currently lack genuine measurements in the four pinned architecture/capability tables; no zero, +placeholder or copied hardware pin is supplied. Keep the integration Draft until that evidence +is produced. No ROCm implementation or AMD execution is implied by this CUDA attachment. + + ## 1. Sources and observed artifact @@ -188,7 +220,10 @@ alignment padding, at byte **5,470,528**. Bytes after that boundary were ignored `fb0cc4b520f927f0b679193397ee0c36ca252426e7881283c5aa6de3e44a3a2b`. The existing Python header reader and Rust `gguf::inventory_of` agree on **779 tensors** and **25,049,218,176 encoded tensor bytes**. Shape products total **39,785,287,456 parameters** (derived). -No full model was downloaded or executed. +That initial design inspection fetched only the prefix. A later local CPU oracle run used the +complete 25,054,688,704-byte GGUF (SHA-256 +`63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f`). +Mainline CPU completion is reference evidence, not execution of this CUDA body. The official repository says Apache-2.0; this exact GGUF header says `other` / `flint-preview`. The config/header shapes and the byte-identical 2,328-byte chat templates establish those contracts, @@ -290,13 +325,13 @@ requires bias and derives width from a shared-count convention. Invalid mask typ norms/sinks/shared matrices, an out-of-range block, impossible KV grouping, zero dimensions, nonfinite epsilon/base, overflow and malformed optional bias all need named refusals. -At this snapshot there is no Flint architecture reader or selected program. A reader-only landing -must keep `program_of=None` and the existing **`Need::Program`** refusal in [coverage][coverage]. -Do not claim runtime support by adding the name to an enum or relaxing a shape/type row. +The original reader-only landing retained `program_of=None` and `Need::Program` refusals. +The integration now has actual operation consumers and Program admission; adding a name or +loosening a type row alone would still not establish runtime support. ### Concrete common prerequisites -| ID | Current limitation | Required common change before an executable attach | +| ID | Original gap (design snapshot above) | Required common change / acceptance contract | |---|---|---| | G1 | NoPE normalization-need generation is present; the executable head128 norm-only binding is missing | Preserve the emitted normalization **or** rotation need; bind head128 normalization with `mode=None` truthfully. No identity-RoPE table. | | G2 | Existing head128/group8 attention lacks the complete sink/SWA contract; MiMo's window pass is K192/V128 | Complete the described head128 operation in `rope_neox` / `flash_gqa`, including genuine NoPE append, sink merge, window/ring positions, prompt and slot paths. Never pad heads to 192 or copy MiMo's body. | diff --git a/justfile b/justfile index bfc7ec436..b74c2e08d 100644 --- a/justfile +++ b/justfile @@ -32,7 +32,7 @@ default: # 빠른 루프: 타입 검사만, 커널은 안 만든다. 의존을 고친 뒤 cargo가 박스 쪽 Cargo.lock을 고쳐 쓰면 lock-back.sh가 # 그것을 이 트리로 가져온다 — box.sh는 한 방향으로만 싣는다. check: - ./tools/box.sh 'cargo check --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' + ./tools/box.sh 'cargo check --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/flint,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' ./tools/lock-back.sh # check와 같은 이유로 `--features gpu`. 이 피처를 켠 것이 기준 계기다(R26). V4.1 op 게이트의 피처 @@ -40,14 +40,14 @@ check: # V4.1 커널의 컴파일러 결함은 여기가 아니라 op 게이트 빌드에서 드러난다. # lint. 에러 0이 계약이고 경고 수는 RESULTS/AGENTS에 적힌 기준선과 비교한다. lint: - ./tools/box.sh 'cargo clippy --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' + ./tools/box.sh 'cargo clippy --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/flint,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' # The build without `vision` of the bins that refuse an image request by name — serve_seats/ds41.rs:444 and glm.rs:825 (--mmproj), # decide.rs:545 (the Qwen3.5 seat's encoder), gate_glm5next_e2e.rs:5146 (--only media-seat). One bin takes one feature set (tools/recipes.py's # bin check), and every recipe that builds these bins builds them with `vision`, so only this check compiles the # `cfg(not(feature = "vision"))` branches. A cargo check writes no executable, so it replaces no bin another lane runs. mac-combos runs this shape. check-no-vision: - ./tools/box.sh 'cargo check -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41 --bin gate_glm5next_e2e' + ./tools/box.sh 'cargo check -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41 --bin gate_glm5next_e2e' # fmt는 맥에서 돈다. box.sh의 rsync가 단방향이라 박스에서 포맷하면 결과가 돌아오지 # 않고 다음 명령에 덮여 사라진다(2026-09-19에 그렇게 한 번 날렸다). cargo fmt는 컴파일을 @@ -387,7 +387,7 @@ gate-gpu-musespark-prefill: [group('solo')] [group('v41-load')] gate-gpu-musespark-twocard: - BLOOMERY_MODEL=muse-spark BLOOMERY_CARD=both ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_musespark_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_musespark_prefill --two-card && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_musespark_serve --dir $D' + BLOOMERY_MODEL=muse-spark BLOOMERY_CARD=both ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_musespark_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_musespark_prefill --two-card && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_musespark_serve --dir $D' # Muse Spark Open's load gate (gate_musespark_load's doc): the body's placed load of the gate plan proven against its plan — # every expert in one place and the stack bytes the file's, each segment on the card resident at the plan's buffers and nothing @@ -406,7 +406,7 @@ gate-gpu-musespark-load: [group('solo-real')] [group('v41-load')] gate-gpu-musespark-serve: - BLOOMERY_MODEL=muse-spark ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_musespark_serve --dir $D' + BLOOMERY_MODEL=muse-spark ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_musespark_serve --dir $D' # Solar-Open2-250B end-to-end gate (gate_solar2_e2e's doc): the 36 KDA layers (beta = 2 sigmoid, softplus decay) and 12 NoPE gated GQA @@ -459,7 +459,7 @@ gate-gpu-solar2-twocard: [group('solo-real')] [group('v41-load')] gate-gpu-solar2-serve: - BLOOMERY_MODEL=solar-open2 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_solar2_serve && D=target/solar2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=600 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_solar2_serve --dir $D' + BLOOMERY_MODEL=solar-open2 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_solar2_serve && D=target/solar2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=600 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_solar2_serve --dir $D' # The nemotron seat of bloomery-serve on Nemotron 3 Nano 4B Q4_K_M (gate_nemotron_serve's doc): `--plan` with no flag names its # default context (the ladder's rung, held by the ctx line and the rung above refused) and exits before any load, `--parallel 2` is @@ -468,14 +468,14 @@ gate-gpu-solar2-serve: # server's template renders), serves that turn on /v1/chat/completions (usage counts its ids), resends an edited prompt as a # fresh run that keeps nothing and an extended one that keeps the held positions. One card. gate-gpu-nemotron-serve: - BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' # The same seat gate on the moe body (gate_nemotron_serve's doc, every clause one path over both # bodies): the Bolt fixture under the fixture tier (nemotron_h_moe, the fixture's four layers, any # card), the arch word and the layer count read from the file, the session's prompt call the body's # batched `GpuModel::prefill` (several ids; one id is a step). One card. gate-gpu-nemotron-h-moe-serve: - BLOOMERY_TIER=fixture BLOOMERY_MODEL=nemotron_h_moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-h-moe-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + BLOOMERY_TIER=fixture BLOOMERY_MODEL=nemotron_h_moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-h-moe-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' # GLM-5.3-Flash 라우터(288개 중 8개, sigmoid, ×2.5 — `router::glm5next`)를 모델 없이 합성 입력으로 호스트 규칙과 맞춘다. # 규칙마다 절이 하나다: logits·ids·weights·batch는 비트, scores는 sigmoid 밴드, ties·guard·fault·graph는 명시한 값. @@ -1428,7 +1428,7 @@ gate-gpu-qwen35moe-mtp: # target). The card is the A6000: the DSpark draft's load holds the whole target on it, and Qwen3.6's # 29 GB does not fit the 24 GB card (the seat then refuses the draft by name). gate-gpu-qwen3-dspark: - BLOOMERY_MODEL=qwen35moe BLOOMERY_CARD=a6000 ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen3-dspark && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_dspark && D=target/qwen3-dspark-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && MINI=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/minicpmv47.sh) && test -n "$MINI" && Q30=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen3moe.sh) && test -n "$Q30" && bash tools/gpu-gate.sh gate_qwen3_dspark --qwen "$Q36" --qwen-draft /models/Qwen3.6-35B-A3B/dspark-redhat --sage /models/Sage-A1.5/Sage-A1.5-Q4_K_M.gguf --ornith /models/Ornith-1.5-35B-A3B/Ornith-1.5-35B-Q4_K_M.gguf --minicpm "$MINI" --minicpm-draft /models/MiniCPM-V-4.7-35B-A3B/dspark --qwen3moe "$Q30" --dir $D' + BLOOMERY_MODEL=qwen35moe BLOOMERY_CARD=a6000 ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen3-dspark && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_dspark && D=target/qwen3-dspark-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && MINI=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/minicpmv47.sh) && test -n "$MINI" && Q30=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen3moe.sh) && test -n "$Q30" && bash tools/gpu-gate.sh gate_qwen3_dspark --qwen "$Q36" --qwen-draft /models/Qwen3.6-35B-A3B/dspark-redhat --sage /models/Sage-A1.5/Sage-A1.5-Q4_K_M.gguf --ornith /models/Ornith-1.5-35B-A3B/Ornith-1.5-35B-Q4_K_M.gguf --minicpm "$MINI" --minicpm-draft /models/MiniCPM-V-4.7-35B-A3B/dspark --qwen3moe "$Q30" --dir $D' # Qwen3.6-35B-A3B's DSpark draft, RedHatAI's speculator (an HF directory on the box, `--draft DIR` names another): the dense layers on the card against a host f64 computation of the same layers from the file's bf16 # weights (a block of 8 rows over 3 and over 2,100 context rows, the window 2,048), the captured pass replaying to the @@ -1502,7 +1502,7 @@ gate-gpu-qwen35moe-media: [group('solo')] [group('v41-load')] gate-gpu-qwen38-media: - BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen38-media && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen38_media && D=target/qwen38-media-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=2400 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen38_media --dir $D' + BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen38-media && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen38_media && D=target/qwen38-media-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=2400 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen38_media --dir $D' # The decide seat of bloomery-serve (gate_clef_serve's doc): Clef-Flash at bartowski's Q5_K_M (6.9 GB, the # `qwen35` file of rev d7f376ea) with the release's head; the six refusals (a qwen35 file with no head, --model @@ -1520,7 +1520,7 @@ gate-gpu-qwen38-media: # adds the clauses that start the seat from the hub with no projector flag (it fetches the repo's own projector and # serves images) and with --no-mmproj (it serves none): a network read. Either card. gate-gpu-clef-serve *ARGS: - BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_clef_serve && D=target/clef-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_clef_serve --model /models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --clef-model /root/models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --image-model /models/clef-flash/Cloudflare_clef-flash-Q8_0.gguf --mmproj /root/models/clef-flash/mmproj-Cloudflare_clef-flash-bf16.gguf --dir $D {{ARGS}}' + BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_clef_serve && D=target/clef-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_clef_serve --model /models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --clef-model /root/models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --image-model /models/clef-flash/Cloudflare_clef-flash-Q8_0.gguf --mmproj /root/models/clef-flash/mmproj-Cloudflare_clef-flash-bf16.gguf --dir $D {{ARGS}}' # The decide seat of bloomery-serve on lev (gate_lev_serve's doc): ggml-org/lev-GGUF's Q4_K_M (3.0 GB, a `qwen35` # file with `qwen35.decision.type = lev`, fetched into /root/models/lev-4b/) with no head flag; five refusals before @@ -1530,7 +1530,7 @@ gate-gpu-clef-serve *ARGS: # prompts' ids; ARGS `--hf ggml-org/lev-GGUF:Q4_K_M` adds the clause that serves the file from the hub, no head flag, and # answers the first request as the `-m` server does (a network read). Either card. gate-gpu-lev-serve *ARGS: - BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_lev_serve && D=target/lev-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_lev_serve --model /root/models/lev-4b/lev-Q4_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --ref $BLOOMERY_DATA/lev/4b/ref --dir $D {{ARGS}}' + BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_lev_serve && D=target/lev-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_lev_serve --model /root/models/lev-4b/lev-Q4_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --ref $BLOOMERY_DATA/lev/4b/ref --dir $D {{ARGS}}' # clef_hidden on the box (ARGS as its doc: --model, --ids, --out, ...): a qwen35 file's prompt-only pass, every # position's final-norm hidden state to a file and a `clef hidden` record with the call's functional wall. @@ -1668,7 +1668,7 @@ gate-gpu-minimaxm3-prefill: [group('solo-real')] [group('v41-load')] gate-gpu-minimaxm3-serve: - BLOOMERY_MODEL=minimax-m3 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' + BLOOMERY_MODEL=minimax-m3 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' # MiniMax-M3.1-Flash on plan (b′) (gate_minimaxm3_prefill's doc, its `--arm bp`): the stage on the A6000 and the expert tier on the 3090, # both planned under one card budget the gate sets, against the same plan with every tier segment on the stage card (the A6000 alone, no @@ -1681,7 +1681,7 @@ gate-gpu-minimaxm3-serve: [group('solo')] [group('v41-load')] gate-gpu-minimaxm3-twocard: - BLOOMERY_MODEL=minimax-m3 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-minimaxm3-twocard && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features minimaxm3,checkpoint --release --bin gate_minimaxm3_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_minimaxm3_prefill --arm bp && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' + BLOOMERY_MODEL=minimax-m3 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-minimaxm3-twocard && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features minimaxm3,checkpoint --release --bin gate_minimaxm3_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_minimaxm3_prefill --arm bp && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' # Nemotron-H MoE (Lightning) with its 32-value experts on the card (Relu2Leg for the step, the 32-value GEMM for the # prompt): host, card (the 3090's room, split) and whole (the A6000's room, box.sh's pick) arms near the host path [derived: 8..12 min, @@ -1706,7 +1706,7 @@ gate-gpu-gemma-embedding-head256: # greedy tokens with embedding calls in flight, and named refusals; the runner holds one card. [group('pool')] gate-gpu-gemma-embedding2-serve *ARGS: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/gate.sh -p bloomery-gpu-gates --features hf --test sentence_prompts && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding2-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D" {{ARGS}} && if [ -z "{{ARGS}}" ]; then BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D/audio" --audio; fi' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/gate.sh -p bloomery-gpu-gates --features hf --test sentence_prompts && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding2-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D" {{ARGS}} && if [ -z "{{ARGS}}" ]; then BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D/audio" --audio; fi' # EmbeddingGemma 2's encoder (gpu-gemma-embedding2) end to end on the Q8_0 file the sets are dumped from # (refset's gemma_embedding2::MODEL, held to its sha256; the sets' `# model` line names the same path): every @@ -1718,7 +1718,7 @@ gate-gpu-gemma-embedding-e2e: BLOOMERY_MODEL=gemma-embedding ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e --v1' gate-gpu-gemma-embedding-serve: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --v1 --dir "$D"' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --v1 --dir "$D"' gate-gpu-gemma-embedding2-e2e *ARGS: ./tools/box.sh 'bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-gemma-embedding2 --lib -- --include-ignored' @@ -1764,10 +1764,14 @@ gate-gpu-mimo2-flash *ARGS: gate-gpu-flint-attn: ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_flint_attn && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_flint_attn' -# Flint's 256-expert top-6 sigmoid router. Model-free: one token, scale 1, zero bias. +# Flint's floor-normalized top-six router: optional selection bias, rows and captured replay. gate-gpu-flint-router: ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_flint_router && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_flint_router' +# Full-file eager/captured, resident-slot and ring-retention consumer transitions. +gate-gpu-flint-lib: + ./tools/box.sh 'bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-flint --lib -- --include-ignored' + # Muse Spark's 256-expert top-8 sigmoid router (`sigmoid_256_8`). Model-free like the Flint one, # plus the bias clause: the selection follows the biased scores, the weights the unbiased ones. gate-gpu-muse-router: @@ -2377,7 +2381,7 @@ gate-q38-serve-rules: # gate lock. [group('host')] gate-serve-draft-rules: - ./tools/box.sh 'bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41' + ./tools/box.sh 'bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41' # The V4.1 binaries' `--place` word (`generate::Place`, behind the deepseek41 feature like bind): each alias and its card # list (`a6000+3090` is `bp`) planning the machine its plan function makes, a list word planning its own cards, and every @@ -2869,7 +2873,7 @@ gate-gpu-qwen38-serve: # _decode_*): the qwen3moe file's --mmproj refusal and, on the Qwen3.6 file, the seat with its encoder. [group('pool')] gate-gpu-qwen3-serve: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/qwen3-serve-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && Q38D=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen38d.sh) && test -n "$Q38D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen3_serve --model "$BLOOMERY_REF_MODEL" --model "$Q36" --model "$Q38D" --dir $D --images' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/qwen3-serve-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && Q38D=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen38d.sh) && test -n "$Q38D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen3_serve --model "$BLOOMERY_REF_MODEL" --model "$Q36" --model "$Q38D" --dir $D --images' # bloomery-serve-qwen38 on the box, for a person to attach a client to (toktape records from it): the A6000 by # default (SERVE_PLACE=gate for the 3090), the port 8080 unless SERVE_PORT, a four-hour gate bound unless @@ -2988,7 +2992,7 @@ depth-gpu-glm5next *ARMS: # BLOOMERY_BOX_ENV=BLOOMERY_DRY=1 nothing is built and no lease is taken: the plan and each binary's state. depth-serve *ARMS: @[ -n "${BLOOMERY_MODEL:-}" ] || { echo "depth-serve: BLOOMERY_MODEL is unset: name the model profile (tools/ref/models/.sh), e.g. BLOOMERY_MODEL=mimo2 just depth-serve :" >&2; exit 64; } - ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && { here=$(basename "$PWD"); ours=; for a in {{ARMS}}; do t=${a#env=*:}; [ "${t%%:*}" != "$here" ] || ours=1; done; if [ -n "$ours" ] && [ -z "${BLOOMERY_DRY:-}" ]; then cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve; fi; } && bash tools/ref/depth-serve.sh {{ARMS}}' + ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && { here=$(basename "$PWD"); ours=; for a in {{ARMS}}; do t=${a#env=*:}; [ "${t%%:*}" != "$here" ] || ours=1; done; if [ -n "$ours" ] && [ -z "${BLOOMERY_DRY:-}" ]; then cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve; fi; } && bash tools/ref/depth-serve.sh {{ARMS}}' # The schemajump against Clef-Flash sitting: the Clef decide seat and a Qwen3.6 seat, both bloomery-serve, one lease on # the A6000, the arms alternating request by request (the design's §4; the cards are docs/cards/sj-*.card). Two windows @@ -3009,7 +3013,7 @@ depth-serve *ARMS: # bloomery-serve-ds41 picks, with SJ_DS41_MODEL; this recipe builds that binary when the windows name it) and SJ_SUITES # other suite files (`tools/ref/clef/suite-hard.jsonl`, scored against its `.gold.jsonl`); the header of the runner has both. sj-vs-clef: - ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && if [ -z "${BLOOMERY_DRY:-}" ]; then case " $(printf %s "${SJ_WINDOWS:-qwen:on qwen:off}" | tr , " ") " in *" qwen:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve;; esac && case " $(printf %s "${SJ_WINDOWS:-}" | tr , " ") " in *" ds41:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features deepseek41,vision,gemma-embedding2,hf --release --bin bloomery-serve-ds41;; esac; fi && bash tools/ref/sj-vs-clef.sh' + ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && if [ -z "${BLOOMERY_DRY:-}" ]; then case " $(printf %s "${SJ_WINDOWS:-qwen:on qwen:off}" | tr , " ") " in *" qwen:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve;; esac && case " $(printf %s "${SJ_WINDOWS:-}" | tr , " ") " in *" ds41:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features deepseek41,vision,gemma-embedding2,hf --release --bin bloomery-serve-ds41;; esac; fi && bash tools/ref/sj-vs-clef.sh' # Qwen3-30B-A3B decode-step kernel timeline (nsys, A6000, under the lease, lead-only): generate_qwen3moe's seed form at each # depth (default 6 4096), one prefill pass + BLOOMERY_NSYS_N - 1 replays, the cache height depth-qwen3moe.sh uses. The @@ -3572,7 +3576,7 @@ gate-gpu-glm5next-residency: [group('solo')] [group('v41-load')] gate-gpu-glm5next-serve: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-glm5next-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,hf,glm5next-hf --release --bin generate_glm5next && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-serve-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm plain --dir $D/plain && bash tools/gpu-gate.sh gate_glm5next_serve --arm drafted --dir $D/drafted' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-glm5next-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,hf,glm5next-hf --release --bin generate_glm5next && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-serve-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm plain --dir $D/plain && bash tools/gpu-gate.sh gate_glm5next_serve --arm drafted --dir $D/drafted' # The GLM seat's image input beside its draft (gate_glm5next_serve --arm media): one server of --parallel 2 under # BLOOMERY_DRAFT=mtp BLOOMERY_RESIDENCY=off with --mmproj, the GLM-5.3 projector, then an image chat on one slot and a @@ -3587,7 +3591,7 @@ gate-gpu-glm5next-serve: [group('solo')] [group('v41-load')] weekly-gpu-glm5next-media-serve: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-glm5next-media-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-media-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm media --dir $D --mmproj /models/mmproj/unsloth--GLM-5.3-Flash-GGUF/mmproj-BF16.gguf' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-glm5next-media-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-media-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm media --dir $D --mmproj /models/mmproj/unsloth--GLM-5.3-Flash-GGUF/mmproj-BF16.gguf' # The shipped `bloomery-serve` started once per image flag on the GLM-5.3 fixture (gate_serve_start's header has the clauses): (s1) # `-m ` with the serving defaults and one text chat; (s2) `--place a --mmproj `, the tower's @@ -3605,7 +3609,7 @@ weekly-gpu-glm5next-media-serve: # 8.6 GB set and the 1.2 GB projector in threads beside the first two starts: 53 to 98 s warm; the bound is the runner's 900 s. [group('solo')] gate-gpu-serve-start-glm: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-glm && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/glm && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family glm --dir $D' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-glm && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/glm && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family glm --dir $D' # The same gate on the V4.1 fixture (the file's architecture picks the seat, which has no `--plan`, so its plan is read from each # start's own log). The projector is the `# mmproj` line of the V4.1 vision set ($BLOOMERY_DATA/ref-vision/deepseek41v, `just dump-ref-vision`), @@ -3616,7 +3620,7 @@ gate-gpu-serve-start-glm: # beside the first two starts, a warm build: 72 to 132 s warm; the bound is the runner's 900 s. [group('solo')] gate-gpu-serve-start-ds41: - BLOOMERY_MODEL=deepseek41 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-ds41 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/ds41 && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family ds41 --dir $D' + BLOOMERY_MODEL=deepseek41 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-ds41 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/ds41 && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family ds41 --dir $D' # The MiMo seat of `bloomery-serve` (gate_mimo2_serve): the refusals before any load (--plan's default context, the one # slot, the tier card), one server at --ctx 4160 --parallel 1 holding its load and listening records, the three prompts' @@ -3633,7 +3637,7 @@ gate-gpu-serve-start-ds41: [group('solo')] [group('v41-load')] weekly-gpu-mimo2-serve: - BLOOMERY_MODEL=mimo2 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-mimo2-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_mimo2_serve && D=target/mimo2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1680 bash tools/gpu-gate.sh gate_mimo2_serve --dir $D' + BLOOMERY_MODEL=mimo2 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-mimo2-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_mimo2_serve && D=target/mimo2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1680 bash tools/gpu-gate.sh gate_mimo2_serve --dir $D' # glm5next decode CLI, functional run (no timing): generate_glm5next feeds --tokens one step per id, then greedy -n # tokens. The gate placement on the 3090 unless --place a (and BLOOMERY_CARD=a6000). 3090, gate lock. diff --git a/tools/families.tsv b/tools/families.tsv index ef853b823..e8eb4ec92 100644 --- a/tools/families.tsv +++ b/tools/families.tsv @@ -10,7 +10,7 @@ # fixture yes (the family has a fixture file, tools/fixture-dirs.tsv) or no deepseek2 F crates/model/src/arch/deepseek2/** crates/gpu/src/arch/deepseek2/** 7 no deepseek41 R crates/gpu-deepseek41/** crates/model/src/arch/deepseek41/** crates/refset/src/arch/deepseek41/** crates/refset/src/arch/deepseek41v/** crates/vision/src/arch/deepseek41v/** crates/gpu-gates/src/bin/gate_deepseek41_*.rs crates/gpu-gates/src/bin/gate_ds41_*.rs crates/gpu-gates/src/bin/gate_dspark_*.rs crates/gpu-gates/src/bin/shared/ds41_*.rs crates/gpu-gates/src/bin/shared/dspark_hf.rs crates/gpu-gates/src/bin/shared/serve_seats/ds41.rs crates/gpu-gates/src/bin/shared/routed_legs/deepseek41.rs 7 yes -flint F crates/model/src/arch/flint/** crates/model/tests/flint_meta.rs crates/gpu/src/flash_gqa_h128_swa.rs crates/gpu/src/qk_rms.rs crates/gpu-gates/src/bin/gate_flint_attn.rs crates/gpu-gates/src/bin/gate_flint_router.rs 7 no +flint F crates/model/src/arch/flint/** crates/model/tests/flint_meta.rs crates/gpu-flint/** crates/app/src/arch/flint.rs crates/gpu-gates/src/bin/shared/serve_seats/flint.rs crates/gpu/src/attention_h128.rs crates/gpu/src/attention_h128_ring.rs crates/gpu/src/flash_gqa_h128_swa.rs crates/gpu/src/qk_rms.rs crates/gpu-gates/src/bin/gate_flint_attn.rs crates/gpu-gates/src/bin/gate_flint_router.rs 7 no gemma-embedding F crates/model/src/arch/gemma_embedding/** crates/refset/src/arch/gemma_embedding/** crates/gpu-gates/src/bin/gate_gemma_embedding_*.rs 7 no gemma-embedding2 F crates/gpu-gemma-embedding2/** crates/model/src/arch/gemma_embedding2/** crates/refset/src/arch/gemma_embedding2/** crates/gpu-gates/src/bin/gate_gemma_embedding2_*.rs 7 no gemma4a F crates/gpu-gemma4a/** crates/vision/src/arch/gemma4a.rs crates/gpu-gates/src/bin/gate_gemma4a.rs 7 no diff --git a/tools/gate-class.tsv b/tools/gate-class.tsv index 51f3e8d67..6bf0f536b 100644 --- a/tools/gate-class.tsv +++ b/tools/gate-class.tsv @@ -75,6 +75,7 @@ gate-gpu-dspark-kv family deepseek41 draft drafts_dspark gate-gpu-e2e family deepseek2 e2e graph_fault+moe_route+weights_load gate-gpu-flint-attn family flint kernel attn_gqa gate-gpu-flint-router family flint kernel moe_route +gate-gpu-flint-lib family flint e2e graph_fault+attn_gqa+moe_route+tier_legs+placement_sizing gate-gpu-muse-router common kernel moe_route gate-gpu-gates-lib tooling harness - gate-gpu-gemm common kernel quant_gemm diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 20521145e..9366aa37a 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -113,7 +113,7 @@ tools/census/stubs.tsv derive the census manifest (refset::census::MANIFEST name # arch/coverage.rs and arch/qwen35moe/mtp.rs are carved out to their own rows below. crates/model/src/arch/{mod.rs,registry.rs,dspark.rs,classify.rs,stems.rs} derive gate-head gate-r8 gate-ptx-spill the dispatch's own files (Arch and spec, the registry, the shared architecture reads and the DSpark reader): every gate that links one, less the gguf-only kernel gates; the family modules behind the registry's rows are reached from them only by the gates that can open that family (tools/recipes.py, Reach) crates/model/src/arch/deepseek2/** derive gate-head gate-r8 the V2-Lite engine's reader and forward: the gates whose targets name it or call what it calls from another module (qwen35moe/mod.rs embeds through deepseek2::forward), and the gates that open a V2-Lite file (its profile, the names of their targets, the /models paths they name) -crates/model/src/arch/flint/** derive Flint's header-only reader and exact roles: its metadata gate and the common registry's architecture dispatch; no executable program or fixture +crates/model/src/arch/flint/** derive Flint's validated reader, placement and host expert adapter: metadata and the actual head128 body/app/serve consumers; no redistributed fixture crates/model/src/arch/qwen3moe/** derive the Qwen3 reader and plan: the gates whose targets name it, call it from another crate, or open a qwen3moe file (their profile, the names of their targets, the /models paths they name) crates/model/src/arch/glm5next/** derive the GLM reader and plan: the gates whose targets name it, call it from another crate (the GLM body's chains), or open a glm5next file (their profile, the names of their targets, the /models paths they name) crates/model/src/arch/mimo2/** derive weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve MiMo-V2.6-Flash's reader and plan: the gates that open a mimo2 file (its meta test, its flash gate, bloomery-serve's mimo2 seat), and the gates whose code calls it except gate-fixture: gate-fixture walks the registry's rows and runs no mimo2 reader on a file diff --git a/tools/hf-arch-check.py b/tools/hf-arch-check.py index 0cf29f393..af0111ddb 100755 --- a/tools/hf-arch-check.py +++ b/tools/hf-arch-check.py @@ -111,10 +111,10 @@ def load_by_path(name, *rel): "mimo2": ("mimo2",), } SERVES_VIA_FROM_NAME = {"glm": ("Glm5next",), "solar2": ("SolarOpen2",), "minimaxm3": ("MiniMaxM3",), "nemotron": ("NemotronH", "NemotronHMoe"), - "raon1": ("Raon1",), "musespark": ("MuseSpark",)} + "raon1": ("Raon1",), "musespark": ("MuseSpark",), "flint": ("Flint",)} # crates/gpu-gates/src/bin/bloomery_serve.rs:Model::ALL and Model::word — the generative seats in the order # the refusal of an unserved file lists them; crates/serve/src/decide.rs:WORD is the decide seat's word. -SEAT_ORDER = ("ds41", "qwen38", "glm", "qwen3", "mimo2", "solar2", "minimaxm3", "nemotron", "raon1", "musespark") +SEAT_ORDER = ("ds41", "qwen38", "glm", "qwen3", "mimo2", "solar2", "minimaxm3", "nemotron", "raon1", "musespark", "flint") DECIDE_WORD = "decide" # tools/arch-registry.tsv, the registry's print: FROM_NAME is crates/model/src/arch/mod.rs:Arch::from_name — variant -> the # strings that name it; SPEC is crates/model/src/arch/mod.rs:spec — the architectures the model reader takes (the @@ -1929,10 +1929,6 @@ def row_line(seat, arch, tensors=(), extra=(), vocab=0, quant="Q4_K_M", cards=No line, green, hub = row_line("glm", "glm5-next", small) check("a glm5-next file routes to the glm seat", green and "arch=glm5-next route=ok" in line, line) check("a tensor-type line parses", "embd=Q3_K exps=gate:Q4_K,down:Q6_K " in line, line) - line, green, _ = row_line("glm", "glm5nextx", small) - check("a glm5nextx file is refused by name", - not green and "arch=glm5nextx route=REFUSED: a glm5nextx file, which no seat serves; the seats are " - "--model ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, decide" in line, line) line, green, _ = row_line("glm", "deepseek41", small) check("a file the router seats elsewhere is refused for the row's seat", not green and "in the ds41 seat, and the README row says glm" in line, line) diff --git a/tools/levers-direct.txt b/tools/levers-direct.txt index c99b8511d..2c85c03aa 100644 --- a/tools/levers-direct.txt +++ b/tools/levers-direct.txt @@ -62,6 +62,7 @@ crates/model/tests/deepseek4_meta.rs BLOOMERY_REF_MODEL stays: the model file th crates/model/tests/ds41_meta.rs BLOOMERY_DSPARK_MODEL stays: the DSpark draft file crates/model-hf/tests/fixture.rs BLOOMERY_DSPARK_MODEL stays: the DSpark draft file crates/model/tests/flint_meta.rs BLOOMERY_MODEL stays: the pinned model file or header prefix the harness opens +crates/gpu-flint/src/tests.rs BLOOMERY_MODEL stays: cfg(test) full-file hardware consumer opens this exact local model path, not a production serving lever crates/model/tests/gemma_embedding_meta.rs BLOOMERY_MODEL stays: the model file the harness opens crates/model/tests/gemma_embedding2_meta.rs BLOOMERY_MODEL stays: the model file the harness opens crates/model/tests/mt.rs BLOOMERY_MT_CHILD_DUMP stays: the gate's handshake with the child it runs diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index 298e4b9c2..d46c3ed28 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -81,7 +81,7 @@ crates/gpu-gates/src/bin/shared/dspark_hf.rs qwen 1 baseline O 1: own path: an a crates/gpu-gates/src/bin/shared/emb2_image.rs gemma 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu-gates/src/bin/shared/route_trace38.rs qwen 3 baseline O 3: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu-gates/src/bin/shared/routed_legs.rs deepseek 8 baseline P 9: MK2 program by body -crates/gpu-gates/src/bin/shared/routed_legs.rs flint 2 PIN(2026-10-23) flint: family_of and skip_of match every models::Arch exhaustively; the header-only reader has no program or routed-legs adapter +crates/gpu-gates/src/bin/shared/routed_legs.rs flint 2 PIN(2026-10-25) flint: family_of and skip_of match every models::Arch exhaustively; no common routed-legs adapter is registered, and the full-file body transitions have their own owning gate crates/gpu-gates/src/bin/shared/routed_legs.rs gemma 4 baseline P 5: MK2 program by body crates/gpu-gates/src/bin/shared/routed_legs.rs glm 7 baseline P 8: MK2 program by body crates/gpu-gates/src/bin/shared/routed_legs.rs mimo 4 baseline P 5: MK2 program by body @@ -130,7 +130,7 @@ crates/gpu-vision/src/encoder.rs deepseek 2 baseline D 2: media design: VideoRul crates/gpu/src/arch/hybrid.rs nemotron 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu/src/card_slots.rs deepseek 24 PIN(2026-10-21) deepseek: baseline K 20 + 4, the four-slot entry `ds41_ffn_card_acc_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it crates/gpu/src/dspark/head.rs deepseek 3 baseline K 3: rename the kernel entries by what they compute (deferred) -crates/gpu/src/host/handoff.rs deepseek 38 PIN(2026-10-24) deepseek: baseline K 31 + 4 + 3, the four-slot tier entry `ds41_ffn_handoff_4_tier` (MiniMax-M3.1's bp) beside the four-slot entry `ds41_ffn_handoff_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it +crates/gpu/src/host/handoff.rs deepseek 39 PIN(2026-10-25) deepseek: fixed-slot tier preparation and launch select the existing generated ABI names beside the new six-slot entry; the per-entry module methods retain their pinned names until the deferred common-entry rename crates/hf/src/vocab.rs qwen 4 PIN(2026-10-20) qwen: the qwen35 tokenizer pipeline name and regex; tokenizer programs are not yet described by ModelSpec crates/metal/src/bin/bloomery_serve_metal.rs qwen 3 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string crates/metal/src/bin/generate_metal.rs qwen 4 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string @@ -163,7 +163,7 @@ crates/models/src/lib.rs mimo 2 baseline A 2: MK1a/MK1b ArchRow registry crates/models/src/lib.rs nemotron 4 baseline A 4: MK1a/MK1b ArchRow registry crates/models/src/lib.rs qwen 9 baseline A 9: MK1a/MK1b ArchRow registry crates/placement/src/accepts.rs deepseek 10 PIN(2026-10-21) deepseek: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet -crates/placement/src/accepts.rs flint 1 PIN(2026-10-23) flint: program_of matches every models::Arch; the header-only reader must explicitly return no program until a body is registered +crates/placement/src/accepts.rs flint 5 PIN(2026-10-25) flint: the implemented body registers its Program variant, name, architecture selection, expert arms and routed rule in the existing exhaustive program owners crates/placement/src/accepts.rs gemma 5 PIN(2026-10-21) gemma: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs glm 5 PIN(2026-10-21) glm: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs mimo 5 PIN(2026-10-21) mimo: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet @@ -216,3 +216,5 @@ crates/gpu-gates/src/record.rs muse 4 PIN(2026-10-23) muse: MK12 per-bin Kind st crates/model/src/arch/coverage.rs muse 25 PIN(2026-10-23) muse: the coverage rows, the type pins and the expert-stack match are keyed by Program, so each row the musespark body runs lists its variant; the program would own its rows instead of the list naming programs crates/hf/src/vocab.rs minimax 3 PIN(2026-10-23) minimax: the minimax-m2 pre-tokenizer's name, regex and pipeline row; tokenizer programs are not yet described by ModelSpec crates/gpu-gates/src/prompts.rs deepseek 1 PIN(2026-10-23) deepseek: the measured logit σ is named for the one family it was measured on (SIGMA_LOGIT_V41); other families carry it in their gate description until their own is measured +crates/gpu-gates/src/bin/bloomery_serve.rs flint 10 PIN(2026-10-25) flint: MK9 seat by program: the seat word, help/models list, architecture match and dispatch register the implemented shared hybrid seat until the description selects it +crates/model/src/arch/coverage.rs flint 11 PIN(2026-10-25) flint: the implemented Program selects its actual head128 attention, floor router, dense/shared sites, embedding/head and expert-stack capabilities in the existing coverage owners diff --git a/tools/recipes.py b/tools/recipes.py index 2ce417061..cfc84dce3 100755 --- a/tools/recipes.py +++ b/tools/recipes.py @@ -13031,8 +13031,6 @@ def _st_host_crates(fails: list[str], side: Side) -> None: ], f"host nightly: the real tree's set is {host_now} — a new member of the set is proven by a nightly run before this list takes it", ) - left = [n for n, _ in host_left_out(tree, lock_now)] - expect(left == ["bloomery-gpu", "bloomery-gpu-deepseek41", "bloomery-gpu-gates", "bloomery-gpu-gemma-embedding2", "bloomery-gpu-gemma4a", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-minimaxm3", "bloomery-gpu-solar2", "bloomery-gpu-vision"], f"host nightly: left out {left}") expect(not nightly_problems(tree, lock_now), f"host nightly: {nightly_problems(tree, lock_now)}") ran = {u for u, _ in host_units(tree, lock_now)} expect( diff --git a/tools/ref/dense-logits.py b/tools/ref/dense-logits.py index e3c185588..754416789 100644 --- a/tools/ref/dense-logits.py +++ b/tools/ref/dense-logits.py @@ -234,7 +234,7 @@ def native_build(args): env = os.environ.copy() env["CARGO_TARGET_DIR"] = str(ROOT / "target") argv = ["cargo", "oxide", "build", "--arch", args.arch, "--", "-p", "bloomery-gpu-gates", - "--features", "glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint", "--release", + "--features", "glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint", "--release", "--bin", "generate_qwen3moe", "--bin", "bloomery-serve"] subprocess.run(argv, cwd=ROOT, env=env, check=True, timeout=1800) if clean_head(ROOT) != head: diff --git a/tools/release/build.sh b/tools/release/build.sh index d48cd3277..0d999bf40 100755 --- a/tools/release/build.sh +++ b/tools/release/build.sh @@ -191,7 +191,7 @@ else bin_args=() for b in "${BINS[@]}"; do bin_args+=(--bin "$b"); done # The commit goes to cargo alone: the serve build script reads it, and a running binary never does. - BLOOMERY_GIT_COMMIT="$COMMIT" cargo oxide build --arch "$ARCH" -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release "${bin_args[@]}" + BLOOMERY_GIT_COMMIT="$COMMIT" cargo oxide build --arch "$ARCH" -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release "${bin_args[@]}" fi fail=0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index f699c71f3..a742fc370 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -554,7 +554,10 @@ gguf 7 2 0 # +2); 2331 + 83 + 2 + 2, recounted by tools/check-unsafe.sh. # PIN(2026-10-25): gpu 2418 -> 2420 blocks, the six-slot F32 tier entry adds two # handoff body calls under its checked launch contract; the newmain baseline is preserved. -gpu 2420 311 10 +# PIN(2026-10-24): gpu 2420 -> 2475 blocks, 311 -> 315 fns, new code: checked +# head128 ring append/attention, optional-bias floor router, expert-input scaling +# and common loaded launchers. Existing entries retain their contracts; each new site states SAFETY. +gpu 2475 315 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. @@ -577,7 +580,12 @@ gpu 2420 311 10 # PIN(2026-10-24): gpu-deepseek41 382 -> 384 blocks, new code, not old code relaxed: the four-slot tier card sum # (chain/ffn/batch.rs: ds41_ffn_card_acc_4_tier's card_acc_tier_at call and its store, 2 blocks), each block with its # SAFETY line. -gpu-deepseek41 384 44 0 +# PIN(2026-10-24): gpu-deepseek41 384 -> 386 blocks: load the configured +# residual module and launch its checked output rows, each with SAFETY. +gpu-deepseek41 386 44 0 +# PIN(2026-10-24): gpu-flint starts at 2 blocks, 0 fns and 0 impls: step and prompt +# call the checked common attention binding with staged distinct positions, each with SAFETY. +gpu-flint 2 0 0 # PIN(2026-10-01): gpu-gates 60 -> 61 blocks, new code, not old code relaxed: gate_glm5next_serve starts # bloomery-serve with PR_SET_PDEATHSIG set between fork and exec (one pre_exec block, its SAFETY line), as the # Qwen3.8 serve gate does, so a bound that ends the gate does not leave the server holding the card. @@ -643,7 +651,9 @@ gpu-deepseek41 384 44 0 # and the host flag's drop once the stream that waited on it is drained, each with its SAFETY line. # PIN(2026-10-24): gpu-gates 103 -> 104 blocks, new code, not old code relaxed: gate_qwen35moe_attn's launch of the # head-256 append at no query head over the rows rope_gate::run checks distinct, with its SAFETY line. -gpu-gates 104 5 1 +# PIN(2026-10-24): gpu-gates 104 -> 106 blocks: ring-attention consumers launch +# checked distinct positions and bounded chunks under their stated SAFETY invariants. +gpu-gates 106 5 1 # PIN(2026-10-15): gpu-gemma-embedding2 0 -> 1 blocks, new code, not old code relaxed: the head-512 # launcher call requires distinct Consts::pos positions, with its SAFETY line. # PIN(2026-10-15): gpu-gemma-embedding2 1 -> 2 blocks, new code, not old code relaxed: Encoder::pass's call of From f71e8895d4ec559c03a32229d93fd0d1f7419009 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:22:22 +0900 Subject: [PATCH 20/85] test(flint): align mixed expert fixtures with current model description --- crates/gpu/src/card_experts.rs | 2 ++ crates/gpu/src/site.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 5811138e5..3178bda9d 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -914,6 +914,8 @@ mod tests { top_k: USED as u32, expert_ff: FF as u32, latent: None, + pre_exp_norm: None, + post_exp_norm: None, act: models::Act::SwiGlu { limit: None }, router: Router { score: Score::Sigmoid, diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 7c5e5d320..b38551932 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -1567,8 +1567,8 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, file_site, gemm_weight, - kgemv_order, swiglu_launch, + Form, SiteTy, SwigluLaunch, WideActs, WideForms, file_site, gemm_weight, kgemv_order, + swiglu_launch, }; use crate::kquant::Act; use gguf::Split; From 7cb8f03639bd582355c5597a49372cb6e79183ae Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:27:29 +0900 Subject: [PATCH 21/85] test(flint): fix expert norm flag and record real-weight CPU consumers --- crates/gpu/src/card_experts.rs | 2 +- docs/research/flint-attach.md | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 3178bda9d..4fd16c23f 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -914,7 +914,7 @@ mod tests { top_k: USED as u32, expert_ff: FF as u32, latent: None, - pre_exp_norm: None, + pre_exp_norm: false, post_exp_norm: None, act: models::Act::SwiGlu { limit: None }, router: Router { diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index e76c7c082..1d4151727 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -196,8 +196,14 @@ currently lack genuine measurements in the four pinned architecture/capability t placeholder or copied hardware pin is supplied. Keep the integration Draft until that evidence is produced. No ROCm implementation or AMD execution is implied by this CUDA attachment. - - +Local CPU smoke on the complete artifact exercised all 47 routed layers with six fixed selected +experts, two columns and input amplitudes 1/8192. Single-column and union execution compared +385,024 f32 cells bit-for-bit (zero differences), with finite outputs and bounded protected +activations. This is actual expert-weight execution, not a whole-model decode or GPU oracle. +A separate public planner consumer checked group sizes 1/8, contexts 512/1024/2048 and slots 1/3; +all 12 plans conserved full/window KV bytes and repeated group configuration charged no second reserve. +The compiled CLI lists the Flint seat; opening it on this AMD-only machine stops at the named +NVIDIA R580+/`libcuda.so.1` prerequisite, so no HTTP generation run is reported. ## 1. Sources and observed artifact From 313b9f0f88c90fcfae2365a4af53cd077762eb7c Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:37:51 +0900 Subject: [PATCH 22/85] fix(flint): include shared HTTP seat in published integration --- .../src/bin/shared/serve_seats/flint.rs | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 crates/gpu-gates/src/bin/shared/serve_seats/flint.rs diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs b/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs new file mode 100644 index 000000000..e74fdfcf8 --- /dev/null +++ b/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs @@ -0,0 +1,88 @@ +//! Flint's actual slot-capable body behind the common hybrid HTTP seat. +//! Chat, tools, reasoning and end-of-generation tokens use the canonical model contract. + +use bloomery_gpu_gates::generate::Place; +use bloomery_gpu_gates::{GateError, record, side_seat}; +use gguf::Split; +use model::arch::Arch; +use model::arch::flint::place; +use model::arch::place_hybrid::PlanInputs; +use model::placement::PlacementError; +use model::placement::workstation::TierBatchBytes; +use serve::ServeError; + +use super::hybrid::Family; + +const NAME: &str = "bloomery-serve-flint"; +const USAGE: &str = "usage: bloomery-serve --model flint [-m PATH | --hf [:]] \ + [--host H] [--port P] [--ctx C] [--parallel N] [--alias NAME] \ + [--chat-template-file PATH] [--place W] [--plan] [--queue-depth Q] \ + [--slot-save-path DIR] [--api-key KEY] [--api-key-file FNAME]"; + +/// The common placement, host load, residency, prompt-group and caller settings. +pub const ACTS_ON: &[&str] = &[ + bloomery_levers::CARD_BUDGET, + bloomery_levers::RESIDENCY, + bloomery_levers::PREFILL_GROUP, + bloomery_levers::HOST_POPULATE, + bloomery_levers::HOST_LOCK, + bloomery_levers::CARD_DONTNEED, + bloomery_levers::R8, + bloomery_levers::PIN_MAIN, + bloomery_levers::HOST_LANES, + bloomery_levers::SCREEN, + bloomery_levers::SCHEMAJUMP, +]; + +/// Architecture-specific facts; the common owner plans and serves every slot. +pub struct Flint; + +impl Family for Flint { + type Body = bloomery_gpu_flint::Body; + type H = (); + + const NAME: &'static str = self::NAME; + const USAGE: &'static str = self::USAGE; + const ACTS_ON: &'static [&'static str] = self::ACTS_ON; + const ALIAS: &'static str = "flint"; + const FAMILY: &'static str = "flint"; + const SERVES: &'static str = "flint"; + + fn accepts(arch: Arch) -> bool { + matches!(arch, Arch::Flint) + } + + fn read(split: &Split) -> Result, PlacementError> { + place::PlanInputs::read(split) + } + + fn configure(inputs: &mut PlanInputs<()>, group: usize) -> Result<(), GateError> { + Ok(place::configure_group(inputs, group)?) + } + + fn register() { + record::at_main(NAME, record::BLOOMERY_SERVE_HYBRID); + } + + fn listening() -> record::Record { + record::Record::new(&record::LISTENING_HYBRID) + } + + fn load_record() -> record::Record { + record::Record::new(&record::LOAD_HYBRID) + } + + fn tier_batch( + place: Place, + inputs: &PlanInputs<()>, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(inputs)) + .transpose()?) + } +} + +/// Opens the real Flint session and serves resident sequences by select and step. +pub fn run(args: &[String], side: &side_seat::Flags) -> Result { + super::hybrid::run::(args, side) +} From ebce5badb6f375c74930ff26a4eb50e7e74ce97d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Mon, 26 Oct 2026 03:44:17 +0900 Subject: [PATCH 23/85] test(gpu): pin measured Flint cc8.6 resources Record exact-head RTX 3090 gates and partial HTTP/oracle evidence. Pin only ten genuinely measured sm_86/cc8.6 spill and JIT-local rows; retain thirty cc12.1 gaps and the Korean greedy mismatch. No numerical or gate changes. --- docs/research/flint-attach.md | 58 +++++++++++++++++++++++++++++++++++ tools/ref/ptx-shapes.tsv | 14 +++++++++ 2 files changed, 72 insertions(+) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 1d4151727..941b4c78e 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -595,6 +595,64 @@ The Python reader intentionally skips array values. Their recorded values came f `gguf::inventory_of(...).meta`, not `None` values treated as absent GGUF facts. The recipe above revalidates the **whole header hash**, including those arrays, without a second parser. +## 10. Single-card NVIDIA acceptance and remaining numerical boundary + +The runtime at `313b9f0f88c90fcfae2365a4af53cd077762eb7c` was exercised on one RTX 3090, +cc8.6, driver 595.91.07 (CUDA API 13.2), with nightly-2026-08-28, cuda-oxide `87bc9518`, +LLVM 21.1.8 and CUDA toolkit 13.3.0 / ptxas 13.3.33. The complete 25,054,688,704-byte +GGUF was downloaded separately on that runner and its full SHA-256 matched +`63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f`. + +- `just gate-gpu-flint-attn` and `just gate-gpu-flint-router`: exit 0, including loaded + NoPE/NeoX ring consumers, floor/optional-bias routing, capture/replay and named faults. +- `just gate-gpu-flint-lib`: exit 0; the full-file capture/slots/reset/ring-boundary test + ran with **1 passed, 0 failed, 0 ignored**. Its bitwise consistency is not a mainline oracle. +- The real HTTP seat loaded 49 layers and two resident slots, exposed the original template, + generated English and Korean turns to EOG 150003, split reasoning, parsed a required + `get_weather` call with `{"location":"Seoul"}`, and preserved messages under concurrent + requests. The four stop spellings tokenized to the four expected IDs; natural emission of + all four stop IDs was not tested. +- A full `generate_ds41` scan and the pre-pin spill gate JITed all 436 entries without a + resource cache. The only ten violations were the new unpinned entries, each measured + `spill=0, jit_local=0`; the 26 vision entries passed. These observations supply only the + ten `sm_86`/cc8.6 rows. The three cc12.1 tables still need thirty genuine measurements. + +### Independent greedy comparison remains partly red + +The English raw continuation exactly matched pinned mainline CPU +`a37cc46104eb50900d5ab4a9ba02a0abdac039db`. The Korean continuation did not: the first +different generated ID is index 74, after the same 22 prompt IDs and 74 generated IDs +(next absolute position 96). Mainline chooses 26007 (`라`); Bloomery chooses 29263 (` 정`). +Both eventually answer `4`, which does not turn the comparison green. + +A standalone consumer linked to the preserved mainline library replayed that original +prompt-then-single-token trajectory with zero preceding raw-argmax mismatches. Its complete +150,144-logit rows at context 1024 and 2048 were bit-identical. At the divergence: + +| Observation | Margin `logit(26007) - logit(29263)` | Selected ID | +|---|---:|---:| +| Mainline CPU, original trajectory | +0.608115554 | 26007 | +| Bloomery probability diagnostic, original trajectory | -0.018128157 | 29263 | +| Bloomery, same 96 IDs supplied as one prefill | +0.361473739 | 26007 | + +The GPU margins are differences of reported pre-sampling log probabilities; the diagnostic +reproduced the original first 75 generated IDs. Probability collection reads each row and +uses plain per-token stepping, rather than establishing equality of every execution mode. + +This is not an exact tie or a demonstrated context-capacity defect. The common FFN owner +explicitly uses F32-activation Q8_0 GEMV for narrow/decode rows and quantized-activation +GEMM for wide prompt units (`shared_routed.rs`, `GATED_FROM=9`). Mainline also quantizes +the output head's activations and BF16 router operands differently. These are source-backed +numerical differences, not a proven localization of the Korean flip. No precision mode, +compiler pin, tie rule, tolerance or acceptance criterion was changed to force matching text. +Comparable follow-up traces must retain the original prompt/decode trajectory and can use +the existing 49-layer `Body::taps` and mainline `l_out` callbacks. + +Keep the integration Draft. Korean numerical/oracle acceptance, cc12.1 resource rows, +affected common-owner coverage beyond the exercised consumers, two-card operation, dynamic +residency/NVMe, streamed HTTP and spontaneous tool selection are not established by this run. + + [needs]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/models/src/need.rs#L505-L531 [stems]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/model/src/arch/stems.rs#L61-L125 [coverage]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/model/src/arch/coverage.rs#L1654-L1770 diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 03c08288b..897f29081 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -488,6 +488,20 @@ generate_ds41 swiglu_rule_rows 0 0 # PIN(2026-09-25): ds41_attn_seg_stage is the staged prefix segment pass (prefill only, never on the decode path); its ring ⧺ staging source keeps a few registers more than ds41_attn_seg, which already sits at the 128-register cap of a 512-thread block, and ptxas spills 8 bytes. ds41_attn_seg_sel_stage fits in 112 with none. # PIN(2026-10-21): raon_hc_route (Raon's mHC route, one block a token and layer) keeps its 4x3 and 4x4 route matrices in per-thread arrays the while loops index at run time: hz 48 B + e 64 B = 112 B of JIT local; the rest of its work is shared memory. ptxas spills nothing. +# PIN(2026-10-25): Flint entries measured at 313b9f0f88c90fcfae2365a4af53cd077762eb7c: +# sm_86, ptxas 13.3.33, RTX 3090 cc8.6, driver 595.91.07 (CUDA API 13.2). +# Full generate_ds41 JIT: 436 entries, cache off; each new entry has measured spill=0/jit_local=0. +generate_ds41 ffn_handoff_6_tier 0 0 +generate_ds41 gqa_flash_merge_h128_swa_ring 0 0 +generate_ds41 gqa_flash_seg_h128_swa_ring 0 0 +generate_ds41 head_norm_neox_append_ring 0 0 +generate_ds41 normalize 0 0 +generate_ds41 qk_rms_append_ring 0 0 +generate_ds41 restore 0 0 +generate_ds41 routed_shared_residual 0 0 +generate_ds41 sigmoid_floor_256_6_router 0 0 +generate_ds41 sigmoid_floor_256_6_router_pick 0 0 + # gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. # PIN(2026-10-15): the seven Gemma4 entries have zero sm_86 ptxas spill; jit_local=0 is a derived cc8.6 ceiling, # not a cc8.6 driver measurement. Both sm_86 and sm_121 PTX have zero JIT local bytes on GB10 cc12.1. From f17cf1ba67fb1f1e3874af5675bba3a29b621f4f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:32:42 +0900 Subject: [PATCH 24/85] gpu: support ordinary IQ4_XS shared sites --- crates/chain/src/site.rs | 4 + crates/gguf/src/card.rs | 11 +- crates/gguf/src/qtype.rs | 2 +- crates/gpu-deepseek41/src/block.rs | 69 +++--- crates/gpu-deepseek41/src/shared_routed.rs | 185 ++++++++++++++++ crates/gpu-gates/src/act_rule.rs | 45 +--- crates/gpu/src/site.rs | 234 ++++++--------------- crates/placement/src/accepts.rs | 5 +- crates/placement/src/kernels.rs | 121 ----------- docs/research/flint-attach.md | 25 +++ 10 files changed, 333 insertions(+), 368 deletions(-) diff --git a/crates/chain/src/site.rs b/crates/chain/src/site.rs index da5ccefcf..d44ac713e 100644 --- a/crates/chain/src/site.rs +++ b/crates/chain/src/site.rs @@ -38,6 +38,8 @@ pub enum SiteTy { /// entry is none; a backend whose launch reads it names it in the list /// [`file_site`] takes. MlxA4, + /// IQ4_XS in its original blocks, over q8_1 columns, narrow and wide. + Iq4Xs, } impl SiteTy { @@ -56,6 +58,7 @@ impl SiteTy { SiteTy::Mxfp8, SiteTy::Fp8B128, SiteTy::Bf16, + SiteTy::Iq4Xs, ]; /// The family of a file tensor of type `ty`, the one that launches the @@ -99,6 +102,7 @@ impl SiteTy { SiteTy::Mxfp8 => Some(SiteEntry::Mxfp8), SiteTy::Fp8B128 => Some(SiteEntry::Fp8B128), SiteTy::Bf16 => Some(SiteEntry::Bf16), + SiteTy::Iq4Xs => Some(SiteEntry::Iq4xs), SiteTy::MlxA4 => None, } } diff --git a/crates/gguf/src/card.rs b/crates/gguf/src/card.rs index c727e067f..d1575ca56 100644 --- a/crates/gguf/src/card.rs +++ b/crates/gguf/src/card.rs @@ -263,6 +263,9 @@ pub enum SiteEntry { /// `Q8F32Kernels::enqueue_bf16_gemv` over BF16 words and f32 rows; /// wide, `Q8F32Kernels::enqueue_bf16_tile` over the same words and rows. Bf16, + /// `QuantSelKernels::enqueue_gemv_sel` over one matrix in the file's + /// IQ4_XS blocks and `Q8Act` columns; wide, the grouped IQ4_XS GEMM. + Iq4xs, } /// The activation a dense site's launch reads; `crates/gpu/src/site.rs` @@ -289,7 +292,11 @@ impl SiteEntry { #[must_use] pub const fn reads(self) -> SiteAct { match self { - SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128, + SiteEntry::Q3k + | SiteEntry::Q4k + | SiteEntry::Q5k + | SiteEntry::Q6k + | SiteEntry::Iq4xs => SiteAct::Q8x128, SiteEntry::Q8_0 => SiteAct::Q8x32, SiteEntry::F32 => SiteAct::F32, SiteEntry::Q5_0 => SiteAct::Q8Blocks32, @@ -308,7 +315,7 @@ impl SiteEntry { } /// One past the last [`SiteEntry`]: the span [`DENSE_TYPE`] covers. -const DENSE_SPAN: usize = SiteEntry::Bf16 as usize + 1; +const DENSE_SPAN: usize = SiteEntry::Iq4xs as usize + 1; /// Dense entry to the file type of the row that names it. The build refuses /// an entry two rows name, an entry no row names, and an entry past diff --git a/crates/gguf/src/qtype.rs b/crates/gguf/src/qtype.rs index dad5103ed..f579601d2 100644 --- a/crates/gguf/src/qtype.rs +++ b/crates/gguf/src/qtype.rs @@ -115,7 +115,7 @@ pub const QROWS: &[QRow] = &[ card: card!(- - Iq3s (Iq3s, Q8Act) Iq3s - - -) }, QRow { ty: T::IQ2_S, name: "iq2_s", blck: 256, bytes: 82, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::IQ4_XS, name: "iq4_xs", blck: 256, bytes: 136, dequant: true, host_act: H::Q8K, - card: card!(- - Iq4xs (Iq4xs, Q8Act) Iq4xs - - -) }, + card: card!(KQuant - Iq4xs (Iq4xs, Q8Act) Iq4xs Iq4xs - -) }, // The C scalars: ggml.c:621-649. QRow { ty: T::I8, name: "i8", blck: 1, bytes: 1, dequant: false, host_act: H::None, card: Card::NONE }, QRow { ty: T::I16, name: "i16", blck: 1, bytes: 2, dequant: false, host_act: H::None, card: Card::NONE }, diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index 5bd972e7c..0a040e739 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -40,12 +40,13 @@ const BLOCK: &str = "gated block"; /// The types a block's matrices read, one launch family each ([`site::file_site`]'s `allowed`); a model's /// coverage pin for the block lists the same set. -pub const BLOCK_TYPES: [SiteTy; 5] = [ +pub const BLOCK_TYPES: [SiteTy; 6] = [ SiteTy::Q8_0, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K, SiteTy::Mxfp8, + SiteTy::Iq4Xs, ]; /// The most columns [`enqueue_block_cols`] takes: the q8_0 block's multi-column launches' cap, kept for every plan. @@ -56,6 +57,11 @@ fn refuse(what: &'static str, detail: String) -> GpuError { GpuError::Shape { what, detail } } +/// A site whose narrow and wide launches both read the 128-value q8_1 form. +fn q128(ty: SiteTy) -> bool { + ty.reads() == site::Form::Q8x128 && site::gemm_weight(ty).is_some() +} + /// A block's gate·up half, picked once at load from the file's types. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GateUp { @@ -115,20 +121,21 @@ impl BlockPlan { !matches!(self.gate_up, GateUp::Split { .. }) } - /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair, two K-quants (the same Q4_K or Q5_K - /// fused, else split) or two types that read the f32 row for gate·up, any of [`BLOCK_TYPES`] for the down, - /// which a split gate·up needs a K-quant one of. Another mix is refused by name as `what`'s. + /// The plan of a block whose gate, up and down are `tys`: the q8_0 pair, + /// two sites over 128-value q8_1 blocks (the same Q4_K or Q5_K fused, + /// else split), or two f32-form sites. A split pair needs a down over + /// the same q8_1 form. /// /// # Errors - /// A gate and up that are not two q8_0, two K-quants or two f32-form types, or a split pair over a down that - /// is not a K-quant. + /// A gate/up pair with incompatible activation forms, or a split pair + /// over a down of another form. pub fn of_types( what: &'static str, [gate, up, down]: [SiteTy; 3], ) -> Result { let gate_up = match (gate, up) { (SiteTy::Q8_0, SiteTy::Q8_0) => GateUp::Ds41, - (g, u) if g.kquant() && u.kquant() => match (g == u, g) { + (g, u) if q128(g) && q128(u) => match (g == u, g) { (true, SiteTy::Q4K | SiteTy::Q5K) => GateUp::Fused(g), _ => GateUp::Split { gate: g, up: u }, }, @@ -139,13 +146,13 @@ impl BlockPlan { return Err(refuse( what, format!( - "gate is {gate} and up is {up}: a block takes two q8_0, two K-quants or two \ - types that read the f32 row" + "gate is {gate} and up is {up}: a block takes two q8_0, two sites over \ + 128-value q8_1, or two types that read the f32 row" ), )); } }; - if matches!(gate_up, GateUp::Split { .. }) && !down.kquant() { + if matches!(gate_up, GateUp::Split { .. }) && !q128(down) { return Err(refuse( what, format!( @@ -182,7 +189,7 @@ impl BlockPlan { /// an f32 `h` quantizes it first (+1; a split block's SwiGLU already wrote it). #[must_use] pub fn extra_launches(self) -> usize { - let down_quantize = usize::from(self.down.kquant()); + let down_quantize = usize::from(q128(self.down)); match self.gate_up { GateUp::Ds41 => down_quantize, GateUp::Fused(_) => 1 + down_quantize, @@ -296,7 +303,7 @@ impl BlockScratch { let (mut any, mut split_ff) = (false, 0); for (plan, ff) in blocks.into_iter().filter(|(p, _)| !p.is_q8()) { any = true; - if plan.down.kquant() && !ha.iter().any(|(f, _)| *f == ff) { + if q128(plan.down) && !ha.iter().any(|(f, _)| *f == ff) { ha.push((ff, Q8Act::with_k(stream, 1, ff)?)); } if matches!(plan.gate_up, GateUp::Split { .. } | GateUp::Site { .. }) { @@ -455,7 +462,7 @@ pub fn enqueue_block_act( gpu.layer_sink(blk.layer)?, h, )?; - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } } @@ -506,7 +513,7 @@ pub fn enqueue_block_act( .enqueue_swiglu_rule(stream, rule, (&*g, &*u), blk.ff, (fault, &mut *h))?; } } - if !blk.plan.down.kquant() { + if !q128(blk.plan.down) { return site::resident_gemv(gpu, w, down, h, y); } let (kk, sc) = kit(k, s)?; @@ -656,16 +663,6 @@ mod tests { BlockPlan::of_types(WHAT, tys) } - fn refused(r: Result, words: &[&str]) { - let msg = match r { - Err(GpuError::Shape { what, detail }) => format!("{what}: {detail}"), - other => panic!("expected a named shape refusal, got {other:?}"), - }; - for w in words { - assert!(msg.contains(w), "{msg:?} lacks {w:?}"); - } - } - /// The plan is the types' dispatch: the q8_0 triple is `BlockPlan::Q8`, one Q4_K or Q5_K for gate and up /// is fused, any other K-quant pair splits, an MXFP8 pair runs the f32-form arm, and every plan reads back /// the types it came from. Mutant: the f32-form arm dropped — the MXFP8 triple is refused. @@ -700,19 +697,19 @@ mod tests { } } - /// A mix no arm covers is refused by name, with the types: gate and up of two kinds, and a split pair over - /// a q8_0 down. #[test] - fn a_mix_no_arm_covers_is_refused() { - use SiteTy::{Mxfp8, Q4K, Q6K, Q8_0}; - refused( - plan([Q8_0, Q4K, Q8_0]), - &["test", "gate is Q8_0", "up is Q4_K"], - ); - refused(plan([Q6K, Q8_0, Q6K]), &["gate is Q6_K", "up is Q8_0"]); - refused(plan([Q6K, Q6K, Q8_0]), &["split", "Q8_0 down"]); - refused(plan([Mxfp8, Q8_0, Mxfp8]), &["gate is MXFP8", "up is Q8_0"]); - refused(plan([Q4K, Mxfp8, Q4K]), &["gate is Q4_K", "up is MXFP8"]); + fn incompatible_activation_forms_refuse_before_launch() { + use SiteTy::{Iq4Xs, Mxfp8, Q4K, Q6K, Q8_0}; + for types in [ + [Q8_0, Q4K, Q8_0], + [Q6K, Q8_0, Q6K], + [Q6K, Q6K, Q8_0], + [Iq4Xs, Iq4Xs, Q8_0], + [Mxfp8, Q8_0, Mxfp8], + [Q4K, Mxfp8, Q4K], + ] { + assert!(matches!(plan(types), Err(GpuError::Shape { .. }))); + } } /// The launches beyond the q8_0 block's two, by plan: 0 for the q8_0 block, and the other arms' derived diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index e71e92788..c5be10e29 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1184,3 +1184,188 @@ fn no_batch() -> GpuError { missing: "the batch's buffers", } } + +#[cfg(test)] +mod tests { + use super::SharedWide; + use crate::block::{ + Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, + enqueue_block_cols_act, + }; + use crate::experts::ExpertKernels; + use bloomery_gpu::Gpu; + use bloomery_gpu::gemm::Gemm32Kernels; + use bloomery_gpu::kquant::Act; + use bloomery_gpu::weights::Weights; + use cuda_core::DeviceBuffer; + use gguf::Split; + use gguf::quant::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + + const N: usize = 256; + const FF: usize = 512; + const NAMES: [&str; 3] = [ + "blk.42.ffn_gate_shexp.weight", + "blk.42.ffn_up_shexp.weight", + "blk.42.ffn_down_shexp.weight", + ]; + + fn iq4_shared_file() -> (std::path::PathBuf, Split) { + let path = + std::env::temp_dir().join(format!("bloomery-iq4-shared-{}.gguf", std::process::id())); + let specs = [(N, FF, 0x1c00u16), (N, FF, 0x1800), (FF, N, 0x1800)]; + let payloads: Vec> = specs + .iter() + .enumerate() + .map(|(i, &(k, rows, scale))| { + let mut bytes = Vec::with_capacity(rows * (k / 256) * 136); + for row in 0..rows { + let d = if i == 2 && row % 2 == 1 { + scale - 0x400 + } else { + scale + }; + let mut block = [0x88u8; 136]; + block[..2].copy_from_slice(&d.to_le_bytes()); + block[2..4].copy_from_slice(&0xaaaau16.to_le_bytes()); + block[4..8].fill(0x11); + for _ in 0..k / 256 { + bytes.extend_from_slice(&block); + } + } + bytes + }) + .collect(); + let decls = NAMES + .iter() + .zip(specs) + .zip(&payloads) + .map(|((&name, (k, rows, _)), bytes)| TensorDecl { + name: name.to_owned(), + dims: vec![u64::try_from(k).unwrap(), u64::try_from(rows).unwrap()], + type_id: GgmlType::IQ4_XS.as_u32(), + nbytes: u64::try_from(bytes.len()).unwrap(), + }) + .collect(); + let layout = Layout::new(&[], decls).unwrap(); + let file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + let mut writer = Writer::new(file, layout).unwrap(); + for (name, bytes) in NAMES.iter().zip(payloads) { + writer.tensor(name, &bytes).unwrap(); + } + writer.finish().unwrap(); + let split = Split::open(&path).unwrap(); + (path, split) + } + + fn check_shared_rows(input: &[f32], rows: &[f32]) { + // Uniform positive rows quantize to code127 without integer rounding. + // The gate is at least128, so SiLU rounds to the gate itself. The + // scalar positive-dot upper bound has four input-width operations, + // two output-width operations and the scale/product roundings. + let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; + let eps = f64::from(f32::EPSILON); + let gamma = operations * eps / (1.0 - operations * eps); + for (c, row) in rows.chunks_exact(N).enumerate() { + let x = f64::from(input[c * N]); + for (r, &got) in row.iter().enumerate() { + let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; + assert!(got.is_finite()); + assert!( + (f64::from(got) - expected).abs() <= gamma * expected, + "column {c} row {r}: {got} vs {expected}, relative bound {gamma}" + ); + } + } + } + + #[test] + #[ignore = "needs a CUDA card"] + fn hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay() { + let (path, split) = iq4_shared_file(); + let gpu = Gpu::new().expect("card"); + let stream = gpu.stream(); + let weights = Weights::load_where(stream, &split, |name| NAMES.contains(&name)).unwrap(); + let plan = BlockPlan::of_file(&split, "IQ4_XS shared", NAMES, (FF, N)).unwrap(); + plan.check_resident(&weights, "IQ4_XS shared", NAMES, (FF, N)) + .unwrap(); + let block = Block { + plan, + names: NAMES, + ff: FF, + limit: 0.0, + layer: 42, + }; + let experts = ExpertKernels::load(gpu.context()).unwrap(); + let quant = KBlockKernels::load(&gpu).unwrap(); + let kernels = BlockKernels { + experts: &experts, + kquant: Some(&quant), + }; + let mut scratch = BlockScratch::new(stream, N, [(plan, FF)]).unwrap(); + let mut h = DeviceBuffer::zeroed(stream, FF).unwrap(); + let g32 = Gemm32Kernels::load(gpu.context()).unwrap(); + for cols in [1, 8, super::GATED_FROM] { + let input: Vec = (0..cols) + .flat_map(|c| std::iter::repeat_n([128.0, 256.0, 384.0][c % 3], N)) + .collect(); + let x = DeviceBuffer::from_host(stream, &input).unwrap(); + let mut y = DeviceBuffer::zeroed(stream, cols * N).unwrap(); + if cols < super::GATED_FROM { + enqueue_block_cols_act( + &gpu, + &weights, + &kernels, + &mut scratch, + (&block, Act::SiluMul), + cols, + BlockIo { + x: &x, + h: &mut h, + y: &mut y, + }, + ) + .unwrap(); + } else { + let types = plan.types(); + let mut wide = + SharedWide::new(&gpu, (&types[..2], &types[2..]), (cols, N, FF)).unwrap(); + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + .unwrap(); + let eager = y.to_host_vec(stream).unwrap(); + check_shared_rows(&input, &eager); + let graph = gpu + .capture(|_| { + wide.run( + (&gpu, &weights, &g32), + (&block, Act::SiluMul), + (cols, N), + (&x, &mut y), + ) + }) + .unwrap(); + graph.launch(stream).unwrap(); + let replay = y.to_host_vec(stream).unwrap(); + assert!( + eager + .iter() + .zip(&replay) + .all(|(a, b)| a.to_bits() == b.to_bits()) + ); + } + check_shared_rows(&input, &y.to_host_vec(stream).unwrap()); + assert_eq!(gpu.take_fault().unwrap(), None); + } + stream.synchronize().unwrap(); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/gpu-gates/src/act_rule.rs b/crates/gpu-gates/src/act_rule.rs index a959cb768..33596c379 100644 --- a/crates/gpu-gates/src/act_rule.rs +++ b/crates/gpu-gates/src/act_rule.rs @@ -237,8 +237,11 @@ impl SiteRel { #[must_use] pub fn site_rel(ty: GgmlType, arm: Arm) -> Option { let ours = match (ty, arm) { - (GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K, _) - | (GgmlType::IQ3_XXS | GgmlType::IQ4_XS, Arm::Expert) => Some(Q8_1_BLOCK), + ( + GgmlType::Q3_K | GgmlType::Q4_K | GgmlType::Q5_K | GgmlType::Q6_K | GgmlType::IQ4_XS, + _, + ) + | (GgmlType::IQ3_XXS, Arm::Expert) => Some(Q8_1_BLOCK), // Bolt's IQ2_XXS: the routed `iq2_xxs_gemv_sel` and the dense `iq2_xxs_rows` read q8_1. (GgmlType::IQ2_XXS, Arm::Expert | Arm::Gemv) => Some(Q8_1_BLOCK), // Bolt's dense NVFP4 (`nvfp4_gemv`, the ssm_in/ssm_out) reads the engine's q8_1. @@ -1012,7 +1015,6 @@ mod site_tests { for ty in [ GgmlType::IQ3_S, GgmlType::IQ3_XXS, - GgmlType::IQ4_XS, GgmlType::Q5_1, GgmlType::IQ4_NL, ] { @@ -1040,43 +1042,6 @@ mod site_tests { } } - /// Ours is the form the launches read, as their sources say it: the text of the kernel - /// table's `SiteEntry::reads` (which `site.rs`'s `SiteTy::reads` returns) and of the 32-value - /// GEMM family's module doc, so a launch that moves to another form is red here until the - /// table follows it. - #[test] - fn site_rel_reads_the_forms_the_launches_name() { - let gpu = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../gpu/src"); - let read = |file: &str| { - std::fs::read_to_string(gpu.join(file)) - .unwrap_or_else(|e| panic!("crates/gpu/src/{file}: {e}")) - }; - let (site, gemm, grouped) = ( - read("../../gguf/src/card.rs"), - read("gemm/mod.rs"), - read("gemm/grouped.rs"), - ); - for line in [ - "SiteEntry::Q3k | SiteEntry::Q4k | SiteEntry::Q5k | SiteEntry::Q6k => SiteAct::Q8x128", - "SiteEntry::Q8_0 => SiteAct::Q8x32", - "SiteEntry::F32 => SiteAct::F32", - "SiteEntry::Q5_0 => SiteAct::Q8Blocks32", - "q8_1 blocks of 128 values", - ] { - assert!(site.contains(line), "kernels.rs no longer says {line:?}"); - } - assert!( - gemm.contains("(Q8_0, IQ4_NL, Q5_1), over [`GemmAct32`]"), - "gemm/mod.rs no longer names the 32-value family" - ); - for variant in ["Iq3Xxs", "Iq4Xs"] { - assert!( - grouped.contains(variant), - "the grouped GEMM no longer decodes {variant}" - ); - } - } - /// ik's side of the table is the transcription this module already holds for the K-quants /// ([`Rule::of`]), and so is ours for Q3_K and Q4_K; Q5_K's ours differs from it by design (the /// V4.1 gemv reads f32, the bodies' site reads q8_1). diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index eaefc6f3c..f36ae5733 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -105,14 +105,14 @@ pub enum Order { /// file type its form holds ([`SiteTy::of_ggml`]; an F32 plane is the f32 /// site whatever type it was widened from). `None` for a form no dense entry /// reads: a stack in the file's blocks of a type whose entry reads another -/// layout (Q8_0's planes, an i-quant), a derived weight and an NVFP4 stack. +/// layout (Q8_0's planes), a derived weight and an NVFP4 stack. /// The resident form is this crate's, so the map is here, beside the /// launches that read it, and not on the chain's [`SiteTy`]. #[must_use] pub fn of_resident(w: &DevWeight) -> Option { let ty = match w { DevWeight::KQuant { ty, .. } => { - return SiteTy::of_ggml(*ty).filter(|s| s.kquant()); + return SiteTy::of_ggml(*ty).filter(|s| s.kquant() || *s == SiteTy::Iq4Xs); } DevWeight::Q5_0 { .. } => GgmlType::Q5_0, DevWeight::Q5_1 { .. } => GgmlType::Q5_1, @@ -127,7 +127,7 @@ pub fn of_resident(w: &DevWeight) -> Option { SiteTy::of_ggml(ty) } -/// How [`kgemv`] lays a site of type `ty` out: Q3_K, Q4_K and Q6_K row-major, Q5_K and NVFP4 token-major. +/// How [`kgemv`] lays a site out: Q3_K, Q4_K and Q6_K row-major, Q5_K, IQ4_XS and NVFP4 token-major. /// `None` for a type [`kgemv`] does not run: Q8_0, F32, MXFP8, FP8_B128 and BF16 ([`gemv`]), Q5_0 /// ([`blocks32_gemv`], whose row-major layout `y[r · m + c]` is that function's own), and MLX_A4, /// a backend's family no card launch reads. @@ -135,7 +135,7 @@ pub fn of_resident(w: &DevWeight) -> Option { pub fn kgemv_order(ty: SiteTy) -> Option { match ty { SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q6K => Some(Order::RowMajor), - SiteTy::Q5K | SiteTy::Nvfp4 => Some(Order::TokenMajor), + SiteTy::Q5K | SiteTy::Iq4Xs | SiteTy::Nvfp4 => Some(Order::TokenMajor), SiteTy::Q5_0 | SiteTy::Q8_0 | SiteTy::F32 @@ -667,6 +667,22 @@ pub fn kgemv( y, ) } + SiteTy::Iq4Xs => { + let wt = kq(w, WHAT, name)?; + kk.kq.enqueue_gemv_sel( + gpu.stream(), + GgmlType::IQ4_XS, + &SelDown { + w: wt, + act, + sel: &kk.one, + n_slots: act.m(), + rows_per_expert: wt.rows(), + }, + fault, + y, + ) + } SiteTy::Nvfp4 => kk .nv .enqueue_gemv(gpu.stream(), w.nvfp4(WHAT, name)?, act, y), @@ -753,9 +769,11 @@ impl<'a> SiteWeight<'a> { /// name as `what`. fn check(self, what: &'static str, ty: SiteTy) -> Result<(), GpuError> { match self { - SiteWeight::Words { name, .. } if !ty.kquant() => Err(GpuError::shape( + SiteWeight::Words { name, .. } if gemm_weight(ty).is_none() => Err(GpuError::shape( what, - format!("{name} is {ty} on words a caller holds; only a K-quant's words launch"), + format!( + "{name} is {ty} on words a caller holds; its grouped GEMM reads no such words" + ), )), SiteWeight::Part { name, .. } if !BY_NAME.contains(&ty) => Err(GpuError::shape( what, @@ -1071,21 +1089,23 @@ pub fn gemm( src.check(WHAT, ty)?; match ty { SiteTy::Nvfp4 => Err(no_wide_gemm(WHAT, name, ty)), - SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K => k.gemm.enqueue_gemm( - stream, - GemmArgs { - ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, - w: match src { - SiteWeight::Words { w, .. } => w, - _ => kq(w, WHAT, name)?, + SiteTy::Q3K | SiteTy::Q4K | SiteTy::Q5K | SiteTy::Q6K | SiteTy::Iq4Xs => { + k.gemm.enqueue_gemm( + stream, + GemmArgs { + ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, + w: match src { + SiteWeight::Words { w, .. } => w, + _ => kq(w, WHAT, name)?, + }, + rows_per_expert: rows, + act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, + route, + input, + y, }, - rows_per_expert: rows, - act: x.q128.ok_or_else(|| missing("q8_1 blocks of 128"))?, - route, - input, - y, - }, - ), + ) + } SiteTy::Q8_0 | SiteTy::F32 | SiteTy::Mxfp8 | SiteTy::Fp8B128 | SiteTy::Bf16 => { dense()?; let (site, part) = src.rows(WHAT)?; @@ -1668,92 +1688,40 @@ mod tests { Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, file_site, gemm_weight, kgemv_order, swiglu_launch, }; - use crate::gemm::GemmWeight; use crate::kquant::Act; use gguf::Split; use gguf::quant::GgmlType; - use gguf::write::{Layout, TensorDecl, Writer}; - use model::placement::kernels; - use std::fs::File; - use std::io::BufWriter; - - /// A site's type is read from the file's header alone: a Q8_0 matrix of - /// two rows of 64 is read as Q8_0 where Q8_0 is launched, and refused by - /// name where only K-quants are; a Q3_K, a Q5_K, a Q5_0 and an MXFP8 one - /// (bloomery's id, sized by the split) are read as theirs; an IQ4_XS one (a type no site launch reads) and a matrix of - /// another shape are refused by name. + use gguf::write::{Layout, TensorDecl}; + #[test] - fn a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused() { - let dir = std::env::temp_dir().join(format!("bloomery-site-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap_or_else(|e| panic!("{e}")); - let path = dir.join("m.gguf"); - let decls = [ - ("q8", vec![64, 2], 8u32, 2 * 2 * 34), - ("iq", vec![256, 1], 23, 136), - ("q3", vec![256, 2], 11, 2 * 110), - ("q5", vec![256, 2], 13, 2 * 176), - ("q50", vec![64, 2], 6, 2 * 2 * 22), - ("mx", vec![128, 2], GgmlType::MXFP8.as_u32(), 2 * 4 * 33), - ("bf", vec![64, 2], GgmlType::BF16.as_u32(), 2 * 64 * 2), - ]; - let tensors = decls - .iter() - .map(|(name, dims, type_id, nbytes)| TensorDecl { - name: (*name).to_string(), - dims: dims.clone(), - type_id: *type_id, - nbytes: *nbytes, - }) - .collect(); - let layout = Layout::new(&[], tensors).unwrap_or_else(|e| panic!("{e}")); - let file = BufWriter::new(File::create(&path).unwrap_or_else(|e| panic!("{e}"))); - let mut w = Writer::new(file, layout).unwrap_or_else(|e| panic!("{e}")); - for (name, _, _, nbytes) in &decls { - w.tensor(name, &vec![0u8; *nbytes as usize]) - .unwrap_or_else(|e| panic!("{e}")); - } - w.finish().unwrap_or_else(|e| panic!("{e}")); - let split = Split::open(&path).unwrap_or_else(|e| panic!("{e}")); - let all = [ - SiteTy::Q3K, - SiteTy::Q4K, - SiteTy::Q5K, - SiteTy::Q6K, - SiteTy::Q5_0, - SiteTy::Q8_0, - SiteTy::F32, - SiteTy::Nvfp4, - SiteTy::Mxfp8, - SiteTy::Bf16, - ]; - for (name, k, want) in [ - ("q8", 64, SiteTy::Q8_0), - ("q3", 256, SiteTy::Q3K), - ("q5", 256, SiteTy::Q5K), - ("q50", 64, SiteTy::Q5_0), - ("mx", 128, SiteTy::Mxfp8), - ("bf", 64, SiteTy::Bf16), + fn iq4_sites_keep_shape_and_launch_admission_separate() { + let decl = |name: &str, ty: GgmlType, bytes| TensorDecl { + name: name.to_owned(), + dims: vec![256, 2], + type_id: ty.as_u32(), + nbytes: bytes, + }; + let layout = Layout::new( + &[], + vec![ + decl("shared", GgmlType::IQ4_XS, 2 * 136), + decl("unsupported", GgmlType::IQ3_S, 2 * 110), + ], + ) + .unwrap(); + let file = Split::header_only("sites.gguf".into(), &layout).unwrap(); + assert_eq!( + file_site(&file, "test", "shared", (2, 256), SiteTy::ALL).ok(), + Some(SiteTy::Iq4Xs) + ); + for (name, shape, allowed) in [ + ("shared", (1, 256), SiteTy::ALL), + ("shared", (2, 512), SiteTy::ALL), + ("shared", (2, 256), &[SiteTy::Q4K][..]), + ("unsupported", (2, 256), SiteTy::ALL), ] { - let got = file_site(&split, "test", name, (2, k), &all); - assert_eq!(got.ok(), Some(want), "{name}"); + assert!(file_site(&file, "test", name, shape, allowed).is_err()); } - let refused = [ - ( - "q8", - (2, 64), - &[SiteTy::Q4K, SiteTy::Q6K][..], - "q8 is q8_0 in the file", - ), - ("iq", (1, 256), &all[..], "iq is iq4_xs in the file"), - ("q8", (4, 64), &all[..], "it takes 4 rows of 64"), - ]; - for (name, shape, allowed, want) in refused { - match file_site(&split, "test", name, shape, allowed) { - Err(e) => assert!(e.to_string().contains(want), "{name}: {e}"), - Ok(t) => panic!("{name} {shape:?} read as {t}"), - } - } - let _ = std::fs::remove_dir_all(&dir); } /// The SwiGLU's launch is the down's form under the rule: every K-quant @@ -1814,72 +1782,6 @@ mod tests { } } - /// Every family is the kernel table's dense row of one file type: a type - /// has a family exactly where the table names a dense entry for it (F16, - /// which loads widened to f32, is the f32 site of its widened rows), the - /// family maps back to that type and is named by it in upper case, its - /// K-quant GEMM is the one the table names for the type, it is a - /// `kgemv` exactly where it reads the q8_1 rows of 128 values, and it - /// launches by name (`BY_NAME`) exactly where it reads the f32 rows or - /// the q8 blocks of 32. - #[test] - fn every_family_is_the_tables_dense_row_of_one_type() { - // PIN(2026-10-21): consolidate the three site pins, preserving their assertions. - let own = [ - GgmlType::F8E4M3_ROWS, - GgmlType::MXFP8, - GgmlType::NVFP4_ROWS, - GgmlType::FP8_B128, - ]; - for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { - let family = SiteTy::of_ggml(ty); - if ty == GgmlType::F16 { - assert_eq!( - family, - Some(SiteTy::F32), - "{ty}: the f32 site of its widened rows" - ); - assert_eq!(kernels::dense(ty), None, "{ty}"); - continue; - } - assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); - if let Some(s) = family { - assert_eq!(s.ggml(), ty, "{s}"); - } - } - let expected = [ - (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), - (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), - (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), - (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), - (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), - (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), - (SiteTy::F32, GgmlType::F32, "F32", None), - (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), - (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), - (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), - (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - ]; - assert_eq!(SiteTy::ALL.len(), expected.len()); - for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { - assert_eq!(s, want); - assert_eq!(s.ggml(), ty); - assert_eq!(s.to_string(), label); - assert_eq!(gemm_weight(s), grouped); - assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); - let by_name = model::placement::accepts::dense_by_name(ty); - assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); - if let Some(entry) = by_name { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - } - assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); - assert_eq!(kgemv_order(SiteTy::Q5_0), None); - assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); - assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); - assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); - } - #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 4b0809668..bb3f77110 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -241,7 +241,8 @@ pub fn qwen35_leg([gate, up, down]: [GgmlType; 3]) -> Option { /// is `None`. An NVFP4 site has a gemv and no wide GEMM, which the body's /// prompt units run every dense site through, and is `None`. A BF16 site /// reads the words only a load under `ModelTensors::bf16_words` keeps, which -/// the qwen3moe files do not set, and is `None`. The body's load and the +/// the qwen3moe files do not set, and is `None`. Its legacy shared-expert +/// dispatch does not feed IQ4_XS sites either. The body's load and the /// coverage census both read this, so a site the arena cannot feed is /// refused at load and listed together. #[must_use] @@ -249,7 +250,7 @@ pub const fn qwen3moe_dense(ty: GgmlType) -> Option { let Some(entry) = kernels::dense(ty) else { return None; }; - if let SiteEntry::Nvfp4 | SiteEntry::Bf16 = entry { + if let SiteEntry::Nvfp4 | SiteEntry::Bf16 | SiteEntry::Iq4xs = entry { return None; } match entry.reads() { diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index f11b23196..7ab1361b8 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -642,32 +642,6 @@ mod tests { ); } - // PIN(2026-10-16): the dense pairs gain NVFP4 (`nvfp4_gemv` through - // `site::kgemv`); every other pair is unchanged. - // PIN(2026-10-19): and bloomery's own MXFP8 (`mxfp8_gemv` through - // `site::gemv`); every other pair is unchanged. - // PIN(2026-10-20): and bloomery's own FP8_B128 (`fp8_b128_gemv` through - // `site::gemv`); BF16 also uses `bf16_gemv` and `bf16_tile_gemm` over its words. - #[test] - fn dense_admits_exactly_the_pinned_pairs() { - assert_eq!( - rows(dense), - [ - (GgmlType::F32, SiteEntry::F32), - (GgmlType::Q5_0, SiteEntry::Q5_0), - (GgmlType::Q8_0, SiteEntry::Q8_0), - (GgmlType::Q3_K, SiteEntry::Q3k), - (GgmlType::Q4_K, SiteEntry::Q4k), - (GgmlType::Q5_K, SiteEntry::Q5k), - (GgmlType::Q6_K, SiteEntry::Q6k), - (GgmlType::BF16, SiteEntry::Bf16), - (GgmlType::NVFP4, SiteEntry::Nvfp4), - (GgmlType::MXFP8, SiteEntry::Mxfp8), - (GgmlType::FP8_B128, SiteEntry::Fp8B128), - ] - ); - } - /// A dense entry is named by one type's row, so its file type is that /// row's type. #[test] @@ -707,101 +681,6 @@ mod tests { ); } - /// The layout a file tensor of each type loads in, and the one a routed - /// stack loads in under the default expert rule: every type's but q5_K's, - /// which that rule does not admit, and mxfp8's, which has no card expert - /// entry. - // PIN(2026-10-23): NVFP4_ROWS loads as NVFP4's planes (its routed card - // columns); every other pair is unchanged. - #[test] - fn the_card_layouts_are_the_pinned_pairs() { - assert_eq!( - rows(CardFormat::of), - [ - (GgmlType::F32, CardFormat::F32), - (GgmlType::F16, CardFormat::Bf16AsF32), - (GgmlType::Q5_0, CardFormat::Q5_0), - (GgmlType::Q5_1, CardFormat::Q5_1), - (GgmlType::Q8_0, CardFormat::Q8_0Planes), - (GgmlType::Q3_K, CardFormat::KQuant), - (GgmlType::Q4_K, CardFormat::KQuant), - (GgmlType::Q5_K, CardFormat::KQuant), - (GgmlType::Q6_K, CardFormat::KQuant), - (GgmlType::BF16, CardFormat::Bf16AsF32), - (GgmlType::NVFP4, CardFormat::Nvfp4Planes), - (GgmlType::IQ2_XXS, CardFormat::KQuant), - (GgmlType::MXFP8, CardFormat::Mxfp8Planes), - (GgmlType::NVFP4_ROWS, CardFormat::Nvfp4Planes), - (GgmlType::FP8_B128, CardFormat::Fp8B128Planes), - ] - ); - for ty in ALL { - let routed = - CardFormat::of(ty).filter(|_| !matches!(ty, GgmlType::Q5_K | GgmlType::MXFP8)); - assert_eq!(CardFormat::of_routed(ty), routed, "{ty:?}"); - } - } - - /// The types each layout holds: the types it is the load layout of, the - /// K-quants as decoded f32, bf16 as raw words, and in the file's blocks - /// every type a routed entry reads in them. - #[test] - fn each_layout_holds_exactly_the_pinned_types() { - let held = |f: CardFormat| -> Vec { - ALL.into_iter().filter(|&ty| f.holds(ty)).collect() - }; - assert_eq!( - held(CardFormat::KQuant), - [ - GgmlType::Q5_1, - GgmlType::Q8_0, - GgmlType::Q3_K, - GgmlType::Q4_K, - GgmlType::Q5_K, - GgmlType::Q6_K, - GgmlType::MXFP4, - GgmlType::IQ2_XXS, - GgmlType::IQ2_XS, - GgmlType::IQ3_XXS, - GgmlType::IQ4_NL, - GgmlType::IQ3_S, - GgmlType::IQ4_XS, - ] - ); - assert_eq!( - held(CardFormat::Bf16AsF32), - [ - GgmlType::F16, - GgmlType::Q3_K, - GgmlType::Q4_K, - GgmlType::Q5_K, - GgmlType::Q6_K, - GgmlType::BF16, - GgmlType::IQ2_XXS, - ] - ); - // PIN(2026-10-22): independent BF16 heads also occupy MXFP8 planes. - assert_eq!( - held(CardFormat::Mxfp8Planes), - [GgmlType::BF16, GgmlType::MXFP8] - ); - // PIN(2026-10-23): NVFP4_ROWS's routed stacks load as NVFP4's planes. - assert_eq!( - held(CardFormat::Nvfp4Planes), - [GgmlType::NVFP4, GgmlType::NVFP4_ROWS] - ); - for (f, ty) in [ - (CardFormat::Q5_0, GgmlType::Q5_0), - (CardFormat::Q5_1, GgmlType::Q5_1), - (CardFormat::Q8_0Planes, GgmlType::Q8_0), - (CardFormat::Fp8B128Planes, GgmlType::FP8_B128), - (CardFormat::F32, GgmlType::F32), - (CardFormat::Bf16Raw, GgmlType::BF16), - ] { - assert_eq!(held(f), [ty], "{f:?}"); - } - } - /// A card entry reads a layout the card loader can build: every type a /// row names has a [`CardFormat`] of its own, or is one of the file-block /// types a program's routed stack keeps as `KQuant`. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0d60c5feb..012ef946d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -66,6 +66,31 @@ Each device slice needs its numerical/fault gates and the maintainer's offered h Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. +### Ordinary IQ4_XS shared-site implementation + +IQ4_XS now has an ordinary file-layout/dense entry in QROWS and a common site family. +The loader keeps its original block words; `site::kgemv` uses the existing IQ4_XS +entry over one matrix and `site::gemm` uses the existing grouped GEMM. The common +gated block selects a split gate/up and sizes the down's q8_1 scratch from the +actual activation form. Its column walk and the shared wide-prompt owner use +those same sites. IQ4_XS is not relabeled as a routed expert or requantized. + +A compiled public `file_site` consumer on the pinned header refused all three +layer42 shared matrices before this change and accepts their IQ4_XS/Q8x128 +bindings after it. This reads no payload and proves neither loading nor arithmetic. +`hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay`, owned by +`gate-gpu-ds41-lib`, covers real packed-weight loading, unequal matrix widths, +one/eight-column block execution and the wide-prompt transition/captured replay. +Its nonnegative constant-weight case derives a scalar floating-operation bound; +it is not a whole-model accuracy pin. The hardware case has not been run here. + +Existing program-specific admission remains unchanged: in particular the legacy +Qwen projection/shared dispatch does not gain this site automatically. Flint +still has no Program, attention/router binding, body or serve seat. Run the new +consumer gate and the affected regressions on NVIDIA before claiming device +support; reconcile full-model coverage only with the actual body integration. + + ## 1. Sources and observed artifact From 614ff913f8efa3c50359f19790577c9b80344280 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:37:10 +0900 Subject: [PATCH 25/85] gpu: use fixed-size rows in the IQ4 consumer gate --- crates/gpu-deepseek41/src/shared_routed.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index c5be10e29..7eaf38e55 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1270,7 +1270,7 @@ mod tests { let operations = 4.0 * 256.0 + 2.0 * 512.0 + 16.0; let eps = f64::from(f32::EPSILON); let gamma = operations * eps / (1.0 - operations * eps); - for (c, row) in rows.chunks_exact(N).enumerate() { + for (c, row) in rows.as_chunks::().0.iter().enumerate() { let x = f64::from(input[c * N]); for (r, &got) in row.iter().enumerate() { let expected = x * x * if r % 2 == 0 { 0.5 } else { 0.25 }; From aa8e1b8f72c3ec7413bdb84d9ba98d627b89ddde Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:34:14 +0900 Subject: [PATCH 26/85] gpu: size mixed expert scratch from selected resident forms --- crates/gpu-deepseek41/src/body.rs | 2 +- .../gpu-gates/src/bin/gate_glm5next_nodes.rs | 2 +- .../src/bin/gate_glm5next_twocard.rs | 2 +- .../gpu-gates/src/bin/gate_solar2_twocard.rs | 2 +- crates/gpu-gates/src/bin/generate_glm5next.rs | 2 +- .../src/bin/shared/glm5next_place.rs | 12 +- .../src/bin/shared/routed_legs/glm5next.rs | 3 +- .../src/bin/shared/serve_seats/glm.rs | 2 +- .../src/bin/shared/serve_seats/hybrid.rs | 15 +- .../src/bin/shared/serve_seats/solar2.rs | 9 +- .../src/bin/shared/serve_seats/tiered.rs | 12 +- .../gpu-gates/src/bin/shared/solar2_seat.rs | 2 +- crates/gpu-glm5next/src/tier.rs | 1 + crates/gpu-mimo2/src/card.rs | 1 + crates/gpu-minimaxm3/src/ffn.rs | 1 + crates/gpu-solar2/src/ffn.rs | 1 + crates/gpu-solar2/src/tier.rs | 1 + crates/gpu/src/arch/musespark/body/tier.rs | 2 + crates/gpu/src/arch/qwen3moe/body38.rs | 2 +- crates/gpu/src/arch/qwen3moe/tier_leg.rs | 34 +- crates/gpu/src/card_experts.rs | 367 +++++++++++++++++- crates/gpu/src/card_leg.rs | 15 +- crates/gpu/src/host/batch.rs | 28 +- crates/gpu/src/host/leg_tier.rs | 72 ++-- crates/gpu/src/host/node.rs | 14 +- crates/model/src/arch/deepseek41/kv.rs | 1 + crates/model/src/arch/glm5next/place.rs | 167 ++++---- crates/model/src/arch/mimo2/place.rs | 6 +- crates/model/src/arch/minimaxm3/place.rs | 12 +- crates/model/src/arch/musespark/place.rs | 3 +- crates/model/src/arch/qwen35moe/place.rs | 10 +- crates/model/src/arch/qwen3moe/place.rs | 4 +- crates/model/src/arch/solar2/place.rs | 157 ++++++-- crates/model/tests/placement.rs | 1 + crates/model/tests/qwen4exp_mtp_reserve.rs | 1 + crates/model/tests/solar_open2_meta.rs | 1 + crates/placement/src/accepts.rs | 37 ++ crates/placement/src/kernels.rs | 157 +++++++- crates/placement/src/placement.rs | 361 +++++++++++++++-- crates/placement/src/placement/nodes_scale.rs | 2 + .../placement/src/placement/qwen38_cards.rs | 2 + crates/placement/src/placement/workstation.rs | 139 +++++-- docs/research/flint-attach.md | 17 + 43 files changed, 1339 insertions(+), 343 deletions(-) diff --git a/crates/gpu-deepseek41/src/body.rs b/crates/gpu-deepseek41/src/body.rs index 0ef84a357..051d00605 100644 --- a/crates/gpu-deepseek41/src/body.rs +++ b/crates/gpu-deepseek41/src/body.rs @@ -2563,7 +2563,7 @@ impl Body { // The tiers' batch staging is reserved on their cards by the plan, // so it is made at load, not at the first prompt. hybrid.prepare_batch(gpu.context(), prefill::T_MAX)?; - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; Some(TierPiece::new(gpu, hybrid.slots(), PAIR_ROWS)?) }; let ced = ced::Ced::new(&CedLayer::table(hp), seq.ring_rows(), cfg.ced); diff --git a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs index 1b025c8c0..2e593a526 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_nodes.rs @@ -30,7 +30,7 @@ mod gate { let file = Split::open(&path)?; let inputs = PlanInputs::describe(&file)?; let (mut machine, node) = - crate::node_history::machine(inputs.model.layers, tier_batch(&inputs))?; + crate::node_history::machine(inputs.model.layers, tier_batch(&inputs)?)?; reserve_checkpoints( &mut machine, Seqs::priced(1, checkpoint_price(&inputs)?), diff --git a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs index 4fc7a7a82..3d30bbeda 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs @@ -517,7 +517,7 @@ mod gate { fn bp_machine(inputs: &PlanInputs) -> Result Machine + Copy, GateError> { glm_machine( Place::Bp, - Some(place::tier_batch(inputs)), + Some(place::tier_batch(inputs)?), 1, checkpoint_price(inputs)?, u64::try_from(CTX)?, diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index c32637070..af0916a7c 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs.hp)))?; + .machine(None, Some(place::tier_batch(&inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu-gates/src/bin/generate_glm5next.rs b/crates/gpu-gates/src/bin/generate_glm5next.rs index 96f430ee2..a6d0b08f2 100644 --- a/crates/gpu-gates/src/bin/generate_glm5next.rs +++ b/crates/gpu-gates/src/bin/generate_glm5next.rs @@ -799,7 +799,7 @@ mod cli { Some(lane_at(placement)?) }; let place = placement.name(); - let tier_batch = glm_place::tier_batch(placement, &inputs); + let tier_batch = glm_place::tier_batch(placement, &inputs)?; let machine = glm_machine( placement, tier_batch, diff --git a/crates/gpu-gates/src/bin/shared/glm5next_place.rs b/crates/gpu-gates/src/bin/shared/glm5next_place.rs index beccc1cab..39fa78350 100644 --- a/crates/gpu-gates/src/bin/shared/glm5next_place.rs +++ b/crates/gpu-gates/src/bin/shared/glm5next_place.rs @@ -39,10 +39,12 @@ pub fn parse(v: &str) -> Result { } /// The expert tier's prompt-batch bytes `place`'s plan reserves for the file -/// `inputs` describes (`model::arch::glm5next::place::tier_batch`), which -/// `Place::machine` takes; `None` under a placement with no tier card. -pub fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| model::arch::glm5next::place::tier_batch(inputs)) +/// described by `inputs` (`model::arch::glm5next::place::tier_batch`), +/// which `Place::machine` takes; `None` under a placement with no tier card. +pub fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| model::arch::glm5next::place::tier_batch(inputs)) + .transpose()?) } /// The experts `place`'s tier cards hold in its plan at `ctx` positions @@ -63,7 +65,7 @@ pub fn tier_experts( ) -> Result { let machine = glm_machine( place, - tier_batch(place, inputs), + tier_batch(place, inputs)?, slots, checkpoint_price(inputs)?, ctx, diff --git a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs index a9c0e2b89..5db7eebca 100644 --- a/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs +++ b/crates/gpu-gates/src/bin/shared/routed_legs/glm5next.rs @@ -172,7 +172,7 @@ fn machine_of( slots: usize, ) -> Result { let price = checkpoint_price(inputs)?; - let tier_batch = glmplace::tier_batch(place, inputs); + let tier_batch = glmplace::tier_batch(place, inputs)?; Ok(glm_machine(place, tier_batch, slots, price, ctx)?( inputs.model.layers, )) @@ -242,7 +242,6 @@ fn place_of(p: &PlaceError) -> Outcome { PlaceError::Placement(e) => return refused_of(e), PlaceError::PastOracle { .. } => "past-oracle", PlaceError::FrontOver { .. } => "front-over", - PlaceError::TierLeg { .. } => "tier-leg", PlaceError::Broken(_) | PlaceError::NextnCards { .. } | PlaceError::Slots { .. } => { return Outcome::Red(p.to_string()); } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs index a982330a6..f670c06ff 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/glm.rs @@ -503,7 +503,7 @@ impl Placed { (set, slots): (Option, usize), (images, side): (&Images<'_>, Option<&Side>), ) -> Result { - let tier_batch = glm_place::tier_batch(place, inputs); + let tier_batch = glm_place::tier_batch(place, inputs)?; // Every machine reserves its checkpoints at the slot context it is // laid for: the choice's probes each at their own, the load's at the // context it is granted; and the encoder's reserve, as the printed plan does. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs index e66507307..495f265e3 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs @@ -78,12 +78,16 @@ pub trait Family: Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s /// tier card ([`Tiered::tier_batch`]); a body that serves no tier card /// reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// The tags the model's description names for its think span, which the @@ -323,7 +327,10 @@ impl Tiered for Hybrid { F::think_tags(inputs).map(Some) } - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { F::tier_batch(place, inputs) } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs index 61af9a5b9..ce13c171c 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/solar2.rs @@ -182,8 +182,13 @@ impl Family for Solar { /// The tier's prompt-batch bytes of the file's widths ([`place::tier_batch`]), reserved on /// the placement's tier card and the host. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, bloomery_gpu_gates::GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(inputs)) + .transpose()?) } } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs index d2ea83afa..696f42120 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs @@ -252,12 +252,16 @@ pub trait Tiered: Seat + Sized + 'static { /// The expert tier's prompt-batch bytes the plan reserves on `place`'s tier card, for the file /// `inputs` describes; `None` under a placement with no tier card. A body that serves no tier /// card reserves none (the default), and its plan refuses the card by name. - fn tier_batch(place: Place, _inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then_some(TierBatchBytes { + fn tier_batch( + place: Place, + _inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()).then_some(TierBatchBytes { staging: 0, scratch: 0, host: 0, - }) + leg: None, + })) } /// Whether the plan of a load on `place` keeps the expert stream's ring out of the expert budget @@ -522,7 +526,7 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result, GateErro inputs.room = (PLAN_ROOM, HostRead::Given); // Plan (b′): the stage on the largest card, the next-largest an expert tier under the host tier, holding the tier's // prompt-batch bytes the file's widths give. - let machine = arm.machine(inputs.model.layers, place::tier_batch(&inputs.hp))?; + let machine = arm.machine(inputs.model.layers, place::tier_batch(&inputs)?)?; let place = PlanLevers::from_levers(&levers)?; let plan = inputs.plan(&machine, u64::try_from(ctx)?, &place)?; let mut m = Body::open_placed(split, &plan, &inputs, 0, levers.host())?; diff --git a/crates/gpu-glm5next/src/tier.rs b/crates/gpu-glm5next/src/tier.rs index e540f4ed2..aee07ed13 100644 --- a/crates/gpu-glm5next/src/tier.rs +++ b/crates/gpu-glm5next/src/tier.rs @@ -62,6 +62,7 @@ impl LegFamily for Glm { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: model::placement::accepts::Program::Glm5nextBody, } } diff --git a/crates/gpu-mimo2/src/card.rs b/crates/gpu-mimo2/src/card.rs index cf71fb1d1..506e518d3 100644 --- a/crates/gpu-mimo2/src/card.rs +++ b/crates/gpu-mimo2/src/card.rs @@ -27,6 +27,7 @@ fn stacks(l: usize) -> [String; 3] { /// MiMo's card experts: 256 routed experts, 8 a token. const FAMILY: CardFamily = CardFamily { what: "mimo2 CardExperts", + program: model::placement::accepts::Program::Mimo2Body, n_expert: N_EXPERT, n_used: N_USED, stacks, diff --git a/crates/gpu-minimaxm3/src/ffn.rs b/crates/gpu-minimaxm3/src/ffn.rs index 828b0897d..3850f6ba4 100644 --- a/crates/gpu-minimaxm3/src/ffn.rs +++ b/crates/gpu-minimaxm3/src/ffn.rs @@ -20,6 +20,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{ /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "minimaxm3 CardExperts", + program: model::placement::accepts::Program::MiniMaxM3Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/ffn.rs b/crates/gpu-solar2/src/ffn.rs index a0f1589d5..7be370e0a 100644 --- a/crates/gpu-solar2/src/ffn.rs +++ b/crates/gpu-solar2/src/ffn.rs @@ -17,6 +17,7 @@ pub use bloomery_gpu_deepseek41::shared_routed::{GATED_FROM, RoutedDims, back, f /// The card experts' family: the file's stack names and the router's counts. pub const CARD: CardFamily = CardFamily { what: "solar2 CardExperts", + program: model::placement::accepts::Program::Solar2Body, n_expert: N_EXPERT, n_used: N_USED, stacks: names::routed_stacks, diff --git a/crates/gpu-solar2/src/tier.rs b/crates/gpu-solar2/src/tier.rs index 071ce8d20..e23773d93 100644 --- a/crates/gpu-solar2/src/tier.rs +++ b/crates/gpu-solar2/src/tier.rs @@ -64,6 +64,7 @@ impl LegFamily for Solar2 { n_used: experts.n_used, n_embd: inputs.hp.n_embd, ff: experts.ff, + program: model::placement::accepts::Program::Solar2Body, } } diff --git a/crates/gpu/src/arch/musespark/body/tier.rs b/crates/gpu/src/arch/musespark/body/tier.rs index 724730577..cce7df8ac 100644 --- a/crates/gpu/src/arch/musespark/body/tier.rs +++ b/crates/gpu/src/arch/musespark/body/tier.rs @@ -30,6 +30,7 @@ use model::arch::models::{LayerSpec, Moe}; use model::arch::musespark::place::{self, PlanInputs}; use model::moe::HostLayer; use model::placement::ModelTensors; +use model::placement::accepts::Program; use model::r8file::R8Source; use crate::card_slots::{CardAcc, CardSlotKernels, ColPlaces}; @@ -87,6 +88,7 @@ impl LegFamily for MuseSpark { n_used: hp.n_used, n_embd: hp.latent, ff: hp.expert_width(), + program: Program::MuseSparkBody, } } diff --git a/crates/gpu/src/arch/qwen3moe/body38.rs b/crates/gpu/src/arch/qwen3moe/body38.rs index 904b2fa4d..4abec069e 100644 --- a/crates/gpu/src/arch/qwen3moe/body38.rs +++ b/crates/gpu/src/arch/qwen3moe/body38.rs @@ -1938,7 +1938,7 @@ impl Body38 { hybrid.prepare_batch(gpu.context(), host_cols)?; if tiered { let cards: Vec = tiers.iter().map(|t| t.card).collect(); - hybrid.check_tier_reserves(plan.machine, &cards)?; + hybrid.check_tier_reserves(plan, &cards)?; } // The PLE table's tier is the plan's: the host set built above holds // it on the host; on the NVMe tier its reader reads it. diff --git a/crates/gpu/src/arch/qwen3moe/tier_leg.rs b/crates/gpu/src/arch/qwen3moe/tier_leg.rs index c82babfa3..f5b383b89 100644 --- a/crates/gpu/src/arch/qwen3moe/tier_leg.rs +++ b/crates/gpu/src/arch/qwen3moe/tier_leg.rs @@ -1,8 +1,7 @@ //! The Qwen3-30B and Qwen3.6 expert tier: one card beside the stage card, //! hung through the common card leg ([`crate::host::leg_tier`]). What is -//! Qwen's here is the node family — the file's reading, `silu(g)·u`, and the -//! placed leg's arms — and the join of the tier's slots with the stage -//! card's. +//! Qwen's here is the node family — the file's reading and `silu(g)·u` — +//! and the join of the tier's slots with the stage card's. //! //! The join is the host leg's combine (`hostleg`) with the tier's slots in //! the same slot order. A slot the stage card holds is the stage card's down; @@ -22,7 +21,7 @@ use gguf::Split; use model::arch::models::{Act as SpecAct, LayerSpec, Moe}; use model::moe::HostLayer; use model::placement::ModelTensors; -use model::placement::accepts::{self, LegArms}; +use model::placement::accepts::Program; use model::r8file::R8Source; use crate::fault::{FaultSink, FaultSite}; @@ -51,13 +50,6 @@ const _: () = assert!(model::arch::qwen35moe::place::LEG_TIER_CHUNK == LEG_CHUNK /// Threads per block of both join entries. const THREADS: u32 = 256; -/// A gate or up stack the placed leg does not run. -const GATE_UP: &str = "a resident routed gate or up stack of a type the placed leg runs \ - (`accepts::prompt_leg_gate_up`)"; - -/// A down stack the placed leg does not run. -const DOWN: &str = - "a resident routed down stack of a type the placed leg runs (`accepts::prompt_leg_down`)"; /// The routed activation the tier runs: `silu(g)·u`. A clamped SwiGLU, and /// any other activation, is refused by name. @@ -78,18 +70,6 @@ fn qwen_act(moe: &Moe) -> Result { } } -/// The placed leg's arms ([`accepts::leg_arms`] of [`accepts::Program::Qwen3moeBody`]). -/// -/// # Panics -/// -/// When that program names no card-leg arms. -fn qwen_arms() -> (LegArms, (&'static str, &'static str)) { - let arms = match accepts::leg_arms(accepts::Program::Qwen3moeBody) { - Some(arms) => arms, - None => panic!("{WHAT}: Program::Qwen3moeBody names no card-leg arms"), - }; - (arms, (GATE_UP, DOWN)) -} /// The host views of the routed layers `run`, the refusal naming the run as /// `word`'s. `n_layer` is the description's layer count, so a trunk layer is @@ -154,6 +134,7 @@ impl LegFamily for Qwen3 { n_used: e.n_used, n_embd: inputs.hp.n_embd, ff: e.ff, + program: Program::Qwen3moeBody, } } @@ -175,9 +156,6 @@ impl LegFamily for Qwen3 { qwen_act(moe) } - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - Some(qwen_arms()) - } } /// Qwen3.6's facts for the common tier node service. The description's @@ -211,6 +189,7 @@ impl LegFamily for Qwen35 { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: Program::Qwen35Body, } } @@ -232,9 +211,6 @@ impl LegFamily for Qwen35 { qwen_act(moe) } - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - Some(qwen_arms()) - } } /// Qwen3-30B's tier node service ([`leg_tier::node_family`]). diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 8ca374858..7c53831d5 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -41,9 +41,11 @@ use std::mem::ManuallyDrop; use cuda_core::{CudaStream, DeviceBuffer}; use model::arch::models::{Ffn, LayerSpec}; +use model::placement::accepts::{self, LegArms, Program}; +use model::placement::kernels::{self, DownForms}; use model::arch::place_hybrid::RingForms; -use model::placement::kernels::{self, DownAct, DownForms}; +use model::placement::kernels::DownAct; use crate::card_leg::{ self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, @@ -57,6 +59,12 @@ use crate::ring_gemm::RingGemm; use crate::weights::Weights; use crate::{DeviceTensor, Gpu, GpuError}; +/// The selected program, rather than a union of unrelated launches, admits stacks. +const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected card gate·up arm runs", + "a resident routed down stack the selected card down arm runs", +); + /// What a family fixes of its card experts. #[derive(Clone, Copy)] pub struct CardFamily { @@ -68,6 +76,8 @@ pub struct CardFamily { pub n_used: usize, /// Layer `l`'s gate, up and down stacks' names. pub stacks: fn(usize) -> [String; 3], + /// The program whose selected gate·up and down launches run the stacks. + pub program: Program, } /// The kernels a run launches: the card leg's (its sum stage included), and @@ -75,6 +85,7 @@ pub struct CardFamily { struct CardKernels { leg: CardLeg, places: CardSlotKernels, + arms: LegArms, } impl CardFamily { @@ -266,8 +277,7 @@ pub struct CardExperts { ff: usize, /// The model's width. n: usize, - /// The forms the card layers' downs read, which every run's buffers hold - /// ([`card_downs`]). + /// The union of the resident layers' selected down activation forms. downs: DownForms, /// The one-token step's buffers. step: CardBufs, @@ -322,6 +332,10 @@ impl CardExperts { tiers: bool, ) -> Result { let what = family.what; + let arms = accepts::leg_arms(family.program).ok_or(GpuError::state( + what, + "the selected program's card leg arms", + ))?; let mut per = Vec::with_capacity(layers.len()); let mut tier_k = Vec::with_capacity(layers.len()); let mut downs = DownForms::NONE; @@ -362,9 +376,13 @@ impl CardExperts { per.push(None); continue; } - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; stacks.check(n_card, family.shape(n, ff, 1))?; - downs = downs.with(down_act(what, &layer, stacks.down_ty())?); + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + what, + "the selected card down's activation form", + ))?; + downs = downs.with(act); per.push(Some(layer)); } let downs = card_downs(downs); @@ -373,6 +391,7 @@ impl CardExperts { k: CardKernels { leg: CardLeg::load(gpu)?, places: CardSlotKernels::load(gpu.context())?, + arms, }, layers: per, tier: tier_k, @@ -804,7 +823,7 @@ impl CardKernels { io: &Io<'_>, sel: &DeviceBuffer, ) -> Result<(), GpuError> { - let stacks = LegStacks::of(w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer: l, stacks: &stacks, @@ -830,22 +849,328 @@ impl CardKernels { #[cfg(test)] mod tests { - use super::{DownAct, DownForms, card_downs}; + use super::*; + use gguf::GgmlType; + use gguf::write::{Layout, TensorDecl, Writer}; + use model::arch::models::{self, Mixer, Moe, Residual, Router, RouterNorm, Score}; + + use crate::card_leg::tests::stack_bytes; + use crate::host::leg_tier::LegTier; + use crate::host::tier::{TierBlock, TierExperts, TierInput, TierIo, TierSet}; + use crate::hybrid::HOST; + + const N: usize = 256; + const FF: usize = 256; + const USED: usize = 8; + const EXPERTS: usize = 16; + const CARD: usize = 4; + const CAP: usize = 3; + const FAMILY: CardFamily = CardFamily { + what: "mixed card wrapper", + n_expert: EXPERTS, + n_used: USED, + stacks: model::arch::names::routed_stacks, + program: Program::Qwen35Body, + }; + + fn layers(n: usize) -> Vec { + vec![ + LayerSpec { + mixer: Mixer::None, + ffn: Ffn::Moe(Moe { + experts: EXPERTS as u32, + top_k: USED as u32, + expert_ff: FF as u32, + latent: None, + act: models::Act::SwiGlu { limit: None }, + router: Router { + score: Score::Sigmoid, + bias: false, + norm: RouterNorm::SelectedSum, + scale: 1.0, + hash: false, + }, + shared: None, + }), + residual: Residual::Plain, + extras: Vec::new(), + }; + n + ] + } + + fn weights(gpu: &Gpu, triples: &[[GgmlType; 3]]) -> Weights { + let mut decls = Vec::new(); + let mut data = Vec::new(); + for (l, triple) in triples.iter().enumerate() { + for (part, (name, &ty)) in (FAMILY.stacks)(l).into_iter().zip(triple).enumerate() { + let (rows, k) = if part == 2 { (N, FF) } else { (FF, N) }; + let bytes = stack_bytes(ty, CARD * rows, k, 7 + (3 * l + part) as u64); + decls.push(TensorDecl { + name, + dims: vec![k as u64, rows as u64, CARD as u64], + type_id: ty.as_u32(), + nbytes: bytes.len() as u64, + }); + data.push(bytes); + } + } + static CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let call = CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "bloomery-mixed-wrappers-{}-{call}.gguf", + std::process::id() + )); + let layout = Layout::new(&[], decls.clone()).expect("a layout"); + let file = std::io::BufWriter::new(std::fs::File::create(&path).expect("a file")); + let mut writer = Writer::new(file, layout).expect("a writer"); + for (decl, bytes) in decls.iter().zip(data) { + writer.tensor(&decl.name, &bytes).expect("a tensor"); + } + writer.finish().expect("a finished file"); + let file = gguf::Split::open(&path).expect("the file"); + let weights = Weights::load_where(gpu.stream(), &file, |_| true).expect("resident stacks"); + std::fs::remove_file(path).expect("remove the fixture"); + weights + } + + fn assert_bits(got: &[f32], want: &[f32]) { + assert_eq!(got.len(), want.len()); + for (i, (&g, &w)) in got.iter().zip(want).enumerate() { + assert!(g.is_finite() && w.is_finite(), "lane {i}: {g} vs {w}"); + assert_eq!(g.to_bits(), w.to_bits(), "lane {i}: {g} vs {w}"); + } + } - /// A load's leg buffers hold the forms its card layers' downs read — the - /// 32-value form for NVFP4's, both for a mix — and the 128-value form on a - /// load with no card layer. Mutant: the buffers back on the 128-value form - /// alone (an NVFP4 layer's down finds no q8 blocks of 32 and is refused at - /// its first run). + /// Each real wrapper uses its resident layers' union, not a model-wide + /// quant: both one-form loads and mixed loads conserve bytes and execute + /// step, pass, batch and tier chunks as the same common leg. Host slots + /// stay untouched on the tier, including a chunk's short final run. #[test] - fn the_leg_buffers_hold_the_card_downs_forms() { - let q8 = DownForms::of(DownAct::Q8Act); - let b32 = DownForms::of(DownAct::Q8Blocks32); - assert_eq!(card_downs(DownForms::NONE), q8); - assert_eq!(card_downs(q8), q8); - assert_eq!(card_downs(b32), b32); - assert!(!card_downs(b32).holds(DownAct::Q8Act)); - let both = q8.with(DownAct::Q8Blocks32); - assert_eq!(card_downs(both), both); + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_mixed_card_and_tier_wrappers_conserve_forms_and_rows() { + use GgmlType::{IQ3_S, IQ4_XS, Q5_K, Q6_K, Q8_0}; + let gpu = Gpu::with_device(0).expect("a Gpu on device 0"); + let stream = gpu.stream(); + let leg = CardLeg::load(&gpu).expect("the reference leg"); + for triples in [ + vec![[IQ3_S, IQ3_S, Q5_K], [IQ4_XS; 3]], + vec![[Q5_K, Q5_K, Q8_0]], + vec![[Q5_K, Q5_K, Q6_K]], + vec![ + [IQ3_S, IQ3_S, Q5_K], + [Q5_K, Q5_K, Q8_0], + [Q5_K, Q5_K, Q6_K], + [IQ4_XS; 3], + ], + ] { + let w = weights(&gpu, &triples); + let specs = layers(triples.len()); + let map = SlotMap::prefix(0..triples.len(), EXPERTS, CARD).expect("a map"); + let mut card = CardExperts::new(&gpu, &w, FAMILY, &specs, &map, (N, FF), CAP) + .expect("the card wrapper"); + let mut batch = card.batch(&gpu, CAP).expect("the batch"); + assert_eq!(card.batch_bytes(CAP), batch.bytes()); + let forms = triples.iter().fold(DownForms::NONE, |forms, tys| { + forms.with(kernels::down(tys[2]).expect("a down").1) + }); + let counted = |cap| { + kernels::leg_bytes(cap as u64, USED as u64, N as u64, FF as u64, forms) as usize + }; + assert_eq!( + card.bytes(), + counted(1) + counted(CAP) + 2 * size_of::() * (1 + CAP) * N, + ); + assert_eq!( + batch.bytes(), + counted(CAP) + 2 * size_of::() * CAP * N + size_of::() * CAP * USED, + ); + let shape = FAMILY.shape(N, FF, 2); + let set = TierSet::new( + 0..triples.len(), + EXPERTS, + vec![(0..CARD as u32).collect(); triples.len()], + ) + .expect("the tier set"); + let mut tier = LegTier::new( + &gpu, &specs, &set, &w, shape, FAMILY.program, + |moe| Ok(Act::SwigluClamp { limit: moe.act.swiglu_limit() }), + ) + .expect("the tier wrapper"); + assert_eq!(tier.block_bytes(), counted(shape.cap)); + let host_x: Vec = (0..CAP * N).map(|i| (i % 71) as f32 / 71.0 - 0.5).collect(); + let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); + let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); + let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); + let route = + DeviceBuffer::from_host(stream, &vec![0.125f32; CAP * USED]).expect("weights"); + let host_sel: Vec = (0..CAP * USED) + .map(|i| { + if i % USED == USED - 1 { + HOST + } else { + (i % CARD) as u32 + } + }) + .collect(); + let sel = DeviceBuffer::from_host(stream, &host_sel).expect("the places"); + let ids = DeviceBuffer::from_host( + stream, + &host_sel + .iter() + .map(|&s| if s == HOST { (EXPERTS - 1) as u32 } else { s }) + .collect::>(), + ) + .expect("the ids"); + let places = DeviceTensor::upload(stream, &map.stage_view(), triples.len(), EXPERTS) + .expect("the map's card view"); + let act = crate::Q8Act::with_tier_cols(stream, CAP, N).expect("the tier block's form"); + for (l, tys) in triples.iter().enumerate() { + let names = (FAMILY.stacks)(l); + let stacks = LegStacks::of_arms( + &w, + names.each_ref().map(String::as_str), + accepts::leg_arms(FAMILY.program).expect("the selected arms"), + STACK_ROLES, + ) + .expect("the reference stacks"); + let mut reference = LegBufs::for_down( + stream, + FAMILY.shape(N, FF, CAP), + kernels::down(tys[2]).expect("a down").1, + ) + .expect("the reference buffers"); + let mut acc = DeviceBuffer::zeroed(stream, CAP * N).expect("the reference sum"); + for m in [1, CAP] { + let io = Io { + normed: &x, + weights: &route, + x1: &x1, + m, + }; + let rule = Act::SwigluClamp { limit: 0.0 }; + leg.enqueue( + &gpu, + &mut reference, + &LegRun { + layer: l, + stacks: &stacks, + normed: &x, + m, + sel: &sel, + n_card: CARD, + rule, + }, + Some(LegSum { + w: &route, + acc: &mut acc, + }), + ) + .expect("the reference"); + let sums = acc.to_host_vec(stream).expect("the reference sums"); + let want: Vec = sums[..m * N] + .iter() + .zip(&base) + .map(|(&a, &b)| a + b) + .collect(); + let got = if m == 1 { + card.enqueue_step(&gpu, &w, l, rule, &io, &sel) + .expect("step"); + card.pre() + } else { + card.enqueue_pass(&gpu, &w, l, rule, &io, &sel) + .expect("pass"); + card.pass_pre().expect("pass output") + }; + assert_bits(&got.to_host_vec(stream).expect("output")[..m * N], &want); + card.enqueue_rows( + &gpu, + &w, + &mut batch, + l, + rule, + &io, + (&ids, &places, l * EXPERTS), + ) + .expect("batch"); + assert_bits( + &batch.pre().to_host_vec(stream).expect("batch output")[..m * N], + &want, + ); + let mut down = + DeviceBuffer::from_host(stream, &vec![1.0e30f32; CAP * USED * N]) + .expect("the tier output"); + if m == 1 { + tier.enqueue_layer( + &gpu, + &w, + l, + TierIo { + act: TierInput::F32(&x), + sel: &sel, + rows: &mut down, + cols: m, + }, + ) + .expect("tier step"); + } else { + tier.enqueue_block( + &gpu, + &w, + l, + TierBlock { + x: &x, + act: &act, + sel: &sel, + cols: m, + down: &mut down, + }, + ) + .expect("tier chunks"); + } + let got = down.to_host_vec(stream).expect("tier output"); + let want = reference + .down() + .to_host_vec(stream) + .expect("reference rows"); + for (s, &place) in host_sel[..m * USED].iter().enumerate() { + if place == HOST { + assert!(got[s * N..(s + 1) * N].iter().all(|&v| v == 1.0e30)); + } else { + assert_bits(&got[s * N..(s + 1) * N], &want[s * N..(s + 1) * N]); + } + } + } + } + assert!(gpu.take_fault().expect("the fault word").is_none()); + } + for (triple, program) in [ + ([IQ3_S, Q5_K, Q8_0], Program::Qwen35Body), + ([Q5_K, Q5_K, Q6_K], Program::Solar2Body), + ] { + let w = weights(&gpu, &[triple]); + let specs = layers(1); + let map = SlotMap::prefix(0..1, EXPERTS, CARD).expect("a map"); + let family = CardFamily { program, ..FAMILY }; + assert!( + matches!( + CardExperts::new(&gpu, &w, family, &specs, &map, (N, FF), CAP), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + let set = + TierSet::new(0..1, EXPERTS, vec![(0..CARD as u32).collect()]).expect("a tier set"); + assert!( + matches!( + LegTier::new( + &gpu, &specs, &set, &w, family.shape(N, FF, 2), program, + |moe| Ok(Act::SwigluClamp { limit: moe.act.swiglu_limit() }), + ), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + } } } diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index e87c14ce0..6aa1c9afe 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -629,13 +629,14 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: kernels::card_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: accepts::planes_leg_down, }, (GATE_UP, DOWN), ) } + /// [`LegStacks::of`] for a body that loads every tensor in the format /// [`gguf::card::CardFormat::of`] names (`Weights::load`), whose downs may /// also be Q5_0 in the `pack_q5_0` layout ([`accepts::resident_down_runs`]). @@ -647,8 +648,8 @@ impl<'a> LegStacks<'a> { w, names, LegArms { - gate_up: kernels::card_gate_up_runs, - down: accepts::resident_down_runs, + gate_up: accepts::planes_leg_gate_up, + down: |ty| accepts::resident_down_runs(ty) || kernels::card_planes_runs(ty), }, (GATE_UP, DOWN), ) @@ -668,9 +669,9 @@ impl<'a> LegStacks<'a> { ) -> Result, GpuError> { let [gate, up, down] = names; let (gate, up, down) = ( - LegStack::of(w, gate, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, up, arms.gate_up, kernels::card_planes_runs, roles.0)?, - LegStack::of(w, down, arms.down, kernels::card_planes_runs, roles.1)?, + LegStack::of(w, gate, arms.gate_up, arms.gate_up, roles.0)?, + LegStack::of(w, up, arms.gate_up, arms.gate_up, roles.0)?, + LegStack::of(w, down, arms.down, arms.down, roles.1)?, ); if up.ty != gate.ty { return Err(GpuError::Tensor { diff --git a/crates/gpu/src/host/batch.rs b/crates/gpu/src/host/batch.rs index 5a69a4707..29dc6c687 100644 --- a/crates/gpu/src/host/batch.rs +++ b/crates/gpu/src/host/batch.rs @@ -29,7 +29,7 @@ use crate::graph::{MappedHost, cu}; use crate::tensor::WindowMut; use crate::{Gpu, GpuError, Q8Act}; use cuda_core::{CudaContext, CudaEvent, CudaStream, DeviceBuffer, PinnedHostBuffer, sys}; -use model::placement::Machine; +use model::placement::Plan; use model::placement::workstation::{TIER_BATCH_HOST_RESERVE, TIER_BATCH_RESERVE}; use model::{Tensor2, Tensor2View}; use std::mem::size_of; @@ -1079,18 +1079,19 @@ impl HostTier { Ok(()) } - /// The expert tiers' prompt-batch bytes against the reserves `machine`'s + /// The expert tiers' prompt-batch bytes against the selected reserves the /// plan carries for them: on each tier's card — tier `t`'s is index - /// `cards[t]` of [`Machine::all_cards`] — its batch staging and the + /// `cards[t]` of [`model::placement::Machine::all_cards`] — its batch staging and the /// architecture's block scratch ([`TierCard::block_bytes`]) against that - /// card's [`TIER_BATCH_RESERVE`] row; on the host, the tier legs' rows and + /// card's selected [`TIER_BATCH_RESERVE`] row in `plan`; on the host, the tier legs' rows and /// places against the host's [`TIER_BATCH_HOST_RESERVE`] row. A card list /// that is not one card a tier, a missing row, a row named twice, or a /// difference is refused by name, naming the reserve. Load-time only, once /// [`HostTier::prepare_batch`] has made the legs. Step-only tiers make /// no batch leg ([`HostTier::prepare_tier_batch`]) and none is checked. - pub fn check_tier_reserves(&self, machine: &Machine, cards: &[usize]) -> Result<(), GpuError> { + pub fn check_tier_reserves(&self, plan: &Plan<'_>, cards: &[usize]) -> Result<(), GpuError> { const WHAT: &str = "HostTier::check_tier_reserves"; + let machine = plan.machine; if self.tiers.iter().any(|t| !t.supports_blocks()) { return Ok(()); } @@ -1124,6 +1125,7 @@ impl HostTier { &card.reserves, TIER_BATCH_RESERVE, port.tier_device_bytes(i) + t.block_bytes()?, + Some(index), )); } rows.push(( @@ -1131,15 +1133,27 @@ impl HostTier { &machine.host().reserves, TIER_BATCH_HOST_RESERVE, port.tier_host_bytes(), + None, )); - for (on, reserves, name, got) in rows { + for (on, reserves, name, got, card) in rows { let want = match reserves .iter() .filter(|(n, _)| n == name) .collect::>() .as_slice() { - [(_, b)] => *b, + [(_, b)] => match card { + Some(c) => plan + .card_reserve(c, name) + .map_err(|e| GpuError::plan(WHAT, e))? + .ok_or_else(|| { + GpuError::shape( + WHAT, + format!("the plan's selected reserve {name:?} on {on} is invalid"), + ) + })?, + None => *b, + }, named => { return Err(GpuError::shape( WHAT, diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 0762476e1..04572a78a 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -9,7 +9,7 @@ //! The tier's layer is the card leg ([`CardLeg::enqueue_rows`]) over the //! tier's stacks: the q8_1 of the activation, the gate·up `_sel` with the //! routed activation the family names (clamped SwiGLU unless it names -//! another), the q8_1 of the tier slots' columns and the down +//! another), the selected down forms of the tier slots' columns and the down //! `_sel` into the tier's rows; over a prompt batch's block, the same run //! chunk by chunk of `shape.cap` tokens. Each slot's down output is the one //! the stage card would write for it, so the card sum over both cards' slots @@ -30,7 +30,7 @@ use model::ModelError; use model::arch::models::{Ffn, LayerSpec, Moe}; use model::arch::names::routed_stacks; use model::moe::HostLayer; -use model::placement::accepts::LegArms; +use model::placement::accepts::{self, LegArms, Program}; use model::placement::kernels::{self, DownAct, DownForms}; use model::placement::{ModelTensors, Plan}; use model::r8file::R8Source; @@ -53,6 +53,12 @@ const WHAT: &str = "LegTier"; /// What the stage card's side of the tier layers names in its errors. const SIDE: &str = "TierSide"; +/// The selected program's launches admit the same stacks on both cards. +const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected tier gate·up arm runs", + "a resident routed down stack the selected tier down arm runs", +); + /// The tier the stage card's tier entries serve: they hand one tier its /// image and join one tier's rows, so a load holds one tier card /// ([`crate::host::refuse_tier_count`]). @@ -173,9 +179,8 @@ impl TierLayer { /// leg over the tier's stacks into the tier's rows; over a prompt batch's /// block, the same chunk by chunk. Its scratch, one [`LegBufs`] for /// `shape.cap` tokens that a step's one token and each chunk share, is made -/// at load. The activation and the stack arms are the family's -/// ([`LegFamily::tier_act`], [`LegFamily::tier_arms`]); a family that names -/// no arms uses the card leg's own. +/// at load. The activation is the family's ([`LegFamily::tier_act`]); the +/// selected program names the stack arms. pub struct LegTier { leg: CardLeg, bufs: LegBufs, @@ -183,30 +188,33 @@ pub struct LegTier { /// Per layer of `layers`, `None` off the tier. cfg: Vec>, shape: LegShape, - /// The family's stack arms, or the card leg's own when it named none. - arms: Option<(LegArms, (&'static str, &'static str))>, + arms: LegArms, } impl LegTier { /// The tier's computation on `gpu` over `set`, whose stacks `w` holds, /// for the description `layers`, through the leg of `shape` (`cap` tokens /// of `slots_per_col` slots a run, `n` wide, experts `ff` wide). The - /// activation is `F`'s ([`LegFamily::tier_act`]) and the stacks are read - /// by `F`'s arms ([`LegFamily::tier_arms`]; the card leg's own when the - /// family names none). A tier layer whose three stacks are absent, of a - /// type those arms do not read, or of other rows than its experts', and - /// a layer off the tier with resident stacks, are refused by name. The + /// activation is supplied by the family's [`LegFamily::tier_act`] and the + /// selected program names the stack arms. A tier layer whose three stacks + /// are absent, of a type those arms do not read, or of other rows than its + /// experts', and a layer off the tier with resident stacks, are refused by name. The /// leg's buffers hold the columns in the forms the tier layers' downs /// read ([`kernels::down`]), the 128-value form on a tier of no layer. /// Load-time only. - pub fn new( + pub fn new( gpu: &Gpu, layers: &[LayerSpec], set: &TierSet, w: &Weights, shape: LegShape, + program: Program, + activation: fn(&Moe) -> Result, ) -> Result { - let arms = F::tier_arms(); + let arms = accepts::leg_arms(program).ok_or(GpuError::state( + WHAT, + "the selected program's card leg arms", + ))?; let run = set.layers(); let mut cfg = Vec::with_capacity(run.len()); let mut downs = DownForms::NONE; @@ -232,17 +240,16 @@ impl LegTier { }; let layer = TierLayer { k, - rule: F::tier_act(m)?, + rule: activation(m)?, names, }; - let stacks = stacks_of(arms, w, layer.names())?; + let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; stacks.check(k, shape)?; - let (_, form) = kernels::down(stacks.down_ty()).ok_or_else(|| GpuError::Tensor { - what: WHAT, - name: layer.names[2].clone(), - need: "a down stack of a type the kernel table names a down entry for", - })?; - downs = downs.with(form); + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + WHAT, + "the selected tier down's activation form", + ))?; + downs = downs.with(act); cfg.push(Some(layer)); } if downs == DownForms::NONE { @@ -323,7 +330,7 @@ impl TierExperts for LegTier { .. } = *shape; let tl = layer_of(layers, cfg, layer, W)?; - let stacks = stacks_of(arms, weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), *arms, STACK_ROLES)?; for c0 in (0..io.cols).step_by(cap) { let cn = cap.min(io.cols - c0); let slots = cn * slots_per_col; @@ -367,7 +374,7 @@ impl TierExperts for LegTier { .to_string(), }); }; - let stacks = stacks_of(arms, weights, tl.names())?; + let stacks = LegStacks::of_arms(weights, tl.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer, stacks: &stacks, @@ -482,6 +489,8 @@ pub struct LegWidths { pub n_embd: usize, /// A routed expert's width. pub ff: usize, + /// The program whose card leg runs these experts. + pub program: Program, } impl LegWidths { @@ -561,12 +570,6 @@ pub trait LegFamily { }) } - /// The arms a tier layer's stacks are read by, with the gate-or-up - /// refusal's words and the down's, or `None` for the card leg's own - /// arms ([`LegStacks::of`]). - fn tier_arms() -> Option<(LegArms, (&'static str, &'static str))> { - None - } } /// The family `F`'s tier node service: the placement tensors of a file, the @@ -609,13 +612,16 @@ fn node_experts( open: &NodeOpen, ) -> Result, GpuError> { let inputs = node_inputs::(file, open)?; - let shape = F::widths(&inputs).shape(F::CHUNK); - Ok(Box::new(LegTier::new::( + let widths = F::widths(&inputs); + let shape = widths.shape(F::CHUNK); + Ok(Box::new(LegTier::new( gpu, F::layers(&inputs), &open.set, w, shape, + widths.program, + F::tier_act, )?)) } @@ -696,5 +702,5 @@ pub fn hang_tiers( let planned: Vec = hang.tiers.iter().map(|t| t.card).collect(); hybrid.attach_tiers(cards, gpu)?; hybrid.prepare_batch(gpu.context(), hang.cols)?; - hybrid.check_tier_reserves(plan.machine, &planned) + hybrid.check_tier_reserves(plan, &planned) } diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index 5c7f2a3a3..282a1157d 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -321,13 +321,13 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let reserve = plan - .machine - .card(self.card) - .ok_or_else(|| { - GpuError::shape(WHAT, "the projected tier card is absent from the plan") - })? - .set_aside_bytes() + let totals = plan.cards.get(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + let reserve = totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, diff --git a/crates/model/src/arch/deepseek41/kv.rs b/crates/model/src/arch/deepseek41/kv.rs index d7fd1ebb7..8a45d8fd6 100644 --- a/crates/model/src/arch/deepseek41/kv.rs +++ b/crates/model/src/arch/deepseek41/kv.rs @@ -263,6 +263,7 @@ mod tests { let machine = Machine { cards: vec![Card { name: "pool".into(), + tier_leg: None, device: None, usable_bytes: pool, context_bytes: 0, diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 7be86b816..d497ca063 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -146,53 +146,26 @@ fn down_forms(downs: impl Iterator) -> DownForms { .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)) } -/// The down forms of the expert tier's leg buffers: those of every routed layer the card experts -/// run ([`carded_layers`]), the tier holding experts of any of them, tiles' layers included (the -/// tier runs the card leg over all of them); the 128-value form when the file has none, the form -/// the load's leg buffers default to. -fn tier_downs(model: &ModelTensors) -> DownForms { - let forms = down_forms(carded_layers(model).map(|(_, [_, _, d])| d)); - if forms == DownForms::NONE { - DownForms::of(DownAct::Q8Act) - } else { - forms - } -} - -/// The first routed layer of `model` the card experts run that the expert -/// tier card's plan does not place on its leg, with its stacks' types: the -/// plan places the file's blocks there over the 128-value form, each part by -/// its arm ([`kernels::card_gate_up_runs`] for the gate and up, -/// [`kernels::card_q8act_down_runs`] for the down, a Q6_K down among them), -/// so a plan that hangs a tier card is refused it ([`PlaceError::TierLeg`]). -fn off_tier_layer(model: &ModelTensors) -> Option<(usize, [GgmlType; 3])> { - carded_layers(model).find(|&(_, [g, u, d])| { - !(kernels::card_gate_up_runs(g) - && kernels::card_gate_up_runs(u) - && kernels::card_q8act_down_runs(d)) - }) -} /// Tokens a chunk of the tier's prompt-batch service runs at most: the /// stage card's chunk of the card experts' launches, which the body binds /// to its own constant. pub const TIER_CHUNK: u64 = 8; -/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for the file `inputs` describes and -/// blocks of the host union's columns: the tier card's staging, one run of the card leg's buffers -/// over [`TIER_CHUNK`] tokens in each form the tier layers' downs read ([`tier_downs`]; the -/// card leg's own formula, [`workstation::tier_leg_batch_bytes_of`]) and the host's rows and -/// places, which the plan reserves on the tier card and the host and the load checks its -/// allocations against. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of +/// the host union's columns: the tier card's staging, the chunked card +/// path's scratch ([`workstation::tier_chunk_scratch_bytes`]) and the host's +/// rows and places, which the plan reserves on the tier card and the host +/// and the load checks its allocations against. +pub fn tier_batch(inputs: &PlanInputs) -> Result { let hp = &inputs.hp; - workstation::tier_leg_batch_bytes_of( + workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, - (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), - tier_downs(&inputs.model), + TIER_CHUNK, + crate::moe::UNION_MAX_COLS as u64, + crate::placement::accepts::leg_down_forms(Program::Glm5nextBody, &inputs.model), ) } @@ -448,16 +421,7 @@ pub enum PlaceError { /// A plan of no resident sequence: a load serves at least one. #[error("a plan of {slots} resident sequences: a load serves at least one")] Slots { slots: usize }, - /// A machine that hangs an expert tier card beside a file with a card - /// layer the plan does not place on the tier's leg (NVFP4's planes, a down - /// over the 32-value form): the plan places the file's blocks over the - /// 128-value form alone there. - #[error( - "layer {layer}: routed stacks {tys:?} run on the stage card's leg, and the expert tier \ - card's leg reads the file's blocks over 128-value q8_1 columns alone; plan it without \ - the tier card" - )] - TierLeg { layer: usize, tys: [GgmlType; 3] }, + } impl PlanRefusal for PlaceError { @@ -683,7 +647,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(lanes); let terms = self.seq_terms(&kv, None); let n = slots as u64; @@ -730,17 +693,6 @@ impl PlanInputs { Ok(plan) } - /// Refused by name ([`PlaceError::TierLeg`]) when `machine` hangs an - /// expert tier card and a card layer of the file is one the tier's leg - /// does not run. - fn refuse_off_tier(&self, machine: &Machine) -> Result<(), PlaceError> { - match off_tier_layer(&self.model) { - Some((layer, tys)) if machine.expert_tiers().next().is_some() => { - Err(PlaceError::TierLeg { layer, tys }) - } - _ => Ok(()), - } - } } /// Card bytes the NextN walk's arena may hold beside the plan's tensors @@ -861,6 +813,7 @@ fn nextn_machine() -> Machine { Machine { cards: vec![Card { name: "nextn".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -1045,7 +998,6 @@ impl PlanInputs { if slots == 0 { return Err(PlaceError::Slots { slots }); } - self.refuse_off_tier(machine)?; let kv = self.kv.with_lanes(KdaLanes::Two); let terms = self.seq_terms(&kv, Some(nextn)); let n = slots as u64; @@ -1622,6 +1574,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 1 << 20, @@ -2275,44 +2228,74 @@ mod tests { } } - /// A machine that hangs an expert tier card is refused, by layer - /// and types, a file with a card layer the tier's leg does not run - /// (an NVFP4 layer: its planes and 32-value form); without the tier - /// card the same file plans, the NVFP4 layer's experts on the stage - /// card, and a K-quant file is not refused with it - /// ([`node_ranges_use_the_common_loaded_assignment`]). Mutants: the - /// refusal dropped, which leaves the tier's load to refuse the - /// layer's stacks; the body's plan rule left on the common leg's, which - /// keeps the NVFP4 layer on the host, off the stage card's planes. + /// The tier reserves precisely the selected layers' down forms, + /// including a mixed 128/32-value load; a gate/up mismatch remains + /// host-only and cannot add a form. #[test] - fn a_tier_card_is_refused_a_layer_its_leg_does_not_run() { + fn tier_scratch_follows_selected_mixed_layers() { + use super::super::TIER_CHUNK; + use crate::placement::kernels::{self, DownAct, DownForms}; + use GgmlType::{IQ3_S, NVFP4, Q5_K, Q8_0}; let mut inputs = routed(64 << 30); - for t in inputs + inputs .model .tensors - .iter_mut() - .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(1)) - { - t.ty = GgmlType::NVFP4; - t.file_bytes = t.dims.iter().product::() / 64 * 36; - } - let plain = unified(64 << 30); - let mut machine = plain.clone(); - let mut tier = machine.cards[0].clone(); - tier.name = "remote".into(); - tier.head = false; - tier.layers = 0..0; - machine.tiers.push(tier); - match inputs.plan(&machine, CTX, &PlanLevers::default()) { - Err(PlaceError::TierLeg { layer, tys }) => { - assert_eq!((layer, tys), (1, [GgmlType::NVFP4; 3])); + .retain(|t| t.role != Role::RoutedExperts || t.layer.is_some_and(|l| l < 2)); + let set = |inputs: &mut PlanInputs, l, tys: [GgmlType; 3]| { + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match crate::placement::ExpertPart::of_name(&t.name) { + Some(crate::placement::ExpertPart::Gate) => 0, + Some(crate::placement::ExpertPart::Up) => 1, + Some(crate::placement::ExpertPart::Down) => 2, + None => unreachable!(), + }; + t.ty = tys[part]; } - other => panic!("{:?}", other.map(|p| p.n_l.clone())), + }; + for (layers, forms) in [ + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q5_K]], + DownForms::of(DownAct::Q8Act), + ), + ( + [[Q5_K, Q5_K, Q8_0], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [NVFP4; 3]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [Q5_K, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32), + ), + ( + [[IQ3_S, IQ3_S, Q5_K], [IQ3_S, Q5_K, Q8_0]], + DownForms::of(DownAct::Q8Act), + ), + ] { + for (l, tys) in layers.into_iter().enumerate() { + set(&mut inputs, l, tys); + } + let bytes = super::super::tier_batch(&inputs).expect("tier bytes"); + let hp = &inputs.hp; + assert_eq!( + bytes.scratch, + kernels::leg_bytes( + TIER_CHUNK, + hp.n_used as u64, + hp.n_embd as u64, + hp.expert_ff as u64, + forms, + ), + "{layers:?}", + ); } - let p = inputs - .plan(&plain, CTX, &PlanLevers::default()) - .expect("the plan without the tier card"); - assert!(p.n_l[1] > 0, "the NVFP4 layer on the card: {:?}", p.n_l); } #[test] diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 14ad7aa30..be65dc527 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -120,13 +120,14 @@ pub const TIER_CHUNK: u64 = 16; /// and places ([`workstation::tier_leg_batch_bytes`]). The plan reserves them on the tier card and /// the host, and the load checks its allocations against them. #[must_use] -pub fn tier_batch(hp: &Hparams, cols: u64) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams, cols: u64) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, TIER_CHUNK, cols, + DownForms::of(DownAct::Q8Act), ) } @@ -274,10 +275,11 @@ mod tests { hp.n_used as u64, TIER_CHUNK, COLS, + DownForms::of(DownAct::Q8Act), ) ); let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); - let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS)); + let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS).expect("tier bytes")); let plain = workstation::plan_gate(hp.n_trunk); assert_eq!( inputs.prompt_batch_bytes(&plain).expect("a plain machine"), diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 6cf405c84..64b6a8407 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -135,13 +135,14 @@ pub const TIER_CHUNK: u64 = 16; /// and places. The plan reserves them on the tier card and the host, and the load checks its /// allocations against them. #[must_use] -pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result { let hp = &inputs.hp; - workstation::tier_leg_batch_bytes_of( + workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, - (TIER_CHUNK, cols), + TIER_CHUNK, + cols, leg_downs(inputs), ) } @@ -441,9 +442,12 @@ mod tests { hp.experts.n_used as u64, TIER_CHUNK, COLS, + DownForms::of(DownAct::Q8Act), ) ); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs, COLS)); + let tiered = workstation::plan_bp( + LAYERS, None, tier_batch(&inputs, COLS).expect("tier bytes"), + ); let plain = workstation::plan_gate(LAYERS); let places = 3 * 4 * 256 * 2; assert_eq!(inputs.tier_reserve, Some(places)); diff --git a/crates/model/src/arch/musespark/place.rs b/crates/model/src/arch/musespark/place.rs index 01572dc42..de91cf878 100644 --- a/crates/model/src/arch/musespark/place.rs +++ b/crates/model/src/arch/musespark/place.rs @@ -77,7 +77,7 @@ pub const fn pass_rows_max(n_used: usize) -> usize { /// the tier's rows and its staging are the latent's width, through the card /// leg in chunks of [`TIER_CHUNK`]. #[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { let ff = hp.expert_width(); workstation::tier_leg_batch_bytes( hp.latent as u64, @@ -85,6 +85,7 @@ pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { hp.n_used as u64, TIER_CHUNK, pass_rows_max(hp.n_used) as u64, + crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act), ) } diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index ac9b92eae..98c788951 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1478,13 +1478,15 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. #[must_use] -pub fn leg_tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn leg_tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, hp.n_used as u64, LEG_TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act) + .with(crate::placement::kernels::DownAct::Q8Blocks32), ) } @@ -1518,6 +1520,7 @@ const fn tier_batch_of(n_embd: u64, n_used: u64, ubatch: u64) -> TierBatchBytes staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_route_scratch_bytes(cols), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -2346,6 +2349,7 @@ fn draft_machine() -> Machine { Machine { cards: vec![Card { name: "mtp draft".to_string(), + tier_leg: None, device: None, usable_bytes: u64::MAX, context_bytes: 0, @@ -2824,6 +2828,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, @@ -3595,6 +3600,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: 1 << 40, context_bytes: 0, @@ -4509,6 +4515,7 @@ mod tests { staging: 496_730_112, scratch: 348_980_248, host: 839_188_480, + leg: None, } ); assert_eq!( @@ -4517,6 +4524,7 @@ mod tests { staging: 62_091_264, scratch: 87_233_816, host: 104_898_560, + leg: None, } ); } diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 3c71a02d2..4926c5462 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -46,13 +46,15 @@ pub const TIER_CHUNK: u64 = 8; /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. #[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act) + .with(crate::placement::kernels::DownAct::Q8Blocks32), ) } diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 95f0e5f15..90e91e2dc 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -14,9 +14,10 @@ use super::hparams::Hparams; use super::{roles, spec}; use crate::arch::chat_of; use crate::arch::place_hybrid; +use crate::placement::accepts::{Program, leg_down_forms}; use crate::placement::kernels::q8_act_bytes; use crate::placement::workstation::TierBatchBytes; -use crate::placement::{PlacementError, workstation}; +use crate::placement::{ModelTensors, PlacementError, workstation}; pub use place_hybrid::{GEMM_TILE_COLS, GEMM32_STEP, KvLayout, PlaceError}; use place_hybrid::{route_bytes, tier_places_bytes, wide_bytes}; @@ -40,18 +41,19 @@ pub const PARTIAL_ROWS: usize = 8; /// the stage card's card experts, which the body binds to its own constant. pub const TIER_CHUNK: u64 = 8; -/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a file of hyperparameters `hp`: +/// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for a described file: /// the tier card's staging, the leg's chunk scratch and the host's rows and places, for blocks of /// the host union's columns. The plan reserves them on the tier card and the host, and the load /// checks its allocations against them. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, + leg_down_forms(Program::Solar2Body, &inputs.model), ) } @@ -89,8 +91,12 @@ const fn acts_bytes(k: usize) -> usize { /// The wide forms are a bound: both forms at each width that allows them, where the load makes /// the ones its sites' types read. The units a prefill group past one adds are made out of the /// card's margin. -#[must_use] -pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 { +pub fn prompt_reserve_bytes( + hp: &Hparams, + spec: &ModelSpec, + model: &ModelTensors, + rows: usize, +) -> Result { let (embd, r) = (hp.n_embd, rows); let (mut heads, mut kv_heads, mut head) = (hp.n_head, 0, hp.head_dim); let (mut k_heads, mut v_heads, mut d) = (0, 0, 0); @@ -124,13 +130,21 @@ pub fn prompt_reserve_bytes(hp: &Hparams, spec: &ModelSpec, rows: usize) -> u64 let wide: usize = others.iter().map(|&k| wide_bytes(r, k)).sum::() + 2 * wide_bytes(r, embd) + wide_bytes(r, shared_ff); - let card_leg = workstation::tier_chunk_scratch_bytes( - embd as u64, - experts.ff as u64, - experts.n_used as u64, - r as u64, - ) as usize; - (4 * f32s + acts + wide + 2 * route_bytes(r) + card_leg) as u64 + let downs = leg_down_forms(Program::Solar2Body, model); + let card_leg = workstation::TierLegScratch { + n_embd: embd as u64, + ff: experts.ff as u64, + n_used: experts.n_used as u64, + chunk: r as u64, + downs, + } + .bytes(downs)?; + ((4 * f32s + acts + wide + 2 * route_bytes(r)) as u64) + .checked_add(card_leg) + .ok_or_else(|| PlacementError::Metadata { + key: "solar2 prompt reserve".into(), + detail: "bytes overflow u64".into(), + }) } impl PlanInputs { @@ -147,7 +161,7 @@ impl PlanInputs { let model = roles::classify(split, &hp)?; // No parser is declared: see `spec`. let spec = spec::spec_of(&hp, chat_of(split, None, None)?)?; - let reserve = prompt_reserve_bytes(&hp, &spec, PROMPT_ROWS); + let reserve = prompt_reserve_bytes(&hp, &spec, &model, PROMPT_ROWS)?; let tier = tier_places_bytes(PROMPT_ROWS, hp.experts.n_used); Ok(PlanInputs::new(ENGINE, hp, model, spec)? .with_prompt_reserve(reserve) @@ -247,7 +261,7 @@ mod tests { #[test] fn a_tier_card_is_planned_with_the_batch_places_beside_the_reserve() { let inputs = described("so2-place-tier"); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs.hp)); + let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs).expect("tier bytes")); let plain = workstation::plan_gate(LAYERS); let places = 12 * 512 * inputs.hp.experts.n_used as u64; assert_eq!(inputs.tier_reserve, Some(places)); @@ -305,11 +319,108 @@ mod tests { /// expert. Mutant: the reserve left out of the plan's call (the card plans, and its load stops /// at the allocation). #[test] - fn a_card_short_of_the_prompt_reserve_is_refused_by_name() { - let inputs = described("so2-place-reserve"); - let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, PROMPT_ROWS); - assert!(reserve > 0, "the synthetic header's batch has bytes"); - assert_eq!(inputs.prompt_reserve, reserve, "describe keeps the figure"); + fn a_host_only_mixture_keeps_the_exact_prompt_reserve_boundary() { + use crate::placement::{ExpertPart, Role}; + use gguf::GgmlType::{Q5_K, Q8_0}; + + let mut kv = keys(); + for (key, value) in &mut kv { + if matches!(*key, "embedding_length" | "expert_feed_forward_length") { + *value = V::U32(256); + } + } + let mut ts = tensors(); + for (name, dims) in &mut ts { + let tail = name.rsplit('.').nth(1).unwrap_or(""); + if matches!( + tail, + "ffn_gate_exps" + | "ffn_up_exps" + | "ffn_down_exps" + | "ffn_gate_shexp" + | "ffn_up_shexp" + | "ffn_down_shexp" + ) { + dims[0] = 256; + dims[1] = 256; + } else if tail == "attn_output" { + dims[1] = 256; + } else if matches!( + tail, + "token_embd" + | "output" + | "output_norm" + | "attn_norm" + | "ffn_norm" + | "attn_q" + | "attn_k" + | "attn_v" + | "attn_gate" + | "ssm_f_a" + | "ssm_g_a" + | "ssm_beta" + | "ffn_gate_inp" + ) { + dims[0] = 256; + } + } + let global = [ + ("tokenizer.ggml.pre", V::Str("solar-open")), + ("tokenizer.chat_template", V::Str("{{ messages }}")), + ]; + let path = header_shaped("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); + let split = Split::open(&path).expect("the mixed fixture opens"); + let mut inputs = PlanInputs::describe(&split).expect("valid routed geometry"); + std::fs::remove_file(path).expect("remove the header fixture"); + for t in inputs + .model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.ty = Q5_K; + t.file_bytes = t.dims.iter().product::() / t.ty.blck_size().unwrap() + * t.ty.type_size().unwrap(); + } + let admitted = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("the admitted prompt reserve"); + let admitted_tier = tier_batch(&inputs).expect("the admitted tier reserve"); + let down = inputs + .model + .tensors + .iter_mut() + .find(|t| { + t.role == Role::RoutedExperts + && t.layer == Some(0) + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + }) + .expect("layer zero's down"); + down.ty = Q8_0; + down.file_bytes = down.dims.iter().product::() / down.ty.blck_size().unwrap() + * down.ty.type_size().unwrap(); + let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) + .expect("prompt reserve"); + assert_eq!( + reserve, admitted, + "the host-only Q8_0 down adds no card form" + ); + assert_eq!( + tier_batch(&inputs).expect("mixed tier reserve"), + admitted_tier + ); + inputs.prompt_reserve = reserve; + let ample = workstation::plan_gate(LAYERS); + let placed = inputs + .plan(&le, 64, &PlanLevers::default()) + .expect("the mixed plan"); + assert_eq!( + placed.n_l[0], 0, + "Q8_0 stays on the host under Solar's rule" + ); + assert!( + placed.n_l[1..].iter().all(|&n| n > 0), + "other layers remain card candidates" + ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); machine.cards[0].usable_bytes = machine.cards[0].margin_bytes; @@ -346,10 +457,6 @@ mod tests { let plan = inputs .plan(&machine, 64, &levers) .expect("the card holds the terms and the reserve"); - assert_eq!( - plan.n_l.iter().sum::(), - 0, - "no room is left for an expert" - ); + assert!(plan.violations().is_empty()); } } diff --git a/crates/model/tests/placement.rs b/crates/model/tests/placement.rs index c62b0921c..394803f57 100644 --- a/crates/model/tests/placement.rs +++ b/crates/model/tests/placement.rs @@ -1371,6 +1371,7 @@ fn layered_model(layers: usize) -> ModelTensors { fn byte_card(name: &str, usable: u64, layers: Range) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, diff --git a/crates/model/tests/qwen4exp_mtp_reserve.rs b/crates/model/tests/qwen4exp_mtp_reserve.rs index 09eade30e..60c922e5c 100644 --- a/crates/model/tests/qwen4exp_mtp_reserve.rs +++ b/crates/model/tests/qwen4exp_mtp_reserve.rs @@ -298,6 +298,7 @@ fn machine(usable: u64) -> Machine { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: CONTEXT, diff --git a/crates/model/tests/solar_open2_meta.rs b/crates/model/tests/solar_open2_meta.rs index 0b55bb2ad..730f0a28b 100644 --- a/crates/model/tests/solar_open2_meta.rs +++ b/crates/model/tests/solar_open2_meta.rs @@ -398,6 +398,7 @@ fn hw_solar_open2_plan_a() { staging: 0, scratch: 0, host: 0, + leg: None, }; let bp = workstation::plan_bp(layers, None, tier_batch); let tiered = inputs diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index bb3f77110..daa05c3ec 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -449,6 +449,43 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { leg_arms(p).map(|a| gate == up && (a.gate_up)(gate) && (a.down)(down)) } +/// The union of down activation forms of the complete routed layers that +/// `p`'s placement rule admits, resolved from the file's descriptor at load. +#[must_use] +pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> kernels::DownForms { + use crate::placement::{ExpertPart, Role}; + + let RoutedRule::Leg { gate_up, down } = routed_rule(p) else { + return kernels::DownForms::NONE; + }; + let mut forms = kernels::DownForms::NONE; + for l in 0..model.layers { + let mut parts = [None; 3]; + for t in model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + { + let part = match ExpertPart::of_name(&t.name) { + Some(ExpertPart::Gate) => 0, + Some(ExpertPart::Up) => 1, + Some(ExpertPart::Down) => 2, + None => continue, + }; + parts[part] = Some(t.ty); + } + if let [Some(g), Some(u), Some(d)] = parts + && g == u + && gate_up(g).is_some() + && down(d).is_some() + && let Some((_, act)) = kernels::down(d) + { + forms = forms.with(act); + } + } + forms +} + /// Program `p`'s plan rule for its routed stacks ([`RoutedRule`]): on a card /// leg, each part by the arm its load reads ([`leg_runs`]), so a layer the /// plan gives card experts is one the load runs. The mimo2 and solar2 diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 7ab1361b8..61778ae70 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -100,12 +100,30 @@ pub const fn embed(ty: GgmlType) -> Option { /// it holds; a form whose last super-block is padded passes the padded width. #[must_use] pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { - let n_sb = k / 256; - 8 * cols * 64 * n_sb.div_ceil(2) - + 4 * cols * 256 * n_sb.div_ceil(4) - + 4 * cols * 128 * n_sb.div_ceil(2) - + 4 * cols * 8 * n_sb - + 4 * cols * 2 * n_sb + match try_q8_act_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_act_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_act_bytes`] with overflow returned to the placement boundary. +#[must_use] +pub(crate) const fn try_q8_act_bytes(cols: u64, k: u64) -> Option { + let n_sb = (k / 256) as u128; + // A u64 width's per-column count fits u128; its column product may not. + let per_col = 512 * n_sb.div_ceil(2) + + 1024 * n_sb.div_ceil(4) + + 512 * n_sb.div_ceil(2) + + 32 * n_sb + + 8 * n_sb; + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The card bytes of the 32-value q8_1 activation of `cols` columns of `k` @@ -114,8 +132,25 @@ pub const fn q8_act_bytes(cols: u64, k: u64) -> u64 { /// and scales, four bytes each, `k_blocks = k / 32`. #[must_use] pub const fn q8_blocks32_bytes(cols: u64, k: u64) -> u64 { - let k_blocks = k / 32; - cols * 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks) + match try_q8_blocks32_bytes(cols, k) { + Some(bytes) => bytes, + None => panic!("q8_blocks32_bytes: activation bytes overflow u64"), + } +} + +/// [`q8_blocks32_bytes`] with overflow returned to the placement boundary. +#[must_use] +const fn try_q8_blocks32_bytes(cols: u64, k: u64) -> Option { + let k_blocks = (k / 32) as u128; + let per_col = 4 * (256 * k_blocks.div_ceil(32) + 2 * k_blocks); + let Some(bytes) = per_col.checked_mul(cols as u128) else { + return None; + }; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// The down activation forms ([`DownAct`]) a card leg's buffers hold: the @@ -174,30 +209,68 @@ impl DownForms { /// them read it. #[must_use] pub const fn leg_bytes(cap: u64, slots_per_col: u64, n: u64, ff: u64, downs: DownForms) -> u64 { - let slots = cap * slots_per_col; + match try_leg_bytes(cap, slots_per_col, n, ff, downs) { + Some(bytes) => bytes, + None => panic!("leg_bytes: scratch bytes overflow u64"), + } +} + +/// [`leg_bytes`] with overflow returned to the placement boundary. This +/// counts the same whole blocks and padded planes as the infallible sizing. +#[must_use] +pub(crate) const fn try_leg_bytes( + cap: u64, + slots_per_col: u64, + n: u64, + ff: u64, + downs: DownForms, +) -> Option { + let Some(slots) = cap.checked_mul(slots_per_col) else { + return None; + }; + let Some(row_bytes) = slots.checked_mul(4) else { + return None; + }; + let Some(h) = row_bytes.checked_mul(ff) else { + return None; + }; + let Some(down) = row_bytes.checked_mul(n) else { + return None; + }; + let Some(input) = try_q8_act_bytes(cap, n) else { + return None; + }; let q8 = if downs.q8_act { - q8_act_bytes(slots, ff) + let Some(bytes) = try_q8_act_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; let b32 = if downs.blocks32 { - q8_blocks32_bytes(slots, ff) + let Some(bytes) = try_q8_blocks32_bytes(slots, ff) else { + return None; + }; + bytes } else { 0 }; - q8_act_bytes(cap, n) + 4 * slots * ff + q8 + b32 + 4 * slots * n + let bytes = input as u128 + h as u128 + q8 as u128 + b32 as u128 + down as u128; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } } /// Whether the common card leg (`crates/gpu/src/card_leg.rs`) launches the /// gate·up entry of a routed gate or up stack of type `ty` in the file's /// blocks ([`crate::placement::CardFormat::KQuant`]): the table's [`gate_up`] /// row, for the entries whose launchers read a `Q8Act` of more columns than a -/// run uses. The Q8_0 row has a launcher of that shape and an arm in the leg -/// (`CardLeg::launch`), which this set leaves out: the families that plan by -/// it (GLM-5.3, MiMo, Solar-Open2, MiniMax-M3, Raon) hold `Q8Act` down buffers -/// only, so a Q8_0 layer they planned on the card would fail at its first -/// launch where it runs on the host today; a program whose buffers follow the -/// down's form reads the arm through its own row ([`crate::accepts::prompt_leg_gate_up`]). +/// run uses. The Q8_0 and Q6_K rows have launchers of that shape and arms in the +/// leg, but only programs whose selected gate·up policy admits them read those +/// arms ([`crate::accepts::prompt_leg_gate_up`]). /// A stack in its planes is [`card_planes_runs`]'. #[must_use] pub const fn card_gate_up_runs(ty: GgmlType) -> bool { @@ -324,6 +397,54 @@ mod tests { use super::*; use crate::placement::CardFormat; + /// Plane-padding transitions retain their byte counts, and both sizing + /// contracts share the last representable column and aggregate boundary. + #[test] + fn activation_and_leg_bytes_keep_padding_and_overflow_boundaries() { + for (k, bytes) in [(1024, 3232), (1280, 5320)] { + assert_eq!(try_q8_act_bytes(1, k), Some(bytes)); + assert_eq!(q8_act_bytes(1, k), bytes); + } + for (k, bytes) in [(1024, 1280), (1056, 2312)] { + assert_eq!(try_q8_blocks32_bytes(1, k), Some(bytes)); + assert_eq!(q8_blocks32_bytes(1, k), bytes); + } + let q8 = DownForms::of(DownAct::Q8Act); + let b32 = DownForms::of(DownAct::Q8Blocks32); + for (forms, bytes) in [ + (DownForms::NONE, 4136), + (q8, 6224), + (b32, 5224), + (q8.with(DownAct::Q8Blocks32), 7312), + ] { + assert_eq!(try_leg_bytes(1, 1, 256, 256, forms), Some(bytes)); + assert_eq!(leg_bytes(1, 1, 256, 256, forms), bytes); + } + assert_eq!(try_leg_bytes(1, 1, 64, 24, DownForms::NONE), Some(352)); + let act_cols = u64::MAX / 2088; + assert_eq!(try_q8_act_bytes(act_cols, 256), Some(act_cols * 2088)); + assert_eq!(q8_act_bytes(act_cols, 256), act_cols * 2088); + assert_eq!(try_q8_act_bytes(act_cols + 1, 256), None); + let block_cols = u64::MAX / 1088; + assert_eq!( + try_q8_blocks32_bytes(block_cols, 256), + Some(block_cols * 1088) + ); + assert_eq!(q8_blocks32_bytes(block_cols, 256), block_cols * 1088); + assert_eq!(try_q8_blocks32_bytes(block_cols + 1, 256), None); + let cap = u64::MAX / 6224; + assert_eq!(try_leg_bytes(cap, 1, 256, 256, q8), Some(cap * 6224)); + assert_eq!(leg_bytes(cap, 1, 256, 256, q8), cap * 6224); + assert_eq!(try_leg_bytes(cap + 1, 1, 256, 256, q8), None); + assert_eq!(try_leg_bytes(1 << 61, 8, 256, 256, q8), None); + } + + #[test] + #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] + fn infallible_leg_sizing_names_an_overflow() { + let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); + } + /// Every `GgmlType` variant in declaration order, then one id the enum /// does not name. const ALL: [GgmlType; 42] = [ diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 1e3a0bbc6..25ef5133d 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -250,11 +250,10 @@ pub struct Card { /// draft model's resident bytes, say): taken from the budget with the /// context and the scratch ([`Card::set_aside_bytes`]). pub reserves: Vec<(String, u64)>, - /// A tier card with an expert cap of 0: it holds its reserves and no - /// expert, and serves no tier. The plan refuses one with no reserve - /// ([`PlacementError::IdleTier`]); a body hangs a tier only under the - /// tier cards that are not reserve-only ([`Machine::expert_tiers`]). + /// A tier card holding only reserves, not experts. pub reserve_only: bool, + /// A tier leg's scratch, jointly selected with its resident experts. + pub tier_leg: Option, } impl Card { @@ -276,10 +275,55 @@ impl Card { .try_fold(0u64, |sum, (_, bytes)| sum.checked_add(*bytes)) } - /// What the card sets aside before any tensor or cache: context, - /// scratch and reserves. The one owner of that sum: the expert rule's - /// budget, the budget floor, the headroom and [`Plan::violations`] all - /// take it from here. The margin is not in it. `None` past u64 bytes. + /// The tier leg's bound and selected scratch; absent on fixed-reserve paths. + fn selected_scratch( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result, PlacementError> { + let Some(leg) = self.tier_leg else { + return Ok(None); + }; + let forms = model + .tensors + .iter() + .filter(|t| { + t.role == Role::RoutedExperts + && t.layer + .is_some_and(|l| counts.get(l).is_some_and(|&n| n > 0)) + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + }) + .filter_map(|t| crate::kernels::down(t.ty)) + .fold(crate::kernels::DownForms::NONE, |forms, (_, act)| { + forms.with(act) + }); + Ok(Some((leg.bytes(leg.downs)?, leg.bytes(forms)?))) + } + + fn selected_reserve_bytes( + &self, + model: &ModelTensors, + counts: &[u64], + ) -> Result { + let fixed = self.reserve_bytes().ok_or_else(|| self.bytes_past_u64())?; + match self.selected_scratch(model, counts)? { + Some((bound, selected)) => fixed + .checked_sub(bound) + .and_then(|n| n.checked_add(selected)) + .ok_or_else(|| self.bytes_past_u64()), + None => Ok(fixed), + } + } + + fn floor_with_reserves(&self, dense: u64, kv: u64, past: u64, reserves: u64) -> Option { + [kv, self.context_bytes, self.scratch_bytes, reserves, past] + .into_iter() + .try_fold(dense, u64::checked_add) + } + + /// The machine's context, scratch and reserve bound before uploads. + /// Placement replaces a tier leg's bound with its candidate's actual + /// scratch. The margin is not in it. `None` past u64 bytes. #[must_use] pub fn set_aside_bytes(&self) -> Option { self.context_bytes @@ -287,18 +331,13 @@ impl Card { .checked_add(self.reserve_bytes()?) } - /// The card's floor under `dense` bytes of uploads and `kv` bytes of - /// cache: those, its set-aside, and `past` — what the load keeps beside - /// them: a plan's margin and what its caller holds beside the plan - /// (`plan_floor` sums those), or a whole load's own arena and reserve - /// ([`WholeLoad::past_bytes`]). The one owner of that sum: the budget - /// floor, the free floor and the whole fit ([`whole_need`]) all take it - /// from here. `None` past u64 bytes. + /// The fixed/bounded floor under `dense` upload bytes, `kv` cache bytes + /// and `past` bytes held beside them ([`WholeLoad::past_bytes`]). + /// Selected plans use the same sum with their actual reserves. + /// `None` past u64 bytes. #[must_use] pub fn floor_bytes(&self, dense: u64, kv: u64, past: u64) -> Option { - [kv, self.set_aside_bytes()?, past] - .into_iter() - .try_fold(dense, u64::checked_add) + self.floor_with_reserves(dense, kv, past, self.reserve_bytes()?) } fn bytes_past_u64(&self) -> PlacementError { @@ -1011,7 +1050,8 @@ pub struct CardTotals { pub stage_rows_bytes: u64, pub scratch_bytes: u64, pub context_bytes: u64, - /// The card's reserves ([`Card::reserves`]). + /// The selected reserves; a tier leg's descriptor bound is replaced by + /// its actual resident layers' scratch. pub reserve_bytes: u64, /// usable − dense − experts − rounding − KV − stage rows − scratch − /// context − reserves, usable capped by the plan's card budget; the @@ -3008,6 +3048,40 @@ impl Fill<'_, '_> { .unwrap_or(RoutedRule::PerType(CardFormat::of_routed)); let uploads = card_uploads(self.model, self.c, self.rows, self.eligible, format)?; let footprint = Footprint::new(card.granule_bytes, &uploads); + let mut down_acts = if card.tier_leg.is_some() { + vec![None; self.model.layers] + } else { + Vec::new() + }; + if card.tier_leg.is_some() { + for t in &self.model.tensors { + if t.role == Role::RoutedExperts + && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) + && let Some(l) = t.layer + && let Some((_, act)) = crate::kernels::down(t.ty) + { + down_acts[l] = Some(act); + } + } + } + let scratch = |counts: &[u64]| { + card.tier_leg.map_or(Ok(0), |leg| { + let forms = counts + .iter() + .zip(&down_acts) + .filter_map(|(&n, &act)| (n > 0).then_some(act).flatten()) + .fold(crate::kernels::DownForms::NONE, |forms, act| { + forms.with(act) + }); + leg.bytes(forms) + }) + }; + let bound = card + .tier_leg + .map(|leg| leg.bytes(leg.downs)) + .transpose()? + .unwrap_or(0); + let reserves = card.selected_reserve_bytes(self.model, n_l)?; let budget = i128::from(capped(card, self.card_budget)) - i128::from(self.kv) - i128::from( @@ -3015,19 +3089,25 @@ impl Fill<'_, '_> { .ok_or_else(|| card.bytes_past_u64())?, ) - i128::from(card.margin_bytes) - - i128::from(self.reserve); + - i128::from(self.reserve) + + i128::from(bound); if let Some(b) = self.card_budget { let dense = footprint.at(n_l)?; - check_floor(card, b, dense, self.kv, self.reserve)?; + check_floor(card, b, dense, self.kv, self.reserve, reserves)?; } if let Some(free) = card.free_bytes.filter(|&f| f < card.usable_bytes) { let dense = footprint.at(n_l)?; - check_free_floor(card, free, dense, self.kv, self.reserve)?; + check_free_floor(card, free, dense, self.kv, self.reserve, reserves)?; } let caps: Vec = (0..n_l.len()) .map(|l| self.model.layer_experts(l)) .collect(); - spread(n_l, held, self.eligible, &caps, budget, |n| footprint.at(n)) + spread(n_l, held, self.eligible, &caps, budget, |n| { + footprint + .at(n)? + .checked_add(scratch(n)?) + .ok_or_else(|| card.bytes_past_u64()) + }) } } @@ -3036,13 +3116,11 @@ impl Fill<'_, '_> { /// those as its `past`. Both floor checks ([`check_floor`], /// [`check_free_floor`]) and the share split's floors ([`resolve_shares`]) /// take it from here, so the floor they name is the least budget at which -/// the expert rule's own budget (usable less KV, set-aside, margin and -/// `beside`, in [`Fill::run`]) reaches the dense uploads. `None` past u64 -/// bytes. -fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64) -> Option { +/// the expert rule's own budget reaches the dense uploads. `None` past u64 bytes. +fn plan_floor(card: &Card, dense: u64, kv: u64, beside: u64, reserves: u64) -> Option { card.margin_bytes .checked_add(beside) - .and_then(|past| card.floor_bytes(dense, kv, past)) + .and_then(|past| card.floor_with_reserves(dense, kv, past, reserves)) } /// Refuse `card` under the card budget `budget` when its floor — the @@ -3057,8 +3135,9 @@ fn check_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { - let floor = plan_floor(card, dense, kv, beside); + let floor = plan_floor(card, dense, kv, beside, reserves); match floor { Some(floor) if floor <= budget => Ok(()), _ => Err(PlacementError::CardBudgetFloor { @@ -3069,7 +3148,7 @@ fn check_floor( kv, context: card.context_bytes, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, }), @@ -3088,9 +3167,10 @@ fn check_free_floor( dense: u64, kv: u64, beside: u64, + reserves: u64, ) -> Result<(), PlacementError> { let context = card.context_bytes - card.context_in_free(); - let floor = plan_floor(card, dense, kv, beside).map(|f| f - card.context_in_free()); + let floor = plan_floor(card, dense, kv, beside, reserves).map(|f| f - card.context_in_free()); match floor { Some(floor) if floor <= free => Ok(()), floor => Err(PlacementError::CardFreeFloor(Box::new(CardFreeFloor { @@ -3103,7 +3183,7 @@ fn check_free_floor( kv, context, scratch: card.scratch_bytes, - reserves: card.reserve_bytes().ok_or_else(|| card.bytes_past_u64())?, + reserves, margin: card.margin_bytes, beside, held: card @@ -4186,8 +4266,12 @@ fn totals<'a>( .map(|(c, (card, heap))| { let rounding = heap.taken - (card_dense[c] + card_experts[c]); let (kv, _, stage_rows) = kv_bytes[c]; + let counts = c + .checked_sub(machine.cards.len()) + .map_or(n_l.as_slice(), |t| tier_n_l[t].as_slice()); + let reserve_bytes = card.selected_reserve_bytes(model, counts)?; let used = card - .floor_bytes(heap.taken, kv, stage_rows) + .floor_with_reserves(heap.taken, kv, stage_rows, reserve_bytes) .ok_or_else(|| card.bytes_past_u64())?; if machine.unified().is_some() && machine.on_this_node(c) @@ -4207,9 +4291,7 @@ fn totals<'a>( stage_rows_bytes: stage_rows, scratch_bytes: card.scratch_bytes, context_bytes: card.context_bytes, - reserve_bytes: card - .reserve_bytes() - .expect("the card floor checked its reserves"), + reserve_bytes, headroom_bytes: i128::from(capped(card, card_budget::of(&card_budget, card))) - i128::from(used), }) @@ -4279,11 +4361,50 @@ pub struct PoolNeed { /// `card`'s total against usable − margin: its uploads' granules, its KV, /// its stage rows and its set-aside. fn card_total(card: &Card, totals: &CardTotals, heap: &Heap) -> u64 { - card.floor_bytes(heap.taken, totals.kv_bytes, totals.stage_rows_bytes) - .expect("the plan's card totals fit u64") + card.floor_with_reserves( + heap.taken, + totals.kv_bytes, + totals.stage_rows_bytes, + totals.reserve_bytes, + ) + .expect("the plan's card totals fit u64") } impl Plan<'_> { + /// One named reserve at its selected amount, not the machine's scratch + /// bound. Missing or duplicate rows and an invalid card return `None`; + /// invalid scratch descriptors propagate their named error. + pub fn card_reserve(&self, c: usize, name: &str) -> Result, PlacementError> { + let Some(card) = self.machine.card(c) else { + return Ok(None); + }; + let mut rows = card.reserves.iter().filter(|(n, _)| n == name); + let Some((_, bytes)) = rows.next() else { + return Ok(None); + }; + if rows.next().is_some() { + return Ok(None); + } + if name != workstation::TIER_BATCH_RESERVE { + return Ok(Some(*bytes)); + } + let counts = match c.checked_sub(self.machine.cards.len()) { + None => self.n_l.as_slice(), + Some(t) => match self.tier_n_l.get(t) { + Some(counts) => counts.as_slice(), + None => return Ok(None), + }, + }; + match card.selected_scratch(self.model, counts)? { + Some((bound, selected)) => bytes + .checked_sub(bound) + .and_then(|bytes| bytes.checked_add(selected)) + .map(Some) + .ok_or_else(|| card.bytes_past_u64()), + None => Ok(Some(*bytes)), + } + } + fn reserve_beside(&mut self, reserve: u64) -> Result<(), PlacementError> { if self.machine.unified().is_some() { let here = (0..self.machine.all_cards().count()) @@ -4908,6 +5029,7 @@ mod tests { fn card(name: &str, layers: Range, head: bool, token_embedding: bool) -> Card { Card { name: name.to_string(), + tier_leg: None, device: None, usable_bytes: 0, context_bytes: 0, @@ -6253,6 +6375,169 @@ mod tests { assert_eq!(plan.n_l, vec![8]); } + /// Scratch enters each expert candidate's footprint monotonically. + /// A boundary that admits one down form cannot pay for an unused second + /// form, and adding the first expert of that form pays its exact bytes. + #[test] + fn a_tier_jointly_selects_expert_rows_and_down_forms() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use GgmlType::{Q5_K, Q8_0}; + use workstation::{TIER_BATCH_RESERVE, tier_leg_batch_bytes}; + + let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + let batch = tier_leg_batch_bytes(256, 256, 1, 2, 3, both).expect("tier bytes"); + let leg = batch.leg.expect("a tier leg descriptor"); + let rule = routed_rule(Program::Qwen35Body); + for (downs, first) in [ + ([Q5_K, Q8_0], DownAct::Q8Act), + ([Q8_0, Q5_K], DownAct::Q8Blocks32), + ] { + let mut model = legged(&[[Q5_K, Q5_K, downs[0]], [Q5_K, Q5_K, downs[1]]]); + model.experts = 1; + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.dims[2] = 1; + t.file_bytes *= 8; + } + let expert = |l| { + model + .tensors + .iter() + .filter(|t| t.role == Role::RoutedExperts && t.layer == Some(l)) + .map(|t| t.file_bytes / model.experts) + .sum::() + }; + let one = + batch.staging + leg.bytes(DownForms::of(first)).expect("first scratch") + expert(0); + let two = + batch.staging + leg.bytes(both).expect("mixed scratch") + expert(0) + expert(1); + for (room, counts, forms) in [ + (one - 1, [0, 0], DownForms::NONE), + (one, [1, 0], DownForms::of(first)), + (two - 1, [1, 0], DownForms::of(first)), + (two, [1, 1], both), + ] { + let mut tier = bytes_card("tier", room, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = batch.leg; + let machine = Machine { + cards: vec![bytes_card("stage", HEAD, 0..2)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + }; + let plan = plan_routed(&model, &machine, 1, &NoKv, &PlanLevers::default(), rule) + .expect("the joint plan"); + assert_eq!(plan.n_l, [0, 0]); + assert_eq!(plan.tier_n_l[0], counts, "{downs:?}, {room}"); + let selected = batch.staging + leg.bytes(forms).expect("selected scratch"); + assert_eq!( + plan.card_reserve(1, TIER_BATCH_RESERVE) + .expect("selected reserve"), + Some(selected) + ); + assert_eq!(plan.cards[1].reserve_bytes, selected); + let rows = expert(0) * counts[0] + expert(1) * counts[1]; + assert_eq!(plan.cards[1].bytes(), selected + rows); + assert!(plan.violations().is_empty(), "{downs:?}, {room}"); + } + } + } + + /// A malformed descriptor must refuse a plan and a later selected-reserve + /// read, not wrap to a small scratch or masquerade as a fixed-reserve card. + #[test] + fn a_tier_refuses_invalid_scratch_descriptors_before_placement() { + use crate::accepts::{Program, routed_rule}; + use crate::kernels::{DownAct, DownForms}; + use workstation::{TIER_BATCH_RESERVE, TierLegScratch, tier_leg_batch_bytes}; + + let forms = DownForms::of(DownAct::Q8Act); + let batch = tier_leg_batch_bytes(256, 256, 1, 1, 1, forms).expect("valid batch"); + let good = batch.leg.expect("a leg descriptor"); + let mut model = legged(&[[GgmlType::Q5_K; 3]]); + model.experts_used = 1; + for t in model + .tensors + .iter_mut() + .filter(|t| t.role == Role::RoutedExperts) + { + t.dims[1] = 256; + t.file_bytes *= 64; + } + let machine = |leg| { + let mut tier = bytes_card("tier", 1 << 30, 0..0); + tier.reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + tier.tier_leg = Some(leg); + Machine { + cards: vec![bytes_card("stage", HEAD, 0..1)], + tiers: vec![tier], + nodes: vec![Node { + host: host(), + unified: None, + }], + } + }; + let valid = machine(good); + let rule = routed_rule(Program::Qwen35Body); + let plan = plan_routed(&model, &valid, 1, &NoKv, &PlanLevers::default(), rule) + .expect("valid descriptor plans"); + for leg in [ + TierLegScratch { + chunk: 1 << 61, + n_used: 8, + ..good + }, + TierLegScratch { chunk: 0, ..good }, + TierLegScratch { ff: 0, ..good }, + TierLegScratch { n_embd: 0, ..good }, + ] { + let bad = machine(leg); + assert!( + matches!( + plan_routed(&model, &bad, 1, &NoKv, &PlanLevers::default(), rule), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + let mut stale = plan.clone(); + stale.machine = &bad; + assert!( + matches!( + stale.card_reserve(1, TIER_BATCH_RESERVE), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + assert!( + matches!( + tier_leg_batch_bytes(leg.n_embd, leg.ff, leg.n_used, leg.chunk, 1, leg.downs), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" + ), + "{leg:?}" + ); + } + assert!(tier_leg_batch_bytes(256, 256, 8, 1, 1 << 61, forms).is_err()); + let inconsistent = workstation::TierBatchBytes { + scratch: 0, + ..batch + }; + assert!(matches!( + inconsistent.card(), + Err(PlacementError::Metadata { key, .. }) if key == "tier leg batch" + )); + } + // PIN(2026-10-22): the Qwen3.6 leg runs a Q8_0 down and (once the leg has the // arm) a Q6_K gate and up through the common leg, so the layers that held those // types hold types it still has no GEMM for (an MXFP4 down, an IQ2_XS gate and diff --git a/crates/placement/src/placement/nodes_scale.rs b/crates/placement/src/placement/nodes_scale.rs index 94e8830fe..b85b290b9 100644 --- a/crates/placement/src/placement/nodes_scale.rs +++ b/crates/placement/src/placement/nodes_scale.rs @@ -283,6 +283,7 @@ fn census_budgets_keep_each_node_and_stage_terms() { staging: 123, scratch: 456, host: 789, + leg: None, }, ) .expect("three independent nodes"); @@ -339,6 +340,7 @@ fn cases() -> &'static [Case] { staging: 0, scratch: 0, host: 0, + leg: None, }, ) .expect("two nodes with no batch bytes"); diff --git a/crates/placement/src/placement/qwen38_cards.rs b/crates/placement/src/placement/qwen38_cards.rs index 80fd5697f..c5d122985 100644 --- a/crates/placement/src/placement/qwen38_cards.rs +++ b/crates/placement/src/placement/qwen38_cards.rs @@ -255,6 +255,7 @@ const CHECKPOINT_38: u64 = 36 * 4_093_584 + 17 * 4 * 2560 * 4; fn stage(spec: CardSpec) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -303,6 +304,7 @@ fn machine_bp(stage_spec: CardSpec, tier: CardSpec) -> Machine { cards: vec![s], tiers: vec![Card { name: tier.name.to_string(), + tier_leg: None, device: tier.device, usable_bytes: tier.usable_bytes(), context_bytes: CONTEXT, diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 6b27baac9..74c5d5dd0 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -19,7 +19,7 @@ use super::nodes::{Fabric, Nodes}; use super::{ Card, Device, Host, HostTotals, Machine, Node, PlacementError, Plan, Reserve, UnifiedPool, }; -use crate::kernels::{DownAct, DownForms, leg_bytes, q8_act_bytes}; +use crate::kernels::{DownForms, leg_bytes, q8_act_bytes, try_leg_bytes, try_q8_act_bytes}; pub const MIB: u64 = 1 << 20; @@ -182,6 +182,7 @@ pub fn model_v41() -> String { fn card(spec: CardSpec, layers: Range, head: bool) -> Card { Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -210,6 +211,7 @@ pub fn tier_card(spec: CardSpec, batch: TierBatchBytes) -> Card { .card() .unwrap_or_else(|error| panic!("tier batch card reserve: {error}")), )); + tier.tier_leg = batch.leg; tier } @@ -373,6 +375,7 @@ pub const NO_TIER_BATCH: TierBatchBytes = TierBatchBytes { staging: 0, scratch: 0, host: 0, + leg: None, }; /// What the expert tier's prompt-batch service is called as a reserve: on @@ -401,11 +404,21 @@ pub struct TierBatchBytes { pub staging: u64, pub scratch: u64, pub host: u64, + /// A common leg's candidate-dependent scratch; `scratch` is its bound. + pub leg: Option, } impl TierBatchBytes { - /// The tier card's part; refuses a sum past u64. + /// The tier card's part; refuses an invalid leg bound or a sum past u64. pub fn card(&self) -> Result { + if let Some(leg) = self.leg + && leg.bytes(leg.downs)? != self.scratch + { + return Err(PlacementError::Metadata { + key: "tier leg batch".into(), + detail: "scratch reserve differs from the descriptor's bound".into(), + }); + } self.staging .checked_add(self.scratch) .ok_or(PlacementError::TierBatchCard { @@ -415,6 +428,42 @@ impl TierBatchBytes { } } +/// The common leg's scratch geometry and the descriptor's admitted forms. +/// Placement prices its candidate's selected forms jointly with expert rows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TierLegScratch { + pub n_embd: u64, + pub ff: u64, + pub n_used: u64, + pub chunk: u64, + pub downs: DownForms, +} + +impl TierLegScratch { + /// Scratch for exactly these selected forms, refusing invalid geometry + /// and arithmetic overflow before an allocation or placement candidate. + pub fn bytes(self, downs: DownForms) -> Result { + let invalid = || PlacementError::Metadata { + key: "tier leg scratch".into(), + detail: format!("invalid or overflowing geometry: {self:?}, selected {downs:?}"), + }; + if self.chunk == 0 + || self.n_used == 0 + || self.ff == 0 + || self.n_embd == 0 + || [ + crate::kernels::DownAct::Q8Act, + crate::kernels::DownAct::Q8Blocks32, + ] + .into_iter() + .any(|act| downs.holds(act) && !self.downs.holds(act)) + { + return Err(invalid()); + } + try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs).ok_or_else(invalid) + } +} + /// Experts the tile path's bucket table takes a layer at most, and the /// columns a tile holds: the tile path's shapes, which size its scratch. const TILE_BUCKET_EXPERTS: u64 = 1024; @@ -474,12 +523,18 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// The tier's chunk scratch for blocks of rows of `n_embd`, `n_used` slots a /// token, through experts of `ff`, cut in chunks of `chunk` tokens: one run of -/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`, the GLM-5.3-Flash -/// tier) with its slots' columns in the 128-value q8_1 form the tier's downs -/// read ([`leg_bytes`]), in place of the tile path's scratch. +/// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`) with its slots' +/// columns in exactly the forms the selected downs read ([`leg_bytes`]), +/// in place of the tile path's scratch. #[must_use] -pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { - leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) +pub const fn tier_chunk_scratch_bytes( + n_embd: u64, + ff: u64, + n_used: u64, + chunk: u64, + downs: DownForms, +) -> u64 { + leg_bytes(chunk, n_used, n_embd, ff, downs) } /// The host's part of the tier's prompt-batch service: per exchange set, the @@ -498,6 +553,7 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T staging: tier_batch_staging_bytes(n_embd, n_used, cols), scratch: tier_block_scratch_bytes(n_embd, ff, cols * n_used), host: tier_batch_host_bytes(n_embd, n_used, cols), + leg: None, } } @@ -505,39 +561,48 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T /// scratch is one run of the leg's buffers for `chunk` tokens ([`tier_chunk_scratch_bytes`]), in /// place of the tile path's. The one formula of those bytes for every family whose tier is the card /// leg; a family names its own `chunk` and widths. -#[must_use] -pub const fn tier_leg_batch_bytes( +pub fn tier_leg_batch_bytes( n_embd: u64, ff: u64, n_used: u64, chunk: u64, cols: u64, -) -> TierBatchBytes { - tier_leg_batch_bytes_of( + downs: DownForms, +) -> Result { + let leg = TierLegScratch { n_embd, ff, n_used, - (chunk, cols), - DownForms::of(DownAct::Q8Act), - ) -} - -/// [`tier_leg_batch_bytes`] for a tier whose layers' downs read the forms `downs`: the leg's -/// buffers hold the slots' columns in each of them (`bloomery_gpu::card_leg::LegBufs::for_downs`, -/// [`leg_bytes`]), as an NVFP4 down's 32-value blocks. -#[must_use] -pub const fn tier_leg_batch_bytes_of( - n_embd: u64, - ff: u64, - n_used: u64, - (chunk, cols): (u64, u64), - downs: DownForms, -) -> TierBatchBytes { - TierBatchBytes { - staging: tier_batch_staging_bytes(n_embd, n_used, cols), - scratch: leg_bytes(chunk, n_used, n_embd, ff, downs), - host: tier_batch_host_bytes(n_embd, n_used, cols), - } + chunk, + downs, + }; + let scratch = leg.bytes(downs)?; + let counted = || -> Option<(u64, u64)> { + if cols == 0 { + return None; + } + let slots = cols.checked_mul(n_used)?; + let places = 4u64.checked_mul(slots)?; + let down = places.checked_mul(n_embd)?; + let staging = 4u64 + .checked_mul(cols)? + .checked_mul(n_embd)? + .checked_add(places)? + .checked_add(try_q8_act_bytes(cols, n_embd)?)? + .checked_add(down)?; + let host = 2u64.checked_mul(down.checked_add(places)?)?; + Some((staging, host)) + }; + let (staging, host) = counted().ok_or_else(|| PlacementError::Metadata { + key: "tier leg batch".into(), + detail: format!("invalid or overflowing staging: {leg:?}, columns {cols}"), + })?; + Ok(TierBatchBytes { + staging, + scratch, + host, + leg: Some(leg), + }) } /// Plan (b′): the A6000 runs all `layers` and the head as in [`plan_a`]; @@ -629,6 +694,7 @@ pub fn plan_node_tiers( remote.cards[0] .reserves .extend(tier_card(spec, batch).reserves); + remote.cards[0].tier_leg = batch.leg; machines.push(remote); } Ok(machines) @@ -811,6 +877,7 @@ pub(crate) fn spark_nodes(n: usize, layers: usize, fabric: Fabric) -> Nodes { staging: 0, scratch: 0, host: 0, + leg: None, }; let mut machines = plan_node_tiers(layers, spec, &vec![spec; n - 1], batch) .expect("synthetic nodes with no batch bytes"); @@ -1701,6 +1768,7 @@ mod tests { staging: 1, scratch: 2, host: 3, + leg: None, }, ]; for layers in [0, 1, 20, 43, 47, 48, 61, 78] { @@ -1856,6 +1924,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }; for result in [ plan_tiers(4, A6000, &[RTX_3090], None, huge).map(|_| ()), @@ -1906,6 +1975,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let machines = plan_node_tiers(4, specs[0], &specs[1..], batch) .expect("four nodes of the same card name and ordinal"); @@ -1950,6 +2020,7 @@ mod tests { staging: 0, scratch: 0, host: u64::MAX / 2, + leg: None, }; let remotes = [gb10(80 << 30), gb10(60 << 30)]; let machines = @@ -1985,6 +2056,7 @@ mod tests { staging: 10, scratch: 5, host: 100, + leg: None, }; let m = plan_tiers(4, A6000, &[t1, t2], None, batch).expect("two tiers"); let rows: Vec<&(String, u64)> = m @@ -2065,6 +2137,7 @@ mod tests { assert_eq!(counted_arena_bytes(&m.cards[0]), 7, "{}", spec.name); let want = Card { name: spec.name.to_string(), + tier_leg: None, device: spec.device, usable_bytes: spec.usable_bytes(), context_bytes: CONTEXT, @@ -2112,6 +2185,7 @@ mod tests { staging: u64::MAX, scratch: 1, host: 0, + leg: None, }, ); } @@ -2131,6 +2205,7 @@ mod tests { staging: u64::MAX - 1, scratch: 1, host: 0, + leg: None, }; assert_eq!(edge.card().expect("the sum fits u64"), u64::MAX); assert!(matches!( diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 012ef946d..a5987a823 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -89,6 +89,23 @@ Qwen projection/shared dispatch does not gain this site automatically. Flint still has no Program, attention/router binding, body or serve seat. Run the new consumer gate and the affected regressions on NVIDIA before claiming device support; reconcile full-model coverage only with the actual body integration. +### Selected expert arms and tier scratch + +Common card and tier wrappers resolve `LegArms` from their selected `Program`, rather than assuming +every family has one gate/up type or one down activation form. Their resident layers determine the +union of `Q8Act` and `Q8Blocks32` buffers. Plane layouts remain restricted to the selected program's +arms; Solar's host-only triples do not enlarge its card scratch. + +Placement charges a tier's selected down forms jointly with its expert count. The same kernels-owned, +checked sizing arithmetic is used by allocation and planning; malformed or overflowing public scratch +descriptors are refused by name. The plan records the selected reserve, and loaders check that amount +instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver +is used. + +A host-only public planner consumer exercised the one-byte boundaries before the first resident layer +and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, +not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an +exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. From a65d1d8eb75065327453ae05f0815e5f64510fae Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:12:33 +0900 Subject: [PATCH 27/85] placement: test overflow refusal without pinning panic wording --- crates/placement/src/kernels.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 61778ae70..f2d85d534 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -440,8 +440,8 @@ mod tests { } #[test] - #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] - fn infallible_leg_sizing_names_an_overflow() { + #[should_panic] + fn infallible_leg_sizing_refuses_overflow() { let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); } From d767e0ba38a66216e2fe501805777e1a9e5f1b71 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:24:24 +0900 Subject: [PATCH 28/85] gpu: preserve fixed reserves for host-only node projections --- crates/gpu/src/host/node.rs | 21 +++++++++++++++------ docs/research/flint-attach.md | 3 +++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/gpu/src/host/node.rs b/crates/gpu/src/host/node.rs index 282a1157d..9fe0249bc 100644 --- a/crates/gpu/src/host/node.rs +++ b/crates/gpu/src/host/node.rs @@ -321,14 +321,23 @@ impl NodeOpen { .find(|d| self.device.is_some_and(|id| id.uuid == d.uuid)) .ok_or_else(|| GpuError::shape(WHAT, "the projected tier is absent from the census"))?; let machine = workstation::plan_on(spec_of_device(device), self.layers); - let totals = plan.cards.get(self.card).ok_or_else(|| { + let card = plan.machine.card(self.card).ok_or_else(|| { GpuError::shape(WHAT, "the projected tier card is absent from the plan") })?; - let reserve = totals - .context_bytes - .checked_add(totals.scratch_bytes) - .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) - .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; + // Host-only projections have no card totals; their fixed reserves + // still belong to the machine. Dynamic scratch needs selected totals. + let reserve = if card.tier_leg.is_some() { + let totals = plan.cards.get(self.card).ok_or_else(|| { + GpuError::shape(WHAT, "the projected tier card is absent from the plan") + })?; + totals + .context_bytes + .checked_add(totals.scratch_bytes) + .and_then(|bytes| bytes.checked_add(totals.reserve_bytes)) + } else { + card.set_aside_bytes() + } + .ok_or_else(|| GpuError::shape(WHAT, "the projected tier reserve overflows"))?; let mut projected = model::placement::nodes::routed_host( plan.model, &plan.rows, diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index a5987a823..f3437908e 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -101,6 +101,9 @@ checked sizing arithmetic is used by allocation and planning; malformed or overf descriptors are refused by name. The plan records the selected reserve, and loaders check that amount instead of the machine descriptor's upper bound. No always-both allocation or reserve-equality waiver is used. +Host-only projections intentionally have no card totals: fixed reserves remain owned by their machine. +Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging +budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, From 9da54fbe0dff4f0596af04b075ceb3eb69f1b0d9 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:37:10 +0900 Subject: [PATCH 29/85] gpu: remove redundant wrapper test allocation and borrow --- crates/gpu-gates/src/bin/gate_solar2_twocard.rs | 2 +- crates/gpu/src/card_experts.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index af0916a7c..9bcd34bc7 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -313,7 +313,7 @@ mod gate { let layers = inputs.model.layers; let bp = Place::Bp .on_host()? - .machine(None, Some(place::tier_batch(&inputs)?))?; + .machine(None, Some(place::tier_batch(inputs)?))?; let two = bp(layers); let plan = inputs.plan(&two, u64::try_from(CTX)?, &cfg.place)?; let budget = cfg diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 7c53831d5..468a422bd 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -1002,8 +1002,7 @@ mod tests { let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); let base: Vec = (0..CAP * N).map(|i| (i % 13) as f32 * 0.125).collect(); let x1 = DeviceBuffer::from_host(stream, &base).expect("the base"); - let route = - DeviceBuffer::from_host(stream, &vec![0.125f32; CAP * USED]).expect("weights"); + let route = DeviceBuffer::from_host(stream, &[0.125f32; CAP * USED]).expect("weights"); let host_sel: Vec = (0..CAP * USED) .map(|i| { if i % USED == USED - 1 { From 45a6b4acbc7b3fc8c0afbbac03467dd028782967 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:24:29 +0900 Subject: [PATCH 30/85] gpu: support six-slot F32 tier handoff --- crates/gpu-deepseek41/src/chain/ffn/batch.rs | 215 ++++++++++++ crates/gpu/src/host/handoff.rs | 327 ++++++++++++++----- docs/research/flint-attach.md | 22 ++ tools/unsafe-ratchet.txt | 2 + 4 files changed, 489 insertions(+), 77 deletions(-) diff --git a/crates/gpu-deepseek41/src/chain/ffn/batch.rs b/crates/gpu-deepseek41/src/chain/ffn/batch.rs index 42f350086..c3df4ec36 100644 --- a/crates/gpu-deepseek41/src/chain/ffn/batch.rs +++ b/crates/gpu-deepseek41/src/chain/ffn/batch.rs @@ -2669,3 +2669,218 @@ fn count_of(by_count: &mut [T], m: usize) -> Result<&mut T, GpuError> { detail: format!("a chunk of {m} tokens; the batch holds scratch for 1..={n}"), }) } + +#[cfg(test)] +mod tests { + use super::{CardAccTier, FfnBatchKernels}; + use bloomery_gpu::host::handoff::{Handoff, HandoffKernels}; + use bloomery_gpu::host::page::HandoffLayout; + use bloomery_gpu::host::step::HandoffTarget; + use bloomery_gpu::host::tier::{TierImageLayout, TierTarget}; + use bloomery_gpu::hybrid::HOST; + use bloomery_gpu::{FaultSite, Gpu, GpuError, Window}; + use cuda_core::DeviceBuffer; + + /// The handoff's places feed the production tier sum, not a readback + /// alone: every routed row belongs to exactly one of host, card and + /// tier, and all other copies are poisoned. Both slot entries must + /// conserve that union, including the final routed lane and partial + /// blocks, eagerly and on replay. Invalid ids must still raise. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-ds41-lib` runs it on the box"] + fn hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes() { + const EXPERTS: usize = 16; + const ROW_OFF: usize = 3; + const LAYER: usize = 7; + let gpu = Gpu::new().expect("a GPU"); + let stream = gpu.stream(); + let handoff = HandoffKernels::load(gpu.context()).expect("handoff kernels"); + let sums = FfnBatchKernels::load(gpu.context()).expect("tier sum kernels"); + let mut map = vec![HOST - 1; ROW_OFF + EXPERTS]; + let mut tmap = map.clone(); + for id in 0..EXPERTS { + map[ROW_OFF + id] = if id % 3 == 0 { (id / 3) as u32 } else { HOST }; + tmap[ROW_OFF + id] = if id % 3 == 1 { (id / 3) as u32 } else { HOST }; + } + let map = DeviceBuffer::from_host(stream, &map).expect("card map"); + let tmap = DeviceBuffer::from_host(stream, &tmap).expect("tier map"); + for n_used in [6, 8] { + for hidden in [n_used, 256, 257] { + for bad in [None, Some(EXPERTS as u32), Some(u32::MAX)] { + let mut ids = vec![15, 0, 10, 5, 8, 1, 2, 7]; + ids.truncate(n_used); + if let Some(id) = bad { + ids[n_used - 1] = id; + } + // Neither handoff may read the router's trailing lane. + ids.push(u32::MAX); + let weights: Vec = (0..n_used) + .map(|s| (s + 1) as f32 / 16.0) + .chain([f32::NAN]) + .collect(); + let x: Vec = (0..hidden).map(|d| (d % 7 + 1) as f32 / 8.0).collect(); + let row = |s: usize, d: usize| (ids[s] + 1) as f32 * x[d]; + let mut down = vec![f32::NAN; n_used * hidden]; + let mut trows = down.clone(); + let mut host = vec![0.0f32; hidden]; + let mut want = host.clone(); + for d in 0..hidden { + let mut card_tier = 0.0f32; + for s in 0..n_used { + if ids[s] >= EXPERTS as u32 { + continue; + } + let value = row(s, d); + match ids[s] % 3 { + 0 => down[s * hidden + d] = value, + 1 => trows[s * hidden + d] = value, + _ => host[d] = value.mul_add(weights[s], host[d]), + } + if ids[s] % 3 != 2 { + card_tier = value.mul_add(weights[s], card_tier); + } + } + want[d] = card_tier + host[d]; + } + let ids_d = DeviceBuffer::from_host(stream, &ids).expect("ids"); + let mut weights_d = DeviceBuffer::from_host(stream, &weights).expect("weights"); + let x_d = DeviceBuffer::from_host(stream, &x).expect("activation"); + let seq = DeviceBuffer::from_host(stream, &[19u32]).expect("sequence"); + let down = DeviceBuffer::from_host(stream, &down).expect("card rows"); + let trows = DeviceBuffer::from_host(stream, &trows).expect("tier rows"); + let doubled_host: Vec = host.iter().map(|h| 2.0 * h).collect(); + let mut host = DeviceBuffer::from_host(stream, &host).expect("host sum"); + let lay = HandoffLayout { + seq: 0, + ids: 1, + weights: 1 + n_used, + x: 1 + 2 * n_used, + n_used, + hidden, + }; + let tl = TierImageLayout { + sel: 0, + q3: 0, + d8: 0, + x: n_used, + n_used, + q3_words: 0, + d8_len: 0, + x_len: hidden, + hidden, + cols: 1, + }; + for captured in [false, true] { + let mut image = + DeviceBuffer::zeroed(stream, lay.x + hidden).expect("image"); + let mut timage = + DeviceBuffer::zeroed(stream, tl.x + hidden).expect("tier image"); + let poison = vec![HOST - 1; n_used + 1]; + let mut sel = DeviceBuffer::from_host(stream, &poison).expect("places"); + let mut tsel = + DeviceBuffer::from_host(stream, &poison).expect("tier places"); + let mut acc = DeviceBuffer::zeroed(stream, hidden).expect("sum"); + let mut result = DeviceBuffer::zeroed(stream, hidden).expect("result"); + let mut enqueue = || -> Result<(), GpuError> { + handoff.enqueue_handoff_tier( + stream, + &Handoff { + ids: &ids_d, + weights: &weights_d, + map: &map, + row_off: ROW_OFF, + n_expert: EXPERTS, + }, + &tmap, + ( + HandoffTarget { + x: &x_d, + seq: &seq, + image: &mut image, + layout: lay, + }, + TierTarget { + image: &mut timage, + rows: &trows, + layout: tl, + }, + ), + gpu.layer_sink(LAYER)?, + (&mut sel, &mut tsel), + )?; + let routed_weights = + Window::::of(&image, 4 * lay.weights, n_used)?; + sums.enqueue_card_acc_tier( + stream, + &CardAccTier { + down: &down, + trows: &trows, + w: &routed_weights, + sel: &sel, + tsel: &tsel, + n: hidden, + m: 1, + n_card: EXPERTS.div_ceil(3), + n_tier: EXPERTS.div_ceil(3), + n_used, + }, + &mut acc, + )?; + gpu.elem() + .enqueue_add(stream, &acc, &host, hidden, &mut result) + }; + if captured { + let graph = gpu.capture(|_| enqueue()).expect("capture"); + graph.launch(stream).expect("replay"); + let doubled_weights: Vec = + weights.iter().map(|w| 2.0 * w).collect(); + weights_d + .copy_from_host(stream, &doubled_weights) + .expect("new route weights"); + host.copy_from_host(stream, &doubled_host) + .expect("new host sum"); + graph.launch(stream).expect("second replay"); + stream.synchronize().expect("replays settled"); + } else { + enqueue().expect("eager handoff and consumption"); + } + let scale = if captured { 2.0 } else { 1.0 }; + let got = result.to_host_vec(stream).expect("result"); + let places = sel.to_host_vec(stream).expect("places"); + let tier_places = tsel.to_host_vec(stream).expect("tier places"); + let image = image.to_host_vec(stream).expect("host image"); + let timage = timage.to_host_vec(stream).expect("tier image"); + stream.synchronize().expect("settled"); + let fault = gpu.take_fault().expect("fault word"); + if bad.is_some() { + let fault = fault.expect("an invalid id must raise"); + assert_eq!(fault.layer, LAYER as u32); + assert_eq!(fault.code, FaultSite::ExpertId as u32); + assert_eq!(fault.sites, 1 << FaultSite::ExpertId as u32); + } else { + assert!(fault.is_none(), "{fault:?}"); + } + assert_eq!(places[n_used], HOST - 1, "card places tail"); + assert_eq!(tier_places[n_used], HOST - 1, "tier places tail"); + assert_eq!(image[lay.seq], 19, "sequence"); + assert_eq!(&image[lay.ids..lay.ids + n_used], &ids[..n_used]); + for s in 0..n_used { + assert_eq!(image[lay.weights + s], (weights[s] * scale).to_bits()); + assert_eq!(timage[tl.sel + s], tier_places[s]); + } + for d in 0..hidden { + assert_eq!(image[lay.x + d], x[d].to_bits()); + assert_eq!(timage[tl.x + d], x[d].to_bits()); + } + assert!( + got.iter() + .zip(&want) + .all(|(g, w)| g.to_bits() == (w * scale).to_bits()), + "{n_used} slots, width {hidden}, bad {bad:?}, capture {captured}: result conservation" + ); + } + } + } + } + } +} diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 1e36a2d0a..cd453226c 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,10 +18,9 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_8_tier` and -//! `ds41_ffn_handoff_4_tier` are the eight- and four-slot handoffs of a layer -//! an expert tier holds experts of: the -//! same writes, and the slots' tier places and the activation in f32 into +//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_6_tier` and +//! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an +//! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); //! `ds41_ffn_handoff_10_cols_tier` is `_10_cols`'s of such a layer, every //! column's tier places and activation into the row's tier image @@ -282,6 +281,88 @@ mod handoff_kernels { unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; } + /// [`ds41_ffn_handoff`] and the row's f32 tier image, with the same + /// per-thread writes and following go as [`ds41_ffn_handoff_8_tier`]. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + ids_in.len() >= 6, + w_in.len() >= 6, + map.len() >= row_off + n_expert, + tmap.len() >= row_off + n_expert, + x.len() >= n, + seq.len() >= 1, + n >= 6, + ids_at >= seq_at + 1, + wts_at >= ids_at + 6, + x_at >= wts_at + 6, + image.len() >= x_at + n, + tx_at >= tsel_at + 6, + timage.len() >= tx_at + n, + sel.len() >= 6, + tsel.len() >= 6 + ) + )] + pub fn ds41_ffn_handoff_6_tier( + ids_in: &[u32], + w_in: &[f32], + map: &[u32], + tmap: &[u32], + row_off: u32, + n_expert: u32, + x: &[f32], + seq: &[u32], + n: u32, + seq_at: u32, + ids_at: u32, + wts_at: u32, + x_at: u32, + tsel_at: u32, + tx_at: u32, + fault: FaultSink, + mut image: DisjointSlice, + mut timage: DisjointSlice, + mut sel: DisjointSlice, + mut tsel: DisjointSlice, + ) { + let d = thread::index_1d().get(); + if d >= n as usize { + return; + } + let a = HandoffIn { + ids_in, + w_in, + map, + row_off, + n_expert, + x, + seq, + seq_at, + ids_at, + wts_at, + x_at, + fault, + }; + // SAFETY: d < n, and the launch contract is `handoff_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; + let t = TierIn { + tmap, + tsel_at, + tx_at, + }; + // SAFETY: d < n, and the launch contract is `handoff_tier_at`'s at + // N_USED slots; thread d is the launch's only thread at d. + unsafe { handoff_tier_at::(&a, &t, d, &mut timage, &mut tsel) }; + } + /// [`ds41_ffn_handoff_8`] and the row's f32 tier image /// ([`handoff_tier_at`]): beside the handoff and the stage card's places, /// threads `s < 8` write `tsel[s] = tmap[row_off + id]` — the id's slot @@ -1265,8 +1346,9 @@ impl HandoffKernels { } impl HandoffKernels { - /// Enqueue the handoff of a tier layer of four or eight slots a token - /// (`ds41_ffn_handoff_4_tier`, `ds41_ffn_handoff_8_tier`): [`HandoffKernels::enqueue_handoff`]'s + /// Enqueue the handoff of a tier layer of six or eight slots a token + /// (`ds41_ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and /// with the activation into the f32 tier image `ttarget`. A layout of @@ -1286,79 +1368,109 @@ impl HandoffKernels { fault: FaultSink, (sel, tsel): (&mut DeviceBuffer, &mut DeviceBuffer), ) -> Result<(), GpuError> { - const WHAT: &str = "ds41_ffn_handoff_tier"; let (lay, tl) = (target.layout, ttarget.layout); - if !matches!(lay.n_used, SLOTS_4 | SLOTS_8) - || tl.n_used != lay.n_used - || tl.x_len != lay.hidden - || tl.q3_words != 0 - || tl.d8_len != 0 - { - return Err(GpuError::Shape { - what: WHAT, - detail: format!( - "the boundary carries {} slots of {} values and the tier image {} slots, {} \ - f32, {} code words, {} scales; the entries hand over {SLOTS_4} or {SLOTS_8} \ - slots, the tier image the boundary's, and the activation in f32", - lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len - ), - }); - } + let slots = check_tier(&lay, &tl)?; + let what = match slots { + HandoffSlots::Four => "ds41_ffn_handoff_4_tier", + HandoffSlots::Six => "ds41_ffn_handoff_6_tier", + _ => "ds41_ffn_handoff_8_tier", + }; let n = lay.hidden; - let grid = launch_u32(WHAT, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; + let grid = launch_u32(what, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; let cfg = LaunchConfig1D::new(grid, HANDOFF_THREADS, 0); - if lay.n_used == SLOTS_4 { - let prep = self.module.prepare_ds41_ffn_handoff_4_tier(cfg)?; - self.module.ds41_ffn_handoff_4_tier( - stream, - &prep, - h.ids, - h.weights, - h.map, - tmap, - launch_u32(WHAT, "row_off", h.row_off)?, - launch_u32(WHAT, "n_expert", h.n_expert)?, - target.x, - target.seq, - launch_u32(WHAT, "n", n)?, - launch_u32(WHAT, "seq_at", lay.seq)?, - launch_u32(WHAT, "ids_at", lay.ids)?, - launch_u32(WHAT, "wts_at", lay.weights)?, - launch_u32(WHAT, "x_at", lay.x)?, - launch_u32(WHAT, "tsel_at", tl.sel)?, - launch_u32(WHAT, "tx_at", tl.x)?, - fault, - target.image, - ttarget.image, - sel, - tsel, - )?; - } else { - let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; - self.module.ds41_ffn_handoff_8_tier( - stream, - &prep, - h.ids, - h.weights, - h.map, - tmap, - launch_u32(WHAT, "row_off", h.row_off)?, - launch_u32(WHAT, "n_expert", h.n_expert)?, - target.x, - target.seq, - launch_u32(WHAT, "n", n)?, - launch_u32(WHAT, "seq_at", lay.seq)?, - launch_u32(WHAT, "ids_at", lay.ids)?, - launch_u32(WHAT, "wts_at", lay.weights)?, - launch_u32(WHAT, "x_at", lay.x)?, - launch_u32(WHAT, "tsel_at", tl.sel)?, - launch_u32(WHAT, "tx_at", tl.x)?, - fault, - target.image, - ttarget.image, - sel, - tsel, - )?; + let row_off = launch_u32(what, "row_off", h.row_off)?; + let n_expert = launch_u32(what, "n_expert", h.n_expert)?; + let at = [ + launch_u32(what, "n", n)?, + launch_u32(what, "seq_at", lay.seq)?, + launch_u32(what, "ids_at", lay.ids)?, + launch_u32(what, "wts_at", lay.weights)?, + launch_u32(what, "x_at", lay.x)?, + launch_u32(what, "tsel_at", tl.sel)?, + launch_u32(what, "tx_at", tl.x)?, + ]; + match slots { + HandoffSlots::Four => { + let prep = self.module.prepare_ds41_ffn_handoff_4_tier(cfg)?; + self.module.ds41_ffn_handoff_4_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + HandoffSlots::Six => { + let prep = self.module.prepare_ds41_ffn_handoff_6_tier(cfg)?; + self.module.ds41_ffn_handoff_6_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } + _ => { + let prep = self.module.prepare_ds41_ffn_handoff_8_tier(cfg)?; + self.module.ds41_ffn_handoff_8_tier( + stream, + &prep, + h.ids, + h.weights, + h.map, + tmap, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + at[5], + at[6], + fault, + target.image, + ttarget.image, + sel, + tsel, + )?; + } } Ok(()) } @@ -1514,6 +1626,29 @@ impl HandoffKernels { } } +/// The single-token F32 tier handoff's layout contract, shared by the +/// slot entries. The generated launch checks the input and output spans. +fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result { + let slots = HandoffSlots::of(lay.n_used); + if !matches!(slots, Ok(HandoffSlots::Four | HandoffSlots::Six | HandoffSlots::Eight)) + || tl.n_used != lay.n_used + || tl.x_len != lay.hidden + || tl.q3_words != 0 + || tl.d8_len != 0 + { + return Err(GpuError::Shape { + what: "HandoffKernels::enqueue_handoff_tier", + detail: format!( + "the boundary carries {} slots of {} values and the tier image {} slots, {} \ + f32, {} code words, {} scales; the entries hand over matching 6 or 8 slots \ + and the activation in f32", + lay.n_used, lay.hidden, tl.n_used, tl.x_len, tl.q3_words, tl.d8_len + ), + }); + } + slots.map_err(|e| GpuError::shape("HandoffKernels::enqueue_handoff_tier", e.to_string())) +} + /// [`HandoffKernels::enqueue_handoff_cols`]'s refusals, by name before the /// launch: a layout of another slot count, `m` of 0, a pitch under ten, and a /// `sel` of `sel_len` places shorter than `m` columns of ten. @@ -1600,7 +1735,7 @@ pub struct Places<'a> { #[cfg(test)] mod tests { - use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier}; + use super::{HandoffRefused, HandoffSlots, check_cols, check_cols_tier, check_tier}; use crate::host::page::HandoffLayout; use crate::host::tier::TierImageLayout; @@ -1651,6 +1786,44 @@ mod tests { } } + #[test] + fn a_token_tier_handoff_requires_matching_f32_six_or_eight_slot_images() { + for n_used in 0..=10 { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + for tier_slots in 0..=10 { + let tl = TierImageLayout { + n_used: tier_slots, + x: tier_slots, + ..image(1) + }; + assert_eq!( + check_tier(&lay, &tl).is_ok(), + matches!(n_used, 6 | 8) && tier_slots == n_used, + "host {n_used}, tier {tier_slots}" + ); + } + } + for n_used in [6, 8] { + let lay = HandoffLayout { + n_used, + ..boundary() + }; + let tl = TierImageLayout { + n_used, + x: n_used, + ..image(1) + }; + for x_len in [0, HIDDEN - 1, HIDDEN + 1, 2 * HIDDEN] { + assert!(check_tier(&lay, &TierImageLayout { x_len, ..tl }).is_err()); + } + assert!(check_tier(&lay, &TierImageLayout { q3_words: 1, ..tl }).is_err()); + assert!(check_tier(&lay, &TierImageLayout { d8_len: 1, ..tl }).is_err()); + } + } + /// A tier handoff of `m` columns over a page of 8 refuses, by name, a /// `tsel` or a `sel` shorter than `10·m`, and takes ones of exactly that: /// the launch writes every column's places. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index f3437908e..1df60b76d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -109,6 +109,28 @@ A host-only public planner consumer exercised the one-byte boundaries before the and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. +### Six-slot F32 tier handoff implementation + +`HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry +beside the existing eight-slot entry. Both host and tier layouts must have the +same supported slot count and activation width; compressed tier images remain +refused. The entry uses the existing copy/place/fault helpers and the existing +subsequent go/publication protocol; no Flint router or Program is registered. + +The pure layout contract passed locally after an oxide build. +`hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes`, owned by +`gate-gpu-ds41-lib`, consumes the handoff's weights and emitted places through +the production tier accumulator and adds the host sum. It covers partial widths, +invalid ids, poisoned non-owning rows, trailing sentinels and changed replay +weights. Its host sum is precomputed and activation images are checked by +readback: it is not expert execution or two-card completion/publication proof. +The hardware case has not been run here. + +The new `ds41_ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +owning-gate evidence; the eight-slot and other existing entries must remain +unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. +Actual six-slot body/tier and two-card completion remain integration work, not +capabilities inferred from this primitive. diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 64caa9901..4af6e25ec 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -599,6 +599,8 @@ gguf 7 2 0 # `refuse_or`; nvfp4mma.rs `Mxfp8Out::block`; hc_gated.rs `up_mma_mix_body` with its read-back of the tile's # mixed values, and the two new entries' calls of their bodies), counted by tools/check-unsafe.sh. Each new # block has its SAFETY line. +# PIN(2026-10-25): gpu 2418 -> 2420 blocks, the six-slot F32 tier entry adds two +# handoff body calls under its checked launch contract; the newmain baseline is preserved. gpu 2640 334 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and From 61318d04376cfe1ddd304628d8c5b1713d0b6961 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:59:23 +0900 Subject: [PATCH 31/85] gpu: name the new six-slot tier primitive generically --- crates/gpu/src/host/handoff.rs | 12 ++++++------ docs/research/flint-attach.md | 6 +++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index cd453226c..e42f46a61 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,7 +18,7 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_6_tier` and +//! `crate::card_slots`' `ds41_ffn_places`. `ffn_handoff_6_tier` and //! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an //! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); @@ -310,7 +310,7 @@ mod handoff_kernels { tsel.len() >= 6 ) )] - pub fn ds41_ffn_handoff_6_tier( + pub fn ffn_handoff_6_tier( ids_in: &[u32], w_in: &[f32], map: &[u32], @@ -1347,7 +1347,7 @@ impl HandoffKernels { impl HandoffKernels { /// Enqueue the handoff of a tier layer of six or eight slots a token - /// (`ds41_ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// (`ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and @@ -1372,7 +1372,7 @@ impl HandoffKernels { let slots = check_tier(&lay, &tl)?; let what = match slots { HandoffSlots::Four => "ds41_ffn_handoff_4_tier", - HandoffSlots::Six => "ds41_ffn_handoff_6_tier", + HandoffSlots::Six => "ffn_handoff_6_tier", _ => "ds41_ffn_handoff_8_tier", }; let n = lay.hidden; @@ -1418,8 +1418,8 @@ impl HandoffKernels { )?; } HandoffSlots::Six => { - let prep = self.module.prepare_ds41_ffn_handoff_6_tier(cfg)?; - self.module.ds41_ffn_handoff_6_tier( + let prep = self.module.prepare_ffn_handoff_6_tier(cfg)?; + self.module.ffn_handoff_6_tier( stream, &prep, h.ids, diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 1df60b76d..b0a9b8b4e 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -126,11 +126,15 @@ weights. Its host sum is precomputed and activation images are checked by readback: it is not expert execution or two-card completion/publication proof. The hardware case has not been run here. -The new `ds41_ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and +The new `ffn_handoff_6_tier` entry needs add-class PTX/JIT/resource and owning-gate evidence; the eight-slot and other existing entries must remain unchanged. No PTX resource pin is added without an observed NVIDIA JIT run. Actual six-slot body/tier and two-card completion remain integration work, not capabilities inferred from this primitive. +`check-ptx-names` remains red until observed spill/JIT-local measurements can pin +the new entry in the four existing architecture/capability tables. Offline +no-JIT scans cannot supply that missing column, so no placeholder row is added. + From 6c2eaf8c5df2bade226ce79ed5084453142f2860 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:47:50 +0900 Subject: [PATCH 32/85] gpu: implement the common optional-bias 256/top6 floor router --- crates/gpu-deepseek41/src/experts_mxfp4.rs | 2 +- crates/gpu-gates/src/bin/gate_flint_router.rs | 396 +++++++++++------ crates/gpu/src/route_body.rs | 417 ++++++++++++++++-- crates/gpu/src/route_core.rs | 12 +- crates/gpu/src/routed_block.rs | 20 +- crates/model/src/arch/flint/tests.rs | 13 +- crates/model/tests/flint_meta.rs | 10 +- crates/models/src/shape.rs | 151 +++++-- docs/research/flint-attach.md | 72 +-- 9 files changed, 838 insertions(+), 255 deletions(-) diff --git a/crates/gpu-deepseek41/src/experts_mxfp4.rs b/crates/gpu-deepseek41/src/experts_mxfp4.rs index 6c2888207..033f5f291 100644 --- a/crates/gpu-deepseek41/src/experts_mxfp4.rs +++ b/crates/gpu-deepseek41/src/experts_mxfp4.rs @@ -182,7 +182,7 @@ mod dflash_kernels { // N_EXPERT = n_expert <= bias.len() by the launch contract; score and // sel_v are block-shared arrays of N_EXPERT entries; the barrier below // publishes them. - unsafe { publish_n::(probs.as_mut_ptr(), bias, e0, score, sel_v, tid, fault) }; + unsafe { publish_n::(probs.as_mut_ptr(), Some(bias), e0, score, sel_v, tid, fault) }; thread::sync_threads(); if tid < 32 { diff --git a/crates/gpu-gates/src/bin/gate_flint_router.rs b/crates/gpu-gates/src/bin/gate_flint_router.rs index 790d7f152..fbaee13b9 100644 --- a/crates/gpu-gates/src/bin/gate_flint_router.rs +++ b/crates/gpu-gates/src/bin/gate_flint_router.rs @@ -1,18 +1,6 @@ -//! Flint's 256-expert top-6 sigmoid router (`sigmoid_256_6`), model-free. -//! -//! One token. The weight's column 0 holds the logits and the other columns -//! are 0; the activation is 1 at column 0, so each logit is one product. -//! Bias is 0 and the scale is 1. The logits fall strictly through the first -//! sixteen experts and sit at −20 after that, so the sixth and the seventh -//! do not tie. -//! -//! The slot count checked here is 6, not the instance's `N_USED`. The ids -//! equal the host top 6 of the kernel's scores plus bias (larger value, and -//! a tie to the larger index) and the host sigmoid ranking of the planted -//! logits. The weights are bit-identical to the host renorm of those scores -//! (f64 slot sum, narrowed, `renorm_divisor`, times the scale) and differ -//! from the first 6 weights of a top-7 renorm. A NaN logit raises `Router`; -//! a clean launch raises nothing. +//! The actual 256/top-6 selected-sum-floor binding, through RouterEntry. +//! Optional selection-only bias, active floor/zero sums, ties, exact rows, +//! nonfinite refusal and graph replay. This gate needs an NVIDIA device. #[cfg(feature = "gpu")] fn main() -> std::process::ExitCode { @@ -21,49 +9,79 @@ fn main() -> std::process::ExitCode { #[cfg(feature = "gpu")] mod gate { - use bloomery_gpu::route_body::sigmoid_256_6; - use bloomery_gpu::route_core::{renorm_divisor, sigmoid}; + use bloomery_gpu::gemm::Gemm32Kernels; + use bloomery_gpu::route_body::sigmoid_floor_256_6::{RouterKernels, RouterOut}; + use bloomery_gpu::routed_block::{RouteIn, RoutedRows, RouterEntry}; + use bloomery_gpu::route_core::sigmoid; use bloomery_gpu::{DeviceTensor, Fault, FaultSite, Gpu, LAYER_NONE}; use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, verdict}; use cuda_core::DeviceBuffer; - /// Experts, the pick this gate names, and the activation width. The pick - /// is not read from the instance. const EXPERTS: usize = 256; const USED: usize = 6; - const TOP7: usize = 7; - const K: usize = 32; - const SCALE: f32 = 1.0; - - fn logit(expert: usize) -> f32 { - if expert < 16 { - 20.0 - expert as f32 - } else { - -20.0 - } + const K: usize = 64; + const TOKENS: usize = 5; + const SCALE: f32 = 1.75; + const FLOOR: f32 = 1.0 / 16384.0; + + struct Cx { + gpu: Gpu, + router: RouterKernels, + wide: Gemm32Kernels, + } + + #[derive(Default)] + struct Pick { + scores: Vec, + ids: Vec, + weights: Vec, } - fn weight_matrix() -> Vec { - let mut w = vec![0.0f32; EXPERTS * K]; - for e in 0..EXPERTS { - w[e * K] = logit(e); + impl Pick { + fn same(&self, other: &Self) -> bool { + bits_equal(&self.scores, &other.scores) + && self.ids == other.ids + && bits_equal(&self.weights, &other.weights) } - w } - /// The top `k` of `key`, larger value first, a tie to the larger index. fn top_k(key: &[f32], k: usize) -> Vec { - let mut idx: Vec = (0..key.len() as u32).collect(); - idx.sort_by(|&a, &b| key[b as usize].total_cmp(&key[a as usize]).then(b.cmp(&a))); - idx.truncate(k); - idx + let mut ids: Vec = (0..EXPERTS as u32).collect(); + ids.sort_by(|&a, &b| key[b as usize].total_cmp(&key[a as usize]).then(b.cmp(&a))); + ids.truncate(k); + ids } - /// Unbiased scores at `ids`, summed in slot order in f64, narrowed, guarded, scaled. + // Independent contract expression: do not reuse the device divisor. fn weights_of(scores: &[f32], ids: &[u32], scale: f32) -> Vec { - let g: Vec = ids.iter().map(|&i| scores[i as usize]).collect(); - let div = renorm_divisor(g.iter().fold(0.0f64, |s, &v| s + f64::from(v)) as f32); - g.into_iter().map(|v| v / div * scale).collect() + let sum = ids.iter().fold(0.0f64, |sum, &e| sum + f64::from(scores[e as usize])); + let div = (sum as f32).max(FLOOR); + ids.iter().map(|&e| scores[e as usize] / div * scale).collect() + } + + fn matrix(logits: &[f32]) -> Vec { + let mut w = vec![0.0; EXPERTS * K]; + for (e, &logit) in logits.iter().enumerate() { + w[e * K] = logit; + } + w + } + + fn activations(factors: &[f32]) -> Vec { + let mut x = vec![0.0; factors.len() * K]; + for (t, &factor) in factors.iter().enumerate() { + x[t * K] = factor; + } + x + } + + fn read_one(cx: &Cx, out: &RouterOut) -> Result { + let s = cx.gpu.stream(); + Ok(Pick { + scores: out.probs.to_host_vec(s)?, + ids: out.ids.to_host_vec(s)?, + weights: out.weights.to_host_vec(s)?, + }) } fn clause(name: &str, pass: bool, detail: &str) -> bool { @@ -71,101 +89,217 @@ mod gate { pass } - pub fn run() -> Result<(), GateError> { - bloomery_levers::at_main(&[])?; - let gpu = Gpu::new()?; - println!("gate_flint_router: device {}", gpu.device_name()?); - let s = gpu.stream(); - let rk = sigmoid_256_6::RouterKernels::load(gpu.context())?; - let planted: Vec = (0..EXPERTS).map(logit).collect(); - let by_logit = top_k( - &planted.iter().copied().map(sigmoid).collect::>(), - USED, - ); - let w_host = weight_matrix(); - let x_host = { - let mut x = vec![0.0f32; K]; - x[0] = 1.0; - x - }; - let bias_host = vec![0.0f32; EXPERTS]; - let w = DeviceTensor::upload(s, &w_host, EXPERTS, K)?; - let x = DeviceBuffer::from_host(s, &x_host)?; - let bias = DeviceBuffer::from_host(s, &bias_host)?; - let mut out = sigmoid_256_6::RouterOut::new(s)?; - gpu.take_fault()?; - rk.enqueue_router(s, &w, &x, &bias, SCALE, &mut out, gpu.unlabelled_sink())?; + fn valid_case(cx: &Cx, name: &str, logits: &[f32], bias: Option<&[f32]>) -> Result { + let s = cx.gpu.stream(); + let mut wh = matrix(logits); + if name == "dense" { + for e in 0..EXPERTS { + for j in 0..K { + wh[e * K + j] = ((e * 17 + j * 11) % 67) as f32 / 67.0 - 0.5; + } + } + } + let w = DeviceTensor::upload(s, &wh, EXPERTS, K)?; + let factors = [1.0, 0.0, 2.0, -1.0, 0.5]; + let mut xh = activations(&factors); + if name == "dense" { + for t in 0..TOKENS { + for j in 1..K { + xh[t * K + j] = ((t * 13 + j * 7) % 31) as f32 / 31.0 - 0.5; + } + } + } + let x = DeviceBuffer::from_host(s, &xh)?; + let b = bias.map(|b| DeviceBuffer::from_host(s, b)).transpose()?; + let input = RouteIn { w: &w, bias: b.as_ref(), scale: SCALE, fault: cx.gpu.unlabelled_sink() }; + let mut out = RouterOut::new(s)?; + let mut single = Pick::default(); + let mut first = Pick::default(); + let mut tickets = true; + cx.gpu.take_fault()?; + for t in 0..TOKENS { + let xt = DeviceBuffer::from_host(s, &xh[t * K..(t + 1) * K])?; + cx.router.route(s, &xt, &input, &mut out)?; + s.synchronize()?; + let pick = read_one(cx, &out)?; + tickets &= out.tickets(s)? == 0; + if t == 0 { + first = read_one(cx, &out)?; + } + single.scores.extend(pick.scores); + single.ids.extend(pick.ids); + single.weights.extend(pick.weights); + } + let mut probs = DeviceBuffer::zeroed(s, EXPERTS * TOKENS)?; + let mut ids = DeviceBuffer::zeroed(s, USED * TOKENS)?; + let mut weights = DeviceBuffer::zeroed(s, USED * TOKENS)?; + cx.router.route_rows( + (s, &cx.gpu, &cx.wide), &x, &input, TOKENS, + RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, + )?; s.synchronize()?; - let scores = out.probs.to_host_vec(s)?; - let ids = out.ids.to_host_vec(s)?; - let weights = out.weights.to_host_vec(s)?; - let clean = gpu.take_fault()?; - - let by_score = top_k( - &scores - .iter() - .zip(&bias_host) - .map(|(&p, &b)| p + b) - .collect::>(), - USED, - ); - let len_ok = ids.len() == USED && weights.len() == USED; - let ids_ok = len_ok && ids == by_score && ids == by_logit; - let expect = weights_of(&scores, &by_score, SCALE); - let w_ok = weights.len() == USED && expect.len() == USED && bits_equal(&weights, &expect); - let by7 = top_k( - &scores - .iter() - .zip(&bias_host) - .map(|(&p, &b)| p + b) - .collect::>(), - TOP7, - ); - let w7 = weights_of(&scores, &by7, SCALE); - let differ = - weights.len() == USED && w7.len() == TOP7 && !bits_equal(&weights, &w7[..USED]); - + let batch = Pick { + scores: probs.to_host_vec(s)?, ids: ids.to_host_vec(s)?, weights: weights.to_host_vec(s)?, + }; + let clean = cx.gpu.take_fault()?.is_none(); + let mut expected = true; + for t in 0..TOKENS { + let scores = &single.scores[t * EXPERTS..(t + 1) * EXPERTS]; + let keys: Vec = scores.iter().enumerate() + .map(|(e, &p)| bias.map_or(p, |b| p + b[e])).collect(); + let want_ids = top_k(&keys, USED); + expected &= single.ids[t * USED..(t + 1) * USED] == want_ids; + if name != "dense" { + let planted_keys: Vec = logits.iter().enumerate().map(|(e, &logit)| { + let p = sigmoid(logit * factors[t]); + bias.map_or(p, |b| p + b[e]) + }).collect(); + expected &= single.ids[t * USED..(t + 1) * USED] == top_k(&planted_keys, USED); + } + expected &= bits_equal( + &single.weights[t * USED..(t + 1) * USED], &weights_of(scores, &want_ids, SCALE), + ); + } let mut ok = clause( - "router ids top 6", - ids_ok && clean.is_none(), - &format!( - "len={} ids={ids:?} by_score={by_score:?} by_logit={by_logit:?} fault={clean:?}", - ids.len() - ), - ); - ok &= clause( - "router weights renorm of the top 6", - w_ok && clean.is_none(), - &format!("bit_identical={w_ok} slots={}", weights.len()), - ); - ok &= clause( - "router weights differ from a top-7 renorm", - differ && clean.is_none(), - &format!("differ={differ}"), + &format!("{name} selection and unbiased floor weights"), + expected && clean && tickets, + &format!("bias={} clean={clean} tickets_zero={tickets}", bias.is_some()), ); + ok &= clause(&format!("{name} token-major rows"), single.same(&batch), "bit_identical"); + if name.starts_with("tiny") { + let sum: f32 = first.ids.iter().map(|&e| first.scores[e as usize]).sum(); + let legacy: Vec = first.ids.iter() + .map(|&e| first.scores[e as usize] / (sum + 1e-20) * SCALE).collect(); + ok &= clause("tiny floor active", sum > 0.0 && sum < FLOOR + && !bits_equal(&first.weights, &legacy) + && first.weights.iter().sum::() < SCALE, "not epsilon renormalization"); + } + if name == "zero" || name == "ties" { + ok &= clause(&format!("{name} descending-id ties"), + first.ids == [255, 254, 253, 252, 251, 250], "six distinct ids"); + if name == "zero" { + ok &= clause("zero selected sum", first.weights.iter().all(|&w| w == 0.0), "finite zero weights"); + } + } + if let Some(bias) = bias { + let keys: Vec = first.scores.iter().zip(bias).map(|(&p, &b)| p + b).collect(); + ok &= clause("bias changes selection not weight source", + first.ids != top_k(&first.scores, USED) + && !bits_equal(&first.weights, &weights_of(&keys, &first.ids, SCALE)), + "unbiased selected scores consumed"); + } else if name == "normal" { + let top7 = top_k(&first.scores, 7); + ok &= clause("six not seven selected scores", + !bits_equal(&first.weights, &weights_of(&first.scores, &top7, SCALE)[..USED]), + "top-seven renormalization differs"); + } + + let x1 = DeviceBuffer::from_host(s, &xh[..K])?; + let graph = cx.gpu.capture(|st| cx.router.route(st, &x1, &input, &mut out))?; + let mut replay = graph.node_count() == 1; + for _ in 0..2 { + graph.launch(s)?; + s.synchronize()?; + replay &= read_one(cx, &out)?.same(&first) && out.tickets(s)? == 0; + } + ok &= clause(&format!("{name} single capture"), replay, "one node, two exact replays, zero tickets"); + let graph = cx.gpu.capture(|st| cx.router.route_rows( + (st, &cx.gpu, &cx.wide), &x, &input, TOKENS, + RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, + ))?; + let mut replay = graph.node_count() == 3; + for _ in 0..2 { + graph.launch(s)?; + s.synchronize()?; + let now = Pick { + scores: probs.to_host_vec(s)?, ids: ids.to_host_vec(s)?, weights: weights.to_host_vec(s)?, + }; + replay &= now.same(&single); + } + replay &= cx.gpu.take_fault()?.is_none(); + ok &= clause(&format!("{name} rows capture"), replay, "three nodes, two exact replays, no fault"); + Ok(ok) + } - let mut nan_w = w_host; - nan_w[40 * K] = f32::NAN; - let nan_w = DeviceTensor::upload(s, &nan_w, EXPERTS, K)?; - let mut nan_out = sigmoid_256_6::RouterOut::new(s)?; - gpu.take_fault()?; - rk.enqueue_router( - s, - &nan_w, - &x, - &bias, - SCALE, - &mut nan_out, - gpu.unlabelled_sink(), + fn invalid_case(cx: &Cx, name: &str, logit: Option, bias: Option, scale: f32) -> Result { + let s = cx.gpu.stream(); + let mut logits = vec![0.0; EXPERTS]; + if let Some(logit) = logit { logits[40] = logit; } + let w = DeviceTensor::upload(s, &matrix(&logits), EXPERTS, K)?; + let mut bh = vec![0.0; EXPERTS]; + if let Some(bias) = bias { bh[40] = bias; } + let b = bias.map(|_| DeviceBuffer::from_host(s, &bh)).transpose()?; + let x = DeviceBuffer::from_host(s, &activations(&[1.0, 1.0]))?; + let input = RouteIn { w: &w, bias: b.as_ref(), scale, fault: cx.gpu.unlabelled_sink() }; + let mut out = RouterOut::new(s)?; + cx.gpu.take_fault()?; + cx.router.route(s, &x, &input, &mut out)?; + s.synchronize()?; + let one = cx.gpu.take_fault()?; + let one_poisoned = out.weights.to_host_vec(s)?.iter().all(|w| !w.is_finite()); + let tickets = out.tickets(s)?; + let mut probs = DeviceBuffer::zeroed(s, EXPERTS * 2)?; + let mut ids = DeviceBuffer::zeroed(s, USED * 2)?; + let mut weights = DeviceBuffer::zeroed(s, USED * 2)?; + cx.router.route_rows( + (s, &cx.gpu, &cx.wide), &x, &input, 2, + RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, )?; s.synchronize()?; - let nan_fault = gpu.take_fault()?; + let rows = cx.gpu.take_fault()?; + let rows_poisoned = weights.to_host_vec(s)?.iter().all(|w| !w.is_finite()); let want = Some(Fault::at(LAYER_NONE, FaultSite::Router)); - ok &= clause( - "router NaN logit", - nan_fault == want, - &format!("fault={nan_fault:?} want={want:?}"), - ); + Ok(clause(name, one == want && rows == want && one_poisoned && rows_poisoned && tickets == 0, + &format!("one={one:?} rows={rows:?} poisoned={} tickets={tickets}", one_poisoned && rows_poisoned))) + } + + fn host_boundaries(cx: &Cx) -> Result { + let s = cx.gpu.stream(); + let w = DeviceTensor::upload(s, &matrix(&vec![0.0; EXPERTS]), EXPERTS, K)?; + let x = DeviceBuffer::zeroed(s, K)?; + let short_bias = DeviceBuffer::zeroed(s, EXPERTS - 1)?; + let mut out = RouterOut::new(s)?; + let sink = cx.gpu.unlabelled_sink(); + let short = cx.router.enqueue_router(s, &w, &x, Some(&short_bias), SCALE, &mut out, sink).is_err(); + let mut probs = DeviceBuffer::zeroed(s, EXPERTS)?; + let mut ids = DeviceBuffer::zeroed(s, USED)?; + let mut weights = DeviceBuffer::zeroed(s, USED)?; + let empty = cx.router.enqueue_router_rows(s, &cx.gpu, &cx.wide, &w, &x, None, + SCALE, 0, &mut probs, &mut ids, &mut weights, sink).is_err(); + let short_rows = cx.router.enqueue_router_rows(s, &cx.gpu, &cx.wide, &w, &x, None, + SCALE, 2, &mut probs, &mut ids, &mut weights, sink).is_err(); + let legacy = bloomery_gpu::route_body::sigmoid_256_6::RouterKernels::load(cx.gpu.context())?; + let mut legacy_out = bloomery_gpu::route_body::sigmoid_256_6::RouterOut::new(s)?; + let missing = legacy.route(s, &x, + &RouteIn { w: &w, bias: None, scale: SCALE, fault: sink }, &mut legacy_out).is_err(); + Ok(clause("host boundaries", short && empty && short_rows && missing, + "short bias, empty/short rows, legacy missing bias refused")) + } + + pub fn run() -> Result<(), GateError> { + bloomery_levers::at_main(&[])?; + let gpu = Gpu::new()?; + println!("gate_flint_router: device {}", gpu.device_name()?); + let router = RouterKernels::load(gpu.context())?; + let wide = Gemm32Kernels::load(gpu.context())?; + let cx = Cx { gpu, router, wide }; + let normal: Vec = (0..EXPERTS).map(|e| if e < 16 { 2.0 - e as f32 * 0.25 } else { -20.0 }).collect(); + let tiny: Vec = (0..EXPERTS).map(|e| -30.0 - (e % 16) as f32 * 0.25).collect(); + let mut bias = vec![0.0; EXPERTS]; + for (e, b) in bias.iter_mut().enumerate().skip(240) { *b = 2.0 + e as f32 / 256.0; } + let mut ok = valid_case(&cx, "normal", &normal, None)?; + ok &= valid_case(&cx, "dense", &normal, None)?; + ok &= valid_case(&cx, "biased", &normal, Some(&bias))?; + ok &= valid_case(&cx, "tiny", &tiny, None)?; + ok &= valid_case(&cx, "tiny-biased", &tiny, Some(&bias))?; + ok &= valid_case(&cx, "zero", &vec![-200.0; EXPERTS], None)?; + ok &= valid_case(&cx, "ties", &vec![0.0; EXPERTS], None)?; + for (name, value) in [("NaN", f32::NAN), ("+infinity", f32::INFINITY), ("-infinity", f32::NEG_INFINITY)] { + ok &= invalid_case(&cx, &format!("{name} logit"), Some(value), None, SCALE)?; + ok &= invalid_case(&cx, &format!("{name} selection bias"), None, Some(value), SCALE)?; + } + ok &= invalid_case(&cx, "nonfinite weight scale", None, None, f32::INFINITY)?; + ok &= host_boundaries(&cx)?; if ok { Ok(()) } else { Err(checks_failed()) } } } diff --git a/crates/gpu/src/route_body.rs b/crates/gpu/src/route_body.rs index 0bf4cd545..6058f9154 100644 --- a/crates/gpu/src/route_body.rs +++ b/crates/gpu/src/route_body.rs @@ -27,7 +27,7 @@ //! ones. use crate::q8f32::f32_lane_partial_1col_w32; -use crate::route_core::{Score, renorm_divisor, router_score, take}; +use crate::route_core::{Score, floor_divisor, renorm_divisor, router_score, take}; use crate::{FaultSink, FaultSite}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicU32}; use cuda_device::{DisjointSlice, thread, threadfence, warp}; @@ -44,20 +44,20 @@ const _: () = assert!(ROUTER_THREADS_U32 as usize == ROUTER_THREADS); // The entries' `launch_bounds` and `launch_contract` block are literals. const _: () = assert!(ROUTER_THREADS == 128); -/// V4.1's `publish` over `E` experts: thread `tid` parks `score[e] = probs[at + -/// e]` (a volatile load) and `sel_v[e] = score[e] + bias[e]` for `e = tid + -/// ROUTER_THREADS · i`, raising [`FaultSite::Router`] on a selection value -/// that is not finite. +/// Publish `E` scores: thread `tid` parks `score[e] = probs[at + e]` +/// (a volatile load) and the same value plus any supplied selection bias in +/// `sel_v[e]`, for `e = tid + ROUTER_THREADS · i`. A nonfinite selection +/// value raises [`FaultSite::Router`]. /// /// # Safety /// -/// `probs` addresses at least `at + E` values, `bias` holds `E`, +/// `probs` addresses at least `at + E` values, any `bias` holds `E`, /// `score` and `sel_v` are the block's shared arrays of `E`, and the caller's /// barrier follows before any read of them. #[inline(always)] pub unsafe fn publish_n( probs: *const f32, - bias: &[f32], + bias: Option<&[f32]>, at: usize, score: *mut f32, sel_v: *mut f32, @@ -68,8 +68,11 @@ pub unsafe fn publish_n( while e < E { // SAFETY: at + e < at + E, inside probs by this fn's contract. let p = unsafe { core::ptr::read_volatile(probs.add(at + e)) }; - // SAFETY: e < E <= bias.len() by this fn's contract. - let v = p + unsafe { *bias.get_unchecked(e) }; + // SAFETY: any supplied bias holds E entries by this fn's contract. + let v = match bias { + Some(bias) => p + unsafe { *bias.get_unchecked(e) }, + None => p, + }; if !v.is_finite() { fault.raise(FaultSite::Router); } @@ -86,12 +89,12 @@ pub unsafe fn publish_n( /// V4.1's `select` over `E` experts and `U` slots: `U` rounds of the butterfly /// argmax under [`take`]`::`, lane `L` scanning experts `L + 32 j < E` /// (`j < ⌈E / 32⌉`) no earlier round took; lane 0 writes each round's id to -/// `ids[at + s]`, then sums the `U` scores in f64 in slot order, takes -/// [`renorm_divisor`] of the sum narrowed to f32, and writes each weight -/// `score / sum · scale` to `weights[at + s]`, raising [`FaultSite::Router`] -/// on one that is not finite. With `REFUSE_NAN`, a non-finite selection -/// value writes NaN weights instead: [`take`] never selects a NaN, so the -/// ids stay the finite rank order and no slot's weight is a defined pick. +/// `ids[at + s]`, then sums the `U` scores in f64 in slot order. The divisor +/// is [`floor_divisor`] with `FLOOR`, [`renorm_divisor`] otherwise, of the +/// sum narrowed to f32. Each weight is `score / divisor · scale`, raising +/// [`FaultSite::Router`] on one that is not finite. With `REFUSE_NAN`, any +/// nonfinite selection value writes NaN weights instead, so the ids are +/// not a defined pick. /// /// # Safety /// @@ -104,7 +107,7 @@ pub unsafe fn publish_n( reason = "the selecting block's shared arrays, its lane and scale, the fault and the outputs" )] #[inline(always)] -pub unsafe fn select_n( +pub unsafe fn select_n( score: *const f32, sel_v: *const f32, slot_p: *mut f32, @@ -196,7 +199,11 @@ pub unsafe fn select_n( sum += f64::from(unsafe { *slot_p.add(s) }); s += 1; } - let sum = renorm_divisor(sum as f32); + let sum = if const { FLOOR } { + floor_divisor(sum as f32) + } else { + renorm_divisor(sum as f32) + }; let mut s = 0usize; while s < U { // SAFETY: s < U, this lane's own write. @@ -222,7 +229,7 @@ pub unsafe fn select_n( /// /// the grid is `E / ROWS_PER_BLOCK` blocks of [`ROUTER_THREADS`] /// (the launcher's), every thread arrives converged; `n_expert == E`, `w` -/// holds `n_expert · k` values, `x` `k`, `bias`, `logits` and `probs` +/// holds `n_expert · k` values, `x` `k`, any `bias`, `logits` and `probs` /// `n_expert`, `ids` and `weights` `U`, `done` one, zero before the launch; /// `k` is a positive multiple of 32; `last`, `score`, `sel_v` and `slot_p` /// are this block's own shared arrays of 1, `E`, `E` and `U` entries. @@ -231,10 +238,10 @@ pub unsafe fn select_n( reason = "a kernel entry's arguments and the shared arrays it declares (rust-quality R8)" )] #[inline(always)] -pub unsafe fn router_body( +pub unsafe fn router_body( w: &[f32], x: &[f32], - bias: &[f32], + bias: Option<&[f32]>, n_expert: u32, k: u32, scale: f32, @@ -297,7 +304,7 @@ pub unsafe fn router_body(score, sel_v, slot_p, lane, scale, fault, ids, weights, 0) + select_n::(score, sel_v, slot_p, lane, scale, fault, ids, weights, 0) }; if lane == 0 { // SAFETY: `ticket` is done[0], inside `done` by this fn's @@ -316,7 +323,7 @@ pub unsafe fn router_body( +pub unsafe fn pick_body( probs: &[f32], - bias: &[f32], + bias: Option<&[f32]>, n_expert: u32, t: u32, scale: f32, @@ -352,7 +359,7 @@ pub unsafe fn pick_body( // ids and weights hold U entries from U · token by this fn's // contract. unsafe { - select_n::( + select_n::( score, sel_v, slot_p, @@ -538,10 +545,10 @@ macro_rules! sigmoid_router_instance { // contract's, the launcher passes k a positive multiple of 32, // and `done[0]` is zero before the launch. unsafe { - router_body::( + router_body::( w, x, - bias, + Some(bias), n_expert, k, scale, @@ -599,9 +606,9 @@ macro_rules! sigmoid_router_instance { // reaches each `static mut` without a reference); the block is // 128 threads, converged; the lengths are the launch contract's. unsafe { - pick_body::( + pick_body::( probs, - bias, + Some(bias), n_expert, t, scale, @@ -851,7 +858,11 @@ macro_rules! sigmoid_router_instance { r: &$crate::routed_block::RouteIn<'_>, out: &mut RouterOut, ) -> Result<(), GpuError> { - self.enqueue_router(stream, r.w, x, r.bias, r.scale, out, r.fault) + let bias = r.bias.ok_or($crate::GpuError::State { + what: concat!(stringify!($m), "::route"), + missing: "the required selection bias", + })?; + self.enqueue_router(stream, r.w, x, bias, r.scale, out, r.fault) } fn route_rows( @@ -862,13 +873,17 @@ macro_rules! sigmoid_router_instance { t: usize, rows: $crate::routed_block::RoutedRows<'_>, ) -> Result<(), GpuError> { + let bias = r.bias.ok_or($crate::GpuError::State { + what: concat!(stringify!($m), "::route_rows"), + missing: "the required selection bias", + })?; self.enqueue_router_rows( stream, gpu, wide, r.w, x, - r.bias, + bias, r.scale, t, rows.probs, @@ -946,3 +961,343 @@ crate::sigmoid_router_instance! { refuse_nan: true, } } + +/// A fixed selected-sum-floor sigmoid binding over 256 experts and six +/// slots, with an actual optional selection-only bias. Shares the score, +/// publication, selection and sum order with the legacy instances above. +pub mod sigmoid_floor_256_6 { + use super::{ROUTER_THREADS_U32, ROWS_PER_BLOCK, pick_body, router_body}; + use crate::gemm::Gemm32Kernels; + use crate::route_core::Sigmoid; + use crate::routed_block::{RouteIn, RoutedRows, RouterEntry}; + use crate::{DeviceTensor, FaultSink, Gpu, GpuError, launch_u32}; + use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; + use cuda_device::{DisjointSlice, SharedArray, kernel, launch_bounds, launch_contract}; + use cuda_host::cuda_module; + use model::arch::models::shape::{self, RouterBody, RouterInst, rules}; + use std::sync::Arc; + + pub const N_EXPERT: usize = 256; + pub const N_USED: usize = 6; + /// Both metadata bindings name the same optional-bias device entries. + pub const ROW: RouterInst = + shape::router_row_rule(RouterBody::SigmoidFloor, rules::SIGMOID_FLOOR, 256, 6); + pub const BIASED_ROW: RouterInst = + shape::router_row_rule(RouterBody::SigmoidFloor, rules::BIASED_SIGMOID_FLOOR, 256, 6); + const _: () = assert!(shape::router_row_is(ROW, rules::SIGMOID_FLOOR, false, (6, 6))); + const _: () = assert!(shape::router_row_is( + BIASED_ROW, rules::BIASED_SIGMOID_FLOOR, false, (6, 6) + )); + const _: () = assert!(N_EXPERT.is_multiple_of(ROWS_PER_BLOCK)); + const _: () = assert!(N_EXPERT.div_ceil(32) <= 32); + + #[allow( + dead_code, + reason = "#[cuda_module] derives unchecked launchers that no caller uses" + )] + #[cuda_module] + mod sigmoid_floor_256_6_router_kernels { + use super::*; + + /// One token, with `has_bias` distinguishing an unused borrowed + /// input slice from a real selection bias. No zero-bias storage. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + w.len() >= 256 * k, + x.len() >= k, + bias.len() >= 256, + logits.len() >= 256, + probs.len() >= 256, + ids.len() >= 6, + weights.len() >= 6, + done.len() >= 1 + ) + )] + pub fn sigmoid_floor_256_6_router( + w: &[f32], + x: &[f32], + bias: &[f32], + has_bias: u32, + k: u32, + scale: f32, + mut logits: DisjointSlice, + mut probs: DisjointSlice, + mut ids: DisjointSlice, + mut weights: DisjointSlice, + mut done: DisjointSlice, + fault: FaultSink, + ) { + static mut SCORE: SharedArray = SharedArray::UNINIT; + static mut SEL_V: SharedArray = SharedArray::UNINIT; + static mut LAST: SharedArray = SharedArray::UNINIT; + static mut SLOT_P: SharedArray = SharedArray::UNINIT; + // SAFETY: the launch contract supplies every input/output length; + // the launcher uses k a positive multiple of 32, 64 blocks of 128 + // converged threads, and done is zero before each ordered launch. + // Shared arrays belong to this block and have the body's sizes. + unsafe { + router_body::( + w, x, if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, k, scale, + &mut logits, &mut probs, &mut ids, &mut weights, &mut done, fault, + SharedArray::as_raw_mut_ptr(&raw mut LAST), + SharedArray::as_raw_mut_ptr(&raw mut SCORE), + SharedArray::as_raw_mut_ptr(&raw mut SEL_V), + SharedArray::as_raw_mut_ptr(&raw mut SLOT_P), + ); + } + } + + /// Each block selects one token's scores, with the same optional + /// bias and floor as the one-token entry's final block. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + probs.len() >= 256 * t, + bias.len() >= 256, + ids.len() >= 6 * t, + weights.len() >= 6 * t + ) + )] + pub fn sigmoid_floor_256_6_router_pick( + probs: &[f32], + bias: &[f32], + has_bias: u32, + t: u32, + scale: f32, + mut ids: DisjointSlice, + mut weights: DisjointSlice, + fault: FaultSink, + ) { + static mut SCORE: SharedArray = SharedArray::UNINIT; + static mut SEL_V: SharedArray = SharedArray::UNINIT; + static mut SLOT_P: SharedArray = SharedArray::UNINIT; + // SAFETY: launch-contract lengths and 128 converged threads per + // block; shared arrays belong to this block at the body's sizes. + unsafe { + pick_body::( + probs, if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, t, scale, &mut ids, &mut weights, fault, + SharedArray::as_raw_mut_ptr(&raw mut SCORE), + SharedArray::as_raw_mut_ptr(&raw mut SEL_V), + SharedArray::as_raw_mut_ptr(&raw mut SLOT_P), + ); + } + } + } + + /// One token's expert logits/scores and six rank-ordered ids/weights. + /// Launches sharing this output must be ordered on one stream. + pub struct RouterOut { + pub logits: DeviceBuffer, + pub probs: DeviceBuffer, + pub ids: DeviceBuffer, + pub weights: DeviceBuffer, + done: DeviceBuffer, + } + + impl RouterOut { + /// Allocate result storage at load time, including the zero ticket. + pub fn new(stream: &CudaStream) -> Result { + Ok(Self { + logits: DeviceBuffer::zeroed(stream, N_EXPERT)?, + probs: DeviceBuffer::zeroed(stream, N_EXPERT)?, + ids: DeviceBuffer::zeroed(stream, N_USED)?, + weights: DeviceBuffer::zeroed(stream, N_USED)?, + done: DeviceBuffer::zeroed(stream, 1)?, + }) + } + + /// The ticket is zero between ordered launches, including replay. + pub fn tickets(&self, stream: &CudaStream) -> Result { + Ok(self.done.to_host_vec(stream)?[0]) + } + } + + /// Loaded floor router entries; all enqueue paths are allocation-free + /// and capturable, and borrow the optional bias rather than materialize it. + pub struct RouterKernels { + module: sigmoid_floor_256_6_router_kernels::LoadedModule, + } + + fn weight_k(what: &'static str, w: &DeviceTensor) -> Result { + let k = w.cols(); + if w.rows() != N_EXPERT || k == 0 || !k.is_multiple_of(32) { + return Err(GpuError::shape( + what, + format!("router weight is {} x {k}, want 256 rows of a positive multiple of 32", w.rows()), + )); + } + Ok(k) + } + + fn check_bias(what: &'static str, bias: Option<&DeviceBuffer>) -> Result<(), GpuError> { + if let Some(bias) = bias { + if bias.len() < N_EXPERT { + return Err(GpuError::shape( + what, format!("selection bias has {} entries, want at least 256", bias.len()), + )); + } + } + Ok(()) + } + + impl RouterKernels { + /// Load the embedded module once, before capture. + pub fn load(ctx: &Arc) -> Result { + // SAFETY: this crate owns the module's embedded device bundle. + let module = unsafe { crate::shared_module!(sigmoid_floor_256_6_router_kernels, ctx)? }; + Ok(Self { module }) + } + + /// Six largest sigmoid scores, larger id on ties. Bias changes + /// selection only; unbiased selected scores are summed in f64 slot + /// order, narrowed to f32, divided by max(sum, 2^-14), then scaled. + /// Nonfinite logits, bias-derived keys or weights raise Router and + /// an invalid key poisons every slot weight instead of routing it. + #[allow( + clippy::too_many_arguments, + reason = "the launcher's inputs are the kernel's (rust-quality R8)" + )] + pub fn enqueue_router( + &self, + stream: &CudaStream, + w: &DeviceTensor, + x: &DeviceBuffer, + bias: Option<&DeviceBuffer>, + scale: f32, + out: &mut RouterOut, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "sigmoid_floor_256_6::enqueue_router"; + let k = weight_k(what, w)?; + check_bias(what, bias)?; + if x.len() < k { + return Err(GpuError::shape(what, format!("x.len() {} < k {k}", x.len()))); + } + let k = launch_u32(what, "k", k)?; + let prep = self.module.prepare_sigmoid_floor_256_6_router( + LaunchConfig1D::new((N_EXPERT / ROWS_PER_BLOCK) as u32, ROUTER_THREADS_U32, 0), + )?; + let RouterOut { logits, probs, ids, weights, done } = out; + // With no bias the weight slice is only an ABI borrow: the + // kernel takes None and never loads it as a selection bias. + self.module.sigmoid_floor_256_6_router( + stream, &prep, w.buf(), x, bias.unwrap_or(w.buf()), u32::from(bias.is_some()), + k, scale, logits, probs, ids, weights, done, fault, + )?; + Ok(()) + } + + /// Token-major batch equivalent to ordered single-token launches. + /// The common F32 tile requires k a positive multiple of 64 and + /// aligned weight/input buffers. All buffers are caller-provided; + /// no optional-bias scratch is needed. + #[allow( + clippy::too_many_arguments, + reason = "router inputs, token count and the batch's output buffers (rust-quality R8)" + )] + pub fn enqueue_router_rows( + &self, + stream: &CudaStream, + gpu: &Gpu, + wide: &Gemm32Kernels, + w: &DeviceTensor, + x: &DeviceBuffer, + bias: Option<&DeviceBuffer>, + scale: f32, + t: usize, + probs: &mut DeviceBuffer, + ids: &mut DeviceBuffer, + weights: &mut DeviceBuffer, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "sigmoid_floor_256_6::enqueue_router_rows"; + let k = weight_k(what, w)?; + check_bias(what, bias)?; + let tt = launch_u32(what, "t", t)?; + let input = k.checked_mul(t).ok_or_else(|| GpuError::shape(what, "input size overflow"))?; + let scores = N_EXPERT.checked_mul(t).ok_or_else(|| GpuError::shape(what, "score size overflow"))?; + let slots = N_USED.checked_mul(t).ok_or_else(|| GpuError::shape(what, "slot size overflow"))?; + if t == 0 || x.len() < input || probs.len() < scores || ids.len() < slots || weights.len() < slots { + return Err(GpuError::shape( + what, + format!("{t} tokens of {k} (at least one): x {}, probs {}, ids {}, weights {}", + x.len(), probs.len(), ids.len(), weights.len()), + )); + } + // The tile GEMM uses the one-token row walk and warp sum tree. + wide.enqueue_f32_tile(stream, w, x, t, probs)?; + gpu.elem().enqueue_sigmoid_rows(stream, probs, scores)?; + let prep = self.module.prepare_sigmoid_floor_256_6_router_pick( + LaunchConfig1D::new(tt, ROUTER_THREADS_U32, 0), + )?; + self.module.sigmoid_floor_256_6_router_pick( + stream, &prep, probs, bias.unwrap_or(w.buf()), u32::from(bias.is_some()), + tt, scale, ids, weights, fault, + )?; + Ok(()) + } + } + + impl RouterEntry for RouterKernels { + const N_EXPERT: usize = N_EXPERT; + const N_USED: usize = N_USED; + const OPTIONAL_BIAS: bool = true; + type Out = RouterOut; + + fn load(ctx: &Arc) -> Result { + Self::load(ctx) + } + fn new_out(stream: &CudaStream) -> Result { + RouterOut::new(stream) + } + fn out_ids(out: &RouterOut) -> &DeviceBuffer { + &out.ids + } + fn out_weights(out: &RouterOut) -> &DeviceBuffer { + &out.weights + } + fn out_bytes(out: &RouterOut) -> usize { + out.logits.num_bytes() + out.probs.num_bytes() + out.ids.num_bytes() + + out.weights.num_bytes() + out.done.num_bytes() + } + fn route( + &self, + stream: &CudaStream, + x: &DeviceBuffer, + r: &RouteIn<'_>, + out: &mut RouterOut, + ) -> Result<(), GpuError> { + self.enqueue_router(stream, r.w, x, r.bias, r.scale, out, r.fault) + } + fn route_rows( + &self, + (stream, gpu, wide): (&CudaStream, &Gpu, &Gemm32Kernels), + x: &DeviceBuffer, + r: &RouteIn<'_>, + t: usize, + rows: RoutedRows<'_>, + ) -> Result<(), GpuError> { + self.enqueue_router_rows( + stream, gpu, wide, r.w, x, r.bias, r.scale, t, + rows.probs, rows.ids, rows.weights, r.fault, + ) + } + } +} diff --git a/crates/gpu/src/route_core.rs b/crates/gpu/src/route_core.rs index 4c00bfe9b..b32595b86 100644 --- a/crates/gpu/src/route_core.rs +++ b/crates/gpu/src/route_core.rs @@ -18,9 +18,9 @@ //! [`Sigmoid`]). What a router stores is [`router_score`] of the logit, never //! the score alone. //! -//! A weight stage that renormalizes its selected scores divides by -//! [`renorm_divisor`] of their sum: the one guard every such stage and its -//! host simulation share. +//! A weight stage uses either [`renorm_divisor`] (the legacy epsilon-added +//! policy) or [`floor_divisor`] (Flint's fixed selected-sum floor). The +//! compiled instance chooses the policy, never a per-token launch flag. use crate::elem::argmax_take; @@ -58,6 +58,12 @@ pub fn renorm_divisor(sum: f32) -> f32 { sum + 1e-20 } +/// Flint's selected-score divisor: the f32 sum floored at exactly `2^-14`. +#[inline(always)] +pub fn floor_divisor(sum: f32) -> f32 { + sum.max(1.0 / 16384.0) +} + /// The score a router stores for `logit`: `S::score(logit)`, or NaN for a /// logit that is not finite. A logit overflows to ±infinity when a row's /// finite products sum past f32, and the scores give that a finite value diff --git a/crates/gpu/src/routed_block.rs b/crates/gpu/src/routed_block.rs index b965c0b79..7ad972410 100644 --- a/crates/gpu/src/routed_block.rs +++ b/crates/gpu/src/routed_block.rs @@ -48,11 +48,11 @@ use crate::weights::Weights; use crate::{DeviceTensor, FaultSink, Gpu, GpuError}; /// What a router launch reads besides its activations: the layer's router -/// weight and selection bias, the file's `expert_weights_scale` and the -/// layer's fault sink. +/// weight and optional selection bias, the file's `expert_weights_scale` +/// and the layer's fault sink. pub struct RouteIn<'a> { pub w: &'a DeviceTensor, - pub bias: &'a DeviceBuffer, + pub bias: Option<&'a DeviceBuffer>, pub scale: f32, pub fault: FaultSink, } @@ -72,6 +72,8 @@ pub trait RouterEntry: Sized { const N_EXPERT: usize; /// Experts each token routes to. const N_USED: usize; + /// Whether an absent selection bias is a native binding, not a zero buffer. + const OPTIONAL_BIAS: bool = false; /// Where one token's launch leaves its results. type Out; @@ -111,7 +113,7 @@ pub struct RouteLayer { pub norm: String, /// The router weight. pub router: String, - /// The selection bias; `None` routes with a zero bias. + /// The selection bias; `None` routes without a correction bias. pub bias: Option, /// The layer's row in the slot map's card copy, as a word offset /// (`SlotMap::row_offset`). @@ -141,7 +143,7 @@ pub struct RouteBlock { handoff: HandoffKernels, /// The wide F32 tile GEMM the rows router's scores run on. g32: Gemm32Kernels, - /// The zero selection bias of a layer the file gives none. + /// Legacy bindings' zero selection bias for a layer the file gives none. no_bias: Option>, } @@ -270,7 +272,7 @@ impl RouteBlock { Some(name) => { w.f32_buf(what, name)?; } - None => zero_bias = true, + None => zero_bias |= !R::OPTIONAL_BIAS, } } let ctx = gpu.context(); @@ -335,7 +337,11 @@ impl RouteBlock { gpu: &Gpu, ) -> Result, GpuError> { let layer = self.layer(l)?; - let bias = self.bias_of(w, l)?; + let bias = if layer.bias.is_none() && R::OPTIONAL_BIAS { + None + } else { + Some(self.bias_of(w, l)?) + }; Ok(RouteIn { w: w.f32_tensor(self.what, &layer.router)?, bias, diff --git a/crates/model/src/arch/flint/tests.rs b/crates/model/src/arch/flint/tests.rs index ac2bca5dc..698806d85 100644 --- a/crates/model/src/arch/flint/tests.rs +++ b/crates/model/src/arch/flint/tests.rs @@ -283,10 +283,15 @@ fn normalization_policy_is_architecture_specific_and_bias_is_selection_only() { let shape = models::shape::MoeShape::of(m); assert_eq!(shape.rule.norm, expected); assert_eq!(shape.rule.bias, bias); - assert_eq!( - models::shape::select_router(shape).unwrap_err().why, - models::shape::Refusal::NoRouterRule - ); + let refusal = models::shape::select_router(shape).unwrap_err().why; + if norm { + assert!(matches!( + refusal, + models::shape::Refusal::RouterWidth { ref served } if served == &[256] + )); + } else { + assert_eq!(refusal, models::shape::Refusal::NoRouterRule); + } assert!( uncovered .iter() diff --git a/crates/model/tests/flint_meta.rs b/crates/model/tests/flint_meta.rs index d6121431c..bde81addf 100644 --- a/crates/model/tests/flint_meta.rs +++ b/crates/model/tests/flint_meta.rs @@ -109,12 +109,10 @@ fn hw_flint_header() { let coverage = model::arch::coverage::check(s, &r.tensors); for (need, layer) in &needs { if let Need::Router(shape) = need { - assert_eq!( - model::arch::models::shape::select_router(*shape) - .unwrap_err() - .why, - model::arch::models::shape::Refusal::NoRouterRule - ); + let binding = model::arch::models::shape::select_router(*shape).unwrap(); + assert_eq!(binding.body, model::arch::models::shape::RouterBody::SigmoidFloor); + assert_eq!(binding.rule, shape.rule); + assert_eq!((binding.experts(), binding.top_k_min, binding.top_k_max), (256, 6, 6)); assert!(coverage.iter().any(|u| { u.layer == layer.map(|l| l as usize) && u.feature == need.to_string() })); diff --git a/crates/models/src/shape.rs b/crates/models/src/shape.rs index b70773254..6fd7f1592 100644 --- a/crates/models/src/shape.rs +++ b/crates/models/src/shape.rs @@ -83,6 +83,9 @@ pub enum RouterBody { /// `mimo2_router*`, `sigmoid_320_8_router*`, `sigmoid_128_4_router*`; /// `gpu/src/arch/nemotron_h_moe/router.rs` `nemotron_moe_router_*`. BiasedSigmoid, + /// `gpu/src/route_body.rs` `sigmoid_floor_256_6_router*`: sigmoid, + /// optional selection-only bias and a fixed `2^-14` selected-sum floor. + SigmoidFloor, } /// One compiled router instance. @@ -160,6 +163,16 @@ const BIASED_SIGMOID: RouterRule = RouterRule { norm: RouterNorm::SelectedSum, gated: false, }; +const SIGMOID_FLOOR: RouterRule = RouterRule { + score: Score::Sigmoid, + bias: false, + norm: RouterNorm::SelectedSumFloor, + gated: false, +}; +const BIASED_SIGMOID_FLOOR: RouterRule = RouterRule { + bias: true, + ..SIGMOID_FLOOR +}; const BIASED_SQRT_SOFTPLUS: RouterRule = RouterRule { score: Score::SqrtSoftplus, bias: true, @@ -326,6 +339,24 @@ pub const ROUTERS: &[RouterInst] = &[ top_k_max: 6, at: "gpu/src/route_body.rs sigmoid_256_6_router*", }, + RouterInst { + body: RouterBody::SigmoidFloor, + rule: SIGMOID_FLOOR, + norm_arg: false, + experts: 256, + top_k_min: 6, + top_k_max: 6, + at: "gpu/src/route_body.rs sigmoid_floor_256_6_router*", + }, + RouterInst { + body: RouterBody::SigmoidFloor, + rule: BIASED_SIGMOID_FLOOR, + norm_arg: false, + experts: 256, + top_k_min: 6, + top_k_max: 6, + at: "gpu/src/route_body.rs sigmoid_floor_256_6_router*", + }, // Muse Spark's fixture: 16 experts, the real file's top 8. RouterInst { body: RouterBody::BiasedSigmoid, @@ -531,6 +562,31 @@ pub const fn router_row_k(body: RouterBody, experts: u32, top_k: u32) -> RouterI panic!("no router row for this body at this width and pick count") } +/// A compiled binding whose body, rule, expert count and pick count all +/// agree. Unlike [`router_row_k`], distinguishes optional-bias bindings. +#[must_use] +pub const fn router_row_rule( + body: RouterBody, + rule: RouterRule, + experts: u32, + top_k: u32, +) -> RouterInst { + let mut i = 0; + while i < ROUTERS.len() { + let r = ROUTERS[i]; + if body_eq(r.body, body) + && r.runs(rule) + && r.experts == experts + && r.top_k_min <= top_k + && top_k <= r.top_k_max + { + return r; + } + i += 1; + } + panic!("no router row for this body, rule, width and pick count") +} + /// Whether row `r` is body `body` at `per_lane`, with rule `rule` and the /// pick range `min..=max`: the whole of what a device crate compiled in. #[must_use] @@ -564,6 +620,11 @@ pub mod rules { /// Sigmoid scores, a selection bias steering the choice only, the kept /// unbiased scores renormalized: GLM-5.3's and MiMo-V2's noaux_tc router. pub const BIASED_SIGMOID: RouterRule = super::BIASED_SIGMOID; + /// Sigmoid, no selection bias, selected scores divided by + /// `max(sum_selected, 2^-14)`. + pub const SIGMOID_FLOOR: RouterRule = super::SIGMOID_FLOOR; + /// [`SIGMOID_FLOOR`] with a bias steering selection only. + pub const BIASED_SIGMOID_FLOOR: RouterRule = super::BIASED_SIGMOID_FLOOR; } const fn norm_eq(a: RouterNorm, b: RouterNorm) -> bool { @@ -937,66 +998,62 @@ mod tests { assert!(matches!(e.why, Refusal::RouterWidth { ref served } if served == &[128])); } + /// Legacy rows retain their epsilon-added or unnormalized arithmetic; + /// only the real floor binding accepts the floor policy. #[test] - fn a_selected_sum_floor_is_not_a_legacy_device_policy() { + fn floor_bindings_are_distinct_from_legacy_policies() { for row in ROUTERS { - let rule = RouterRule { - norm: RouterNorm::SelectedSumFloor, - ..row.rule - }; assert_eq!( select_router(moe(row.rule, row.experts(), row.top_k_min)).unwrap(), *row ); - assert!(!row.runs(rule)); - assert!(!router_row_is( - *row, - rule, - row.norm_arg, - (row.top_k_min, row.top_k_max) - )); - let shape = moe(rule, row.experts(), row.top_k_min); - let refusal = select_router(shape).expect_err("a legacy body bound the floor policy"); - assert_eq!(refusal.shape, Shape::Moe(shape)); - assert_eq!(refusal.why, Refusal::NoRouterRule); + if row.body == RouterBody::SigmoidFloor { + assert_eq!(row.rule.norm, RouterNorm::SelectedSumFloor); + assert!(!row.norm_arg); + } else { + assert!(!row.runs(RouterRule { + norm: RouterNorm::SelectedSumFloor, + ..row.rule + })); + } } } - /// Flint's sigmoid top-six is not the biased top-eight binding, and no - /// compiled body runs its rule: the 256-expert top-6 instance runs the - /// biased sigmoid with the epsilon-added `SelectedSum`, so every other - /// pair of bias and policy, Flint's `SelectedSumFloor` with or without a - /// selection bias among them, is refused by name. - // PIN(2026-10-25): the biased `SelectedSum` pair was refused as - // `TopK { min: 8, max: 8 }` while only MiMo's top-8 row held 256 experts; - // the `sigmoid_256_6` instance's row now serves exactly that pair - // (`sigmoid_256_6_router_is_its_row_beside_mimos_top_8` pins its identity). - // The other five pairs keep `NoRouterRule`. #[test] - fn flints_sigmoid_top_six_is_not_the_biased_top_eight_binding() { + fn floor_top_six_serves_both_bias_modes_and_only_its_shape() { for bias in [false, true] { - for norm in [ - RouterNorm::None, - RouterNorm::SelectedSum, - RouterNorm::SelectedSumFloor, - ] { - let rule = RouterRule { - bias, - norm, - ..rules::BIASED_SIGMOID - }; - let selected = select_router(moe(rule, 256, 6)); - if bias && norm == RouterNorm::SelectedSum { - assert!( - selected.is_ok(), - "the 256/6 instance's own rule: {selected:?}" - ); - continue; - } - let refusal = selected.expect_err("a sigmoid top-six binding was fabricated"); - assert_eq!(refusal.why, Refusal::NoRouterRule); + let rule = RouterRule { bias, ..rules::SIGMOID_FLOOR }; + let row = select_router(moe(rule, 256, 6)).unwrap(); + assert_eq!(row.body, RouterBody::SigmoidFloor); + assert_eq!(row.rule, rule); + assert_eq!(row, router_row_rule(RouterBody::SigmoidFloor, rule, 256, 6)); + for top_k in [0, 5, 7, 8, 257] { + assert_eq!( + select_router(moe(rule, 256, top_k)).unwrap_err().why, + Refusal::TopK { min: 6, max: 6 } + ); + } + assert!(matches!( + select_router(moe(rule, 128, 6)).unwrap_err().why, + Refusal::RouterWidth { ref served } if served == &[256] + )); + for (score, gated) in [(Score::Softmax, false), (Score::Sigmoid, true)] { + let wrong = RouterRule { score, gated, ..rule }; + assert_eq!( + select_router(moe(wrong, 256, 6)).unwrap_err().why, + Refusal::NoRouterRule + ); } } + let legacy = select_router(moe(rules::BIASED_SIGMOID, 256, 6)).unwrap(); + assert_eq!(legacy.body, RouterBody::BiasedSigmoid); + assert_eq!( + select_router(moe(RouterRule { + norm: RouterNorm::SelectedSum, + ..rules::SIGMOID_FLOOR + }, 256, 6)).unwrap_err().why, + Refusal::NoRouterRule + ); } /// GLM-5.3-Flash (model/tests/glm5next_meta.rs:83, "router: sigmoid, 288 diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index b0a9b8b4e..5eb89f92d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,9 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. The current tree additionally has a **header-only reader** and a real common +256/top6 floor-router binding; it still has no Flint body, app session or serve seat. +No GPU, inference, performance or weight-content validation is claimed. ## Host reader status @@ -15,10 +16,10 @@ GemmaEmbedding2's sliding mask: missing keys, scalars, wrong lengths and non-boo Present `expert_weights_scale` uses `arch::finite_positive`; only an absent key defaults to 1, with that default recorded. The common needs walk emits q/k normalization even on NoPE layers. `program_of(Flint)` remains -`None`; missing program, head128 sinks/SWA, NoPE normalization and 256/top6 bindings remain coverage -refusals. `Router.norm` and `RouterRule.norm` distinguish no normalization, the existing selected-sum -rule and Flint's selected-sum floor; the floor description does not supply a device implementation -or activated-value scaling. +`None`; missing program, head128 sinks/SWA and NoPE normalization remain coverage refusals. +`Router.norm` and `RouterRule.norm` distinguish no normalization, the existing selected-sum +rule and Flint's selected-sum floor. The real floor-router binding is selectable in both bias +modes; selecting that operation does not supply a whole program or activated-value scaling. Fault formatting uses `bloomery_device::fault`'s shared layout/decoder and the `gpu::arch::step_order` registry. Flint's registry row uses neutral code order; @@ -50,19 +51,41 @@ No full weights or GPU were needed for that check. The maintainer [assigned the model port and offered card runs](https://github.com/midagedev/bloomery/issues/632#issuecomment-6352066886), and named a forthcoming head128 sink/SWA attention, QK RMS and sigmoid256/top6 train. -Those complete contracts are not in this main snapshot or the inspected public heads of #726 (`1723485602c1`) -and #747 (`f883e84f8d2c`). Existing ring and sink-merge ingredients do not establish the combined attention -operation; `route_body` still lacks 256/top6 and its normalization-floor policy. -Obtain the train's fetchable ref before duplicating those primitives. - -The router description/selection contract now distinguishes `RouterNorm::None`, `SelectedSum` and -`SelectedSumFloor`. Flint requests `max(sum_selected, 2^-14)`; existing model descriptions and arithmetic -keep their rules. Compiled rows, including boolean `norm_arg` bindings, refuse the floor policy rather -than borrow an epsilon-added implementation. The absent-bias, 256/top6 and `Need::Program` refusals stay -until real device bindings are supplied. Reconcile the policy interface with the pending train before device work. -Then close shared IQ4_XS, selected card/tier down forms and six-slot F32 handoff in their existing common owners, +Those complete attention contracts are not in this main snapshot or the inspected public heads of +#726 (`1723485602c1`) and #747 (`f883e84f8d2c`). Existing ring and sink-merge ingredients do not +establish the combined attention operation. Obtain the train's fetchable ref before duplicating +those primitives. + +The common `route_body::sigmoid_floor_256_6` binding now supplies Flint's fixed +`max(sum_selected, 2^-14)` rule. `RouterKernels::enqueue_router` and `enqueue_router_rows` +take `Option<&DeviceBuffer>` for a real optional selection-only bias. The common +`RouterEntry`/`RouteIn` consuming paths carry that option; `RouteBlock` allocates no zero-bias +buffer for this binding. Absent bias borrows the existing router-weight slice only as an unused +device ABI argument; no bias data is fabricated, copied or read. The floor policy is a compile-time +specialization of the same score, publication and top-k bodies, not a per-step normalization flag. + +Both metadata bindings are `RouterBody::SigmoidFloor` at exactly 256/top6: sigmoid, no shared +sigmoid gate, `SelectedSumFloor`, with or without selection bias. Bias changes only the ranking. +The six selected unbiased scores are summed in f64 slot order, narrowed to f32, divided by the +floored sum and then scaled. Equal selection keys choose larger expert ids. Nonfinite logits, +selection keys or resulting weights raise `FaultSite::Router`; an invalid selection key poisons +all six weights. Existing epsilon-added entries, launch ABIs and legacy norm-argument rules remain +unchanged. A legacy `RouterEntry` given no required bias returns a named state error. + +`gate-flint-router` now consumes the actual floor binding through `RouterEntry` for planted and +dense rows, both bias modes, active tiny-sum floors, zero sums, ties, nonfinite logits/bias/scale, +one-token/batch bit equivalence and eager/captured replay. Its independent host weight expression +uses the declared floor and rejects an epsilon-added or biased-weight implementation. +**needs-gpu:** this implementation has no local NVIDIA numerical, capture, fault, allocation or +JIT-resource evidence. The two new entries (`sigmoid_floor_256_6_router` and +`sigmoid_floor_256_6_router_pick`) have no invented resource pins; the add-class PTX comparison, +spill/JIT pinning and hardware gate remain required. + +The `Need::Program` refusal stays: a callable router is not a Flint body or session. Close shared +IQ4_XS, selected card/tier down forms and six-slot F32 handoff in their existing common owners, followed by paging/resident slots, reference provenance and the body/app/seat. -Each device slice needs its numerical/fault gates and the maintainer's offered hardware proof before admission. +Each device slice needs its numerical/fault gates and the maintainer's offered hardware proof before +whole-program admission. Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. @@ -230,13 +253,12 @@ source of the gathered weights. The shared SwiGLU output is added without a shar Three numerical obligations must stay explicit: -1. Current [common router normalization][route-core] uses `sum + 1e-20`, not the reference's floor. +1. Legacy [common router normalization][route-core] uses `sum + 1e-20`, not the reference's floor. This is a semantic difference for tiny scores, not a width-instance problem or a measured GPU - failure. The existing `Router`/`RouterRule` contract now carries the explicit policy, and the - reader requests `SelectedSumFloor` for Flint. Selection refuses that policy on every current - router row; `norm_arg` serves only the two legacy policies. The device floor binding is still - missing. Add it in the common operation without changing the global divisor, existing model - arithmetic, or hiding the difference in a Flint-only wrapper. + failure. `Router`/`RouterRule` carries the explicit policy; the reader requests + `SelectedSumFloor` and the common `sigmoid_floor_256_6` binding implements it. Legacy instances + keep their exact arithmetic and `norm_arg` serves only the two legacy policies. The floor + gate checks active tiny sums independently; GPU numerical and resource evidence is still needed. 2. Mainline [rescales Flint's activated expert input before down projection][ref-down] with `build_amax_src1_f16`, then restores its scale before router weighting. This protects F16/Q8 headroom around a maximum magnitude of 1024; it is not an architectural activation clamp. @@ -272,7 +294,7 @@ Do not claim runtime support by adding the name to an enum or relaxing a shape/t |---|---|---| | G1 | NoPE normalization-need generation is present; the executable head128 norm-only binding is missing | Preserve the emitted normalization **or** rotation need; bind head128 normalization with `mode=None` truthfully. No identity-RoPE table. | | G2 | Existing head128/group8 attention lacks the complete sink/SWA contract; MiMo's window pass is K192/V128 | Complete the described head128 operation in `rope_neox` / `flash_gqa`, including genuine NoPE append, sink merge, window/ring positions, prompt and slot paths. Never pad heads to 192 or copy MiMo's body. | -| G3 | The host description/selection policy is explicit; sigmoid256/top6 and its device floor binding are absent, and higher-level `shared_routed` admission requires bias | Add the real instance/floor operation; bind absent bias through `RouteLayer.bias=None`, which the lower `RouteBlock` already supports. Price/load the BF16 router through its supported F32 widening. | +| G3 | Real common sigmoid256/top6 floor operation and both optional-bias metadata bindings are implemented; whole-program binding remains separate | Attach `sigmoid_floor_256_6` through `RouterEntry`, with `RouteLayer.bias=None` retaining genuine absence and no zero buffer. Price/load the BF16 router through its supported F32 widening. Run `gate-flint-router` and obtain genuine new-entry JIT pins on NVIDIA hardware. | | G4 | Layer42 shared IQ4_XS has routed/GEMM operations but no ordinary dense site, and `BlockPlan` rejects it | Extend the **common** dense/gated block's real layout, load, launch, scratch and prompt binding. Keep the role `SharedExpert`; do not relabel it as routed or silently requantize it. | | G5 | The common card wrapper uses Q8Act-only buffers and `LegStacks::of`; Q6_K/Q8_0 down do not both pass that path | Carry selected arms and actual down forms through `CardExperts`, `LegBufs::for_downs`, loader, planner, residency and tier. Existing Q6_K/Q8_0 arithmetic is not the missing piece. | | G6 | Common tier accumulation supports six slots, but F32 `enqueue_handoff_tier` accepts eight | Add the six-slot F32 handoff instance in the existing owner; preserve the eight-slot entry. A DeepSeek Q8_1 handoff is not an interchangeable input contract. | From 781370ecd334bfb8ae0a1097342252b486f4000e Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 07:51:45 +0900 Subject: [PATCH 33/85] gpu: complete current-main selected tier prerequisite cutover --- crates/gpu-gates/src/bin/gate_mimo2_serve.rs | 5 +++-- .../src/bin/gate_musespark_prefill.rs | 2 +- .../gpu-gates/src/bin/shared/mimo2_twocard.rs | 2 +- .../gpu-gates/src/bin/shared/minimaxm3_seat.rs | 4 ++-- .../gpu-gates/src/bin/shared/qwen3moe_place.rs | 4 ++-- .../src/bin/shared/serve_seats/mimo2.rs | 6 ++++-- .../src/bin/shared/serve_seats/minimaxm3.rs | 9 +++++++-- .../src/bin/shared/serve_seats/musespark.rs | 9 +++++++-- crates/gpu-mimo2/src/tier.rs | 5 +++-- crates/gpu-minimaxm3/src/tier.rs | 5 +++-- crates/gpu/src/host/handoff.rs | 18 +++++++++--------- crates/model/src/arch/mimo2/place.rs | 6 ++++-- crates/model/src/arch/minimaxm3/place.rs | 4 ++-- crates/model/src/arch/musespark/place.rs | 1 + crates/model/tests/mimo2_meta.rs | 2 +- crates/model/tests/musespark_meta.rs | 2 +- crates/model/tests/qwen3_bp_plan.rs | 8 ++++++-- crates/placement/src/placement.rs | 5 +++-- crates/placement/src/placement/card_budget.rs | 1 + docs/research/flint-attach.md | 8 +++++++- 20 files changed, 68 insertions(+), 38 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs index 2245c05d3..da2c6271b 100644 --- a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs +++ b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs @@ -892,8 +892,9 @@ mod gate { let mut inputs = PlanInputs::describe(&split)?; inputs.room = (PLAN_ROOM, HostRead::Given); let place = unset_place(cards).on_host()?; - let batch = - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)); + let batch = (!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)) + .transpose()?; let machine = place.machine(None, batch)?(inputs.model.layers); let slots = NonZeroUsize::new(usize::try_from(DEFAULT_SLOTS)?).ok_or("no default slot")?; let plan = diff --git a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs index 6674f694b..5137e5828 100644 --- a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs @@ -530,7 +530,7 @@ mod gate { tier_now: Tier, ) -> Result<(Model, bool, Vec), GateError> { let inputs = PlanInputs::read(&open_split(Arch::MuseSpark, RECIPE)?)?; - let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp))?; + let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp)?)?; place::stage_placed(&mut two)?; let ctx = u64::try_from(ctx)?; // The stage card alone, the reference's machine: the budget reads its unbudgeted plan, which a plan of the two diff --git a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs index 3b8425314..cbf918642 100644 --- a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs +++ b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs @@ -111,7 +111,7 @@ pub fn run(path: &Path, levers: &bloomery_levers::Levers) -> Result Result workstation::placed_with_arena(*stage, layers, arena), [stage, tier] => { let batch = match inputs { - Inputs::Qwen3(i) => q3::tier_batch(&i.hp), - Inputs::Qwen35(i) => q35::leg_tier_batch(&i.hp), + Inputs::Qwen3(i) => q3::tier_batch(&i.hp)?, + Inputs::Qwen35(i) => q35::leg_tier_batch(&i.hp)?, }; let mut machine = workstation::plan_tiers(layers, *stage, &[*tier], None, batch)?; q3::stage_placed(&mut machine, arena); diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs index 019fb66a1..1204c8373 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs @@ -501,8 +501,10 @@ impl Tiered for Mimo { /// The tier card's prompt-batch bytes ([`bloomery_gpu_mimo2::tier::tier_batch`]), under a /// placement with one. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs)) + fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs)) + .transpose()?) } fn flags(flag: &str, value: &str, out: &mut Flags) -> Result { diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs index 6e06cb60b..92f704bf1 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs @@ -501,8 +501,13 @@ impl Tiered for MiniMax { /// The tier's prompt-batch bytes of the file's widths and down forms over the body's prompt /// unit ([`bloomery_gpu_minimaxm3::tier::tier_batch`]), reserved on the tier card and the /// host; `None` on a placement with no tier card. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs)) + .transpose()?) } /// The tags the model's description names for its think span. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs index 267a9019a..650f28a66 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs @@ -458,8 +458,13 @@ impl Tiered for Muse { /// The tier's prompt-batch bytes of the file's widths /// ([`place::tier_batch`]), reserved on the tier card and the host; `None` /// on a placement with no tier card. - fn tier_batch(place: Place, inputs: &PlanInputs) -> Option { - (!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp)) + fn tier_batch( + place: Place, + inputs: &PlanInputs, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(&inputs.hp)) + .transpose()?) } /// The ring is kept out of the expert budget where the common rule starts diff --git a/crates/gpu-mimo2/src/tier.rs b/crates/gpu-mimo2/src/tier.rs index 748c7889e..4a431ba3f 100644 --- a/crates/gpu-mimo2/src/tier.rs +++ b/crates/gpu-mimo2/src/tier.rs @@ -34,7 +34,7 @@ use gguf::Split; use model::ModelError; use model::arch::mimo2::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; +use model::placement::{ModelTensors, PlacementError}; use model::placement::workstation::TierBatchBytes; use model::r8file::R8Source; use models::LayerSpec; @@ -57,7 +57,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. #[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(&inputs.hp, T_MAX as u64) } @@ -91,6 +91,7 @@ impl LegFamily for Mimo2 { n_used: hp.n_used, n_embd: hp.n_embd, ff: hp.expert_ff, + program: model::placement::accepts::Program::Mimo2Body, } } diff --git a/crates/gpu-minimaxm3/src/tier.rs b/crates/gpu-minimaxm3/src/tier.rs index c591db72c..b937c9291 100644 --- a/crates/gpu-minimaxm3/src/tier.rs +++ b/crates/gpu-minimaxm3/src/tier.rs @@ -20,7 +20,7 @@ use gguf::Split; use model::arch::minimaxm3::host as minimax_host; use model::arch::minimaxm3::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; +use model::placement::{ModelTensors, PlacementError}; use model::placement::workstation::TierBatchBytes; use model::r8file::R8Source; use models::{LayerSpec, Moe}; @@ -39,7 +39,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. #[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(inputs, T_MAX as u64) } @@ -73,6 +73,7 @@ impl LegFamily for MiniMax { n_used: experts.n_used, n_embd: inputs.hp.n_embd, ff: experts.ff, + program: model::placement::accepts::Program::MiniMaxM3Body, } } diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index e42f46a61..51df65d8d 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -18,9 +18,9 @@ //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a //! row, for a walk whose routing reaches the host by a download, through -//! `crate::card_slots`' `ds41_ffn_places`. `ffn_handoff_6_tier` and -//! `ds41_ffn_handoff_8_tier` hand over six or eight slots of a layer an -//! expert tier holds experts of: the same writes, the slots' tier places and the activation in f32 into +//! `crate::card_slots`' `ds41_ffn_places`. `ds41_ffn_handoff_4_tier`, +//! `ffn_handoff_6_tier` and `ds41_ffn_handoff_8_tier` hand over four, six or +//! eight slots of a tier layer: the same writes, its tier places and f32 activation into //! the row's tier image ([`HandoffKernels::enqueue_handoff_tier`]); //! `ds41_ffn_handoff_10_cols_tier` is `_10_cols`'s of such a layer, every //! column's tier places and activation into the row's tier image @@ -1346,8 +1346,8 @@ impl HandoffKernels { } impl HandoffKernels { - /// Enqueue the handoff of a tier layer of six or eight slots a token - /// (`ffn_handoff_6_tier` or `ds41_ffn_handoff_8_tier`): + /// Enqueue the handoff of a tier layer of four, six or eight slots a token + /// (`ds41_ffn_handoff_4_tier`, `ffn_handoff_6_tier`, `ds41_ffn_handoff_8_tier`): /// [`HandoffKernels::enqueue_handoff`]'s /// writes into `target`, and each slot's tier place (from `tmap`, the /// slot map's tier copy at the card copy's row offsets) into `tsel` and @@ -1640,7 +1640,7 @@ fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result u64 { hp.expert_ff as u64, hp.n_used as u64, cap as u64, + DownForms::of(DownAct::Q8Act), ) as usize; leg + 2 * size_of::() * cap * n + size_of::() * cap * hp.n_used } else { @@ -268,7 +270,7 @@ mod tests { let _ = std::fs::remove_file(&path); let hp = &inputs.hp; assert_eq!( - tier_batch(hp, COLS), + tier_batch(hp, COLS).expect("tier bytes"), workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, @@ -276,7 +278,7 @@ mod tests { TIER_CHUNK, COLS, DownForms::of(DownAct::Q8Act), - ) + ).expect("tier bytes") ); let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS).expect("tier bytes")); diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 64b6a8407..9ee262b4f 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -435,7 +435,7 @@ mod tests { let inputs = described("mm3-place-tier"); let hp = &inputs.hp; assert_eq!( - tier_batch(&inputs, COLS), + tier_batch(&inputs, COLS).expect("tier bytes"), workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, @@ -443,7 +443,7 @@ mod tests { TIER_CHUNK, COLS, DownForms::of(DownAct::Q8Act), - ) + ).expect("tier bytes") ); let tiered = workstation::plan_bp( LAYERS, None, tier_batch(&inputs, COLS).expect("tier bytes"), diff --git a/crates/model/src/arch/musespark/place.rs b/crates/model/src/arch/musespark/place.rs index de91cf878..273ce4c2b 100644 --- a/crates/model/src/arch/musespark/place.rs +++ b/crates/model/src/arch/musespark/place.rs @@ -629,6 +629,7 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), + tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, diff --git a/crates/model/tests/mimo2_meta.rs b/crates/model/tests/mimo2_meta.rs index 64ecda04a..db975961b 100644 --- a/crates/model/tests/mimo2_meta.rs +++ b/crates/model/tests/mimo2_meta.rs @@ -401,7 +401,7 @@ fn hw_mimo2_plan() { let tiered = workstation::plan_bp( layers, None, - model::arch::mimo2::place::tier_batch(&inputs.hp, 512), + model::arch::mimo2::place::tier_batch(&inputs.hp, 512).expect("tier batch bytes"), ); let bp = inputs .plan(&tiered, ctx, &PlanLevers::default()) diff --git a/crates/model/tests/musespark_meta.rs b/crates/model/tests/musespark_meta.rs index d3cabdd9c..e765e3d56 100644 --- a/crates/model/tests/musespark_meta.rs +++ b/crates/model/tests/musespark_meta.rs @@ -368,7 +368,7 @@ fn hw_musespark_plan() { workstation::A6000, &[workstation::RTX_3090], None, - place::tier_batch(&hp), + place::tier_batch(&hp).expect("tier batch bytes"), ) .unwrap_or_else(|e| panic!("the tiered machine: {e}")); place::stage_placed(&mut m).unwrap_or_else(|e| panic!("the tiered machine's stage: {e}")); diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index dc4fb39a6..972d82125 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -528,7 +528,9 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); - let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.hp)); + let machine = tiered( + Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.hp).expect("tier batch bytes"), + ); let plan = inputs.plan_rule( &machine, CTX, @@ -543,7 +545,9 @@ fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place #[test] fn qwen3_30b_q8_bp_tier_experts() { let inputs = qwen3_inputs(); - let machine = tiered(Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp)); + let machine = tiered( + Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp).expect("tier batch bytes"), + ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) .expect("the A6000 stage plans a tier"); diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 25ef5133d..9599ba578 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -2897,7 +2897,8 @@ fn resolve_shares( let uploads = card_uploads(model, c, rows, &own, routed)?; let dense = Footprint::new(card.granule_bytes, &uploads).at(&zeros)?; let kv = kv_card.saturating_add(stage_rows); - let floor = plan_floor(card, dense, kv, reserve); + let reserves = card.selected_reserve_bytes(model, &zeros)?; + let floor = plan_floor(card, dense, kv, reserve, reserves); let cap = capped(card, None); let Some((floor, room)) = floor.and_then(|f| Some((f, cap.checked_sub(f)?))) else { return Err(refuse(format!( @@ -2909,7 +2910,7 @@ fn resolve_shares( kv, card.context_bytes, card.scratch_bytes, - card.reserve_bytes().unwrap_or(u64::MAX), + reserves, card.margin_bytes, ))); }; diff --git a/crates/placement/src/placement/card_budget.rs b/crates/placement/src/placement/card_budget.rs index 3f8d2e734..52830b93c 100644 --- a/crates/placement/src/placement/card_budget.rs +++ b/crates/placement/src/placement/card_budget.rs @@ -209,6 +209,7 @@ mod tests { fn card(name: &str, ordinal: Option) -> Card { Card { name: name.to_owned(), + tier_leg: None, device: ordinal.map(|o| DeviceId { ordinal: o, uuid: [u8::try_from(o).expect("small") + 1; 16], diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 5eb89f92d..64ad6ee9b 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -128,6 +128,12 @@ Host-only projections intentionally have no card totals: fixed reserves remain o Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. +The current-main cutover keeps reserve-only draft holders, per-card budgets and each +family's activation contract. `LegTier` receives its activation converter at load; +selected `Program` arms replace the redundant family arm hook. Qwen, Muse, MiMo and +MiniMax sizing callers propagate checked tier descriptors rather than bypassing +their validation. Share-budget floors use the same selected reserve calculation. + A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an @@ -135,7 +141,7 @@ exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain abse ### Six-slot F32 tier handoff implementation `HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry -beside the existing eight-slot entry. Both host and tier layouts must have the +beside the existing four- and eight-slot entries. Both host and tier layouts must have the same supported slot count and activation width; compressed tier images remain refused. The entry uses the existing copy/place/fault helpers and the existing subsequent go/publication protocol; no Flint router or Program is registered. From c443e8d5249c0f9891388cc197365604b2929684 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:17:57 +0900 Subject: [PATCH 34/85] feat(flint): attach floor router, ring attention and hosted runtime --- Cargo.lock | 16 + Cargo.toml | 2 +- crates/app/Cargo.toml | 2 + crates/app/src/arch/flint.rs | 76 ++ crates/app/src/arch/minimaxm3/mod.rs | 8 +- crates/app/src/arch/mod.rs | 106 ++- crates/app/src/arch/musespark/mod.rs | 4 +- crates/app/src/arch/solar2/mod.rs | 4 +- crates/app/src/lib.rs | 9 +- crates/gpu-deepseek41/src/experts_mxfp4.rs | 4 +- crates/gpu-deepseek41/src/prompt_walk.rs | 18 +- crates/gpu-deepseek41/src/shared_routed.rs | 366 ++++++-- crates/gpu-deepseek41/src/tier_join.rs | 16 +- crates/gpu-flint/Cargo.toml | 19 + crates/gpu-flint/src/body.rs | 583 ++++++++++++ crates/gpu-flint/src/lib.rs | 13 + crates/gpu-flint/src/prefill.rs | 128 +++ crates/gpu-flint/src/step.rs | 183 ++++ crates/gpu-flint/src/tests.rs | 252 +++++ crates/gpu-flint/src/tier.rs | 65 ++ crates/gpu-gates/Cargo.toml | 4 +- crates/gpu-gates/src/bin/bloomery_serve.rs | 27 +- crates/gpu-gates/src/bin/gate_flint_attn.rs | 318 +++++++ crates/gpu-gates/src/bin/gate_flint_router.rs | 302 ++++-- .../gpu-gates/src/bin/gate_solar2_twocard.rs | 1 + .../src/bin/shared/serve_seats/hybrid.rs | 42 +- .../src/bin/shared/serve_seats/mod.rs | 4 +- .../src/bin/shared/serve_seats/tiered.rs | 11 +- crates/gpu-gates/src/record.rs | 68 +- crates/gpu-mimo2/src/tier.rs | 3 +- crates/gpu-minimaxm3/src/tier.rs | 3 +- crates/gpu/src/arch/qwen3moe/tier_leg.rs | 4 - crates/gpu/src/attention_h128.rs | 617 ++++++++++++ crates/gpu/src/attention_h128_ring.rs | 880 ++++++++++++++++++ crates/gpu/src/card_experts.rs | 122 ++- crates/gpu/src/card_leg.rs | 119 ++- crates/gpu/src/expert_input.rs | 257 +++++ crates/gpu/src/flash_gqa_h128_swa.rs | 2 +- crates/gpu/src/gemm/route.rs | 5 + crates/gpu/src/host/handoff.rs | 10 +- crates/gpu/src/host/leg_tier.rs | 39 +- crates/gpu/src/host/run.rs | 11 +- crates/gpu/src/lib.rs | 3 + crates/gpu/src/ring_gemm.rs | 33 +- crates/gpu/src/route_body.rs | 210 ++++- crates/gpu/src/routed_block.rs | 2 + crates/model/src/arch/coverage.rs | 131 ++- crates/model/src/arch/flint/host.rs | 83 ++ crates/model/src/arch/flint/mod.rs | 13 +- crates/model/src/arch/flint/place.rs | 478 ++++++++++ crates/model/src/arch/flint/tests.rs | 12 +- crates/model/src/arch/glm5next/place.rs | 9 +- crates/model/src/arch/mimo2/place.rs | 10 +- crates/model/src/arch/minimaxm3/place.rs | 12 +- crates/model/src/arch/musespark/place.rs | 2 +- crates/model/src/arch/names.rs | 6 + crates/model/src/arch/place_hybrid.rs | 42 +- crates/model/src/arch/qwen35moe/place.rs | 2 +- crates/model/src/arch/qwen3moe/place.rs | 2 +- crates/model/src/arch/solar2/place.rs | 2 + crates/model/src/moe.rs | 148 ++- crates/model/src/ops.rs | 113 +++ crates/model/tests/flint_meta.rs | 48 +- crates/model/tests/qwen3_bp_plan.rs | 10 +- crates/models/src/shape.rs | 40 +- crates/placement/src/accepts.rs | 12 +- crates/placement/src/kernels.rs | 3 + crates/placement/src/kernels/attention.rs | 79 ++ crates/placement/src/kernels/ffn.rs | 234 +++++ crates/placement/src/placement.rs | 10 +- crates/placement/src/placement/workstation.rs | 48 +- docs/research/flint-attach.md | 95 +- justfile | 56 +- tools/families.tsv | 2 +- tools/gate-class.tsv | 1 + tools/gate-paths.tsv | 2 +- tools/hf-arch-check.py | 8 +- tools/levers-direct.txt | 1 + tools/model-tax.tsv | 8 +- tools/recipes.py | 2 - tools/ref/dense-logits.py | 2 +- tools/release/build.sh | 2 +- tools/unsafe-ratchet.txt | 14 +- 83 files changed, 6229 insertions(+), 484 deletions(-) create mode 100644 crates/app/src/arch/flint.rs create mode 100644 crates/gpu-flint/Cargo.toml create mode 100644 crates/gpu-flint/src/body.rs create mode 100644 crates/gpu-flint/src/lib.rs create mode 100644 crates/gpu-flint/src/prefill.rs create mode 100644 crates/gpu-flint/src/step.rs create mode 100644 crates/gpu-flint/src/tests.rs create mode 100644 crates/gpu-flint/src/tier.rs create mode 100644 crates/gpu/src/attention_h128.rs create mode 100644 crates/gpu/src/attention_h128_ring.rs create mode 100644 crates/gpu/src/expert_input.rs create mode 100644 crates/model/src/arch/flint/host.rs create mode 100644 crates/model/src/arch/flint/place.rs create mode 100644 crates/placement/src/kernels/attention.rs create mode 100644 crates/placement/src/kernels/ffn.rs diff --git a/Cargo.lock b/Cargo.lock index 1b960d92a..b448c1b87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,6 +74,7 @@ dependencies = [ "bloomery-gguf", "bloomery-gpu", "bloomery-gpu-deepseek41", + "bloomery-gpu-flint", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-minimaxm3", @@ -197,6 +198,20 @@ dependencies = [ "cuda-host", ] +[[package]] +name = "bloomery-gpu-flint" +version = "0.1.0" +dependencies = [ + "bloomery-gguf", + "bloomery-gpu", + "bloomery-gpu-deepseek41", + "bloomery-levers", + "bloomery-model", + "bloomery-models", + "bloomery-runtime", + "cuda-core", +] + [[package]] name = "bloomery-gpu-gates" version = "0.3.9" @@ -208,6 +223,7 @@ dependencies = [ "bloomery-gguf", "bloomery-gpu", "bloomery-gpu-deepseek41", + "bloomery-gpu-flint", "bloomery-gpu-gemma-embedding2", "bloomery-gpu-gemma4a", "bloomery-gpu-glm5next", diff --git a/Cargo.toml b/Cargo.toml index ed941f89b..4313a222a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/qdot", "crates/gguf", "crates/models", "crates/model", "crates/model-hf", "crates/placement", "crates/threads", "crates/levers", "crates/verbs", "crates/gpu", "crates/gpu-deepseek41", "crates/gpu-glm5next", "crates/gpu-mimo2", "crates/gpu-solar2", "crates/gpu-minimaxm3", "crates/gpu-gemma-embedding2", "crates/gpu-gates", "crates/engram", "crates/engram-lab", "crates/sampler", "crates/tokenizer", "crates/serve", "crates/jinja", "crates/vision", "crates/gpu-vision", "crates/gpu-gemma4a", "crates/refset", "crates/runtime", "crates/app", "crates/decision", "crates/embed", "crates/hf", "crates/safetensors", "crates/os", "crates/device", "crates/metal", "crates/record", "crates/chain"] +members = ["crates/qdot", "crates/gguf", "crates/models", "crates/model", "crates/model-hf", "crates/placement", "crates/threads", "crates/levers", "crates/verbs", "crates/gpu", "crates/gpu-deepseek41", "crates/gpu-glm5next", "crates/gpu-mimo2", "crates/gpu-solar2", "crates/gpu-minimaxm3", "crates/gpu-flint", "crates/gpu-gemma-embedding2", "crates/gpu-gates", "crates/engram", "crates/engram-lab", "crates/sampler", "crates/tokenizer", "crates/serve", "crates/jinja", "crates/vision", "crates/gpu-vision", "crates/gpu-gemma4a", "crates/refset", "crates/runtime", "crates/app", "crates/decision", "crates/embed", "crates/hf", "crates/safetensors", "crates/os", "crates/device", "crates/metal", "crates/record", "crates/chain"] # 의도적으로 컴파일되지 않는 재현기다(MUL-7). 워크스페이스에 넣으면 --workspace가 영구히 빨갛다. exclude = ["crates/oxide-ice-unroll", "crates/oxide-ice-unroll/variants/unroll_smoke", "fuzz", "labs/hip-native"] diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index b828c35a2..38f3d0dd0 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -29,6 +29,7 @@ bloomery-gpu-mimo2 = { path = "../gpu-mimo2", optional = true } bloomery-gpu-minimaxm3 = { path = "../gpu-minimaxm3", optional = true } # The Solar-Open2 session (`arch::solar2`). bloomery-gpu-solar2 = { path = "../gpu-solar2", optional = true } +bloomery-gpu-flint = { path = "../gpu-flint", optional = true } # The levers the sessions' own configurations carry (the plan's card budget, the host set's load # settings). The crate has no dependency of its own, so every session reads it. bloomery-levers = { path = "../levers" } @@ -41,3 +42,4 @@ glm5next = ["dep:bloomery-gpu-glm5next"] mimo2 = ["dep:bloomery-gpu-mimo2"] minimaxm3 = ["dep:bloomery-gpu-minimaxm3"] solar2 = ["dep:bloomery-gpu-solar2"] +flint = ["dep:bloomery-gpu-flint"] diff --git a/crates/app/src/arch/flint.rs b/crates/app/src/arch/flint.rs new file mode 100644 index 000000000..5587fe585 --- /dev/null +++ b/crates/app/src/arch/flint.rs @@ -0,0 +1,76 @@ +//! Flint's session over its actual attention-ring body and common hybrid planner. + +use std::num::NonZeroUsize; + +use bloomery_gpu::head::Head; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError, GpuModel}; +use bloomery_gpu_flint::Body; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::PlanInputs; +use model::placement::Plan; + +use super::{Batch, ResidentSequences}; + +const WHAT: &str = "flint session"; + +/// A Flint session, with resident slots backed by the body's own stores. +pub type FlintSession = crate::Session; + +impl Batch for Body { + type Inputs = PlanInputs; + type Sequences = ResidentSequences; + + const WHAT: &'static str = self::WHAT; + + fn read(file: &Split) -> Result { + PlanInputs::read(file).map_err(|e| GpuError::plan(WHAT, e)) + } + + fn configure(inputs: &mut PlanInputs, cfg: &super::HybridCfg) -> Result<(), GpuError> { + model::arch::flint::place::configure_group(inputs, cfg.group) + .map_err(|e| GpuError::plan(WHAT, e)) + } + + fn open_placed( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + host: HostCfg, + residency: Residency, + slots: NonZeroUsize, + ) -> Result, GpuError> { + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) + } + + fn batch( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, + ids: &[u32], + at: u32, + ) -> Result { + bloomery_gpu_flint::prefill::prefill(gpu, w, body, head, ids, at) + } + + fn group(m: &mut GpuModel, g: usize) -> Result { + bloomery_gpu_flint::set_prefill_group(m, g) + } + + fn keepable(m: &GpuModel, n: u32) -> u32 { + m.body(WHAT) + .map_or(0, |b| b.keepable_at(m.pos(), n.min(m.pos()))) + } + + fn cut(m: &mut GpuModel, n: u32) -> Result<(), GpuError> { + if n == 0 { + return m.reset(); + } + let pos = m.pos(); + m.body_parts(WHAT)?.2.stand_for_cut(pos); + m.rollback(n) + } +} diff --git a/crates/app/src/arch/minimaxm3/mod.rs b/crates/app/src/arch/minimaxm3/mod.rs index ba4cadc72..765a0ce3d 100644 --- a/crates/app/src/arch/minimaxm3/mod.rs +++ b/crates/app/src/arch/minimaxm3/mod.rs @@ -9,8 +9,8 @@ //! every layer's key and value rows at those positions, every sparse layer's //! index keys, and the last position's logits. Every store is a plane of //! positions a later position never reads before it writes, so a cut keeps -//! every held position. This session holds one sequence; the serve seat parks -//! more ([`bloomery_gpu_minimaxm3`]'s `Slots`). +//! every held position. The plan counts the configured resident sequences; +//! [`crate::Session::add_slots`] parks them through the body's actual `Slots`. //! //! The session's shape is the common owner's ([`super::Batch`]): what the //! placement plans, how the prompt runs and how a cut is kept are shared with @@ -37,6 +37,7 @@ const WHAT: &str = "minimaxm3 session"; impl Batch for Body { type Inputs = PlanInputs; + type Sequences = super::ResidentSequences; const WHAT: &'static str = self::WHAT; @@ -52,8 +53,9 @@ impl Batch for Body { inputs: &PlanInputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError> { - Body::open_placed_slots(file, plan, inputs, 0, host, residency, NonZeroUsize::MIN) + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) } fn batch( diff --git a/crates/app/src/arch/mod.rs b/crates/app/src/arch/mod.rs index 04b2b3bd2..bff1125c4 100644 --- a/crates/app/src/arch/mod.rs +++ b/crates/app/src/arch/mod.rs @@ -8,10 +8,12 @@ //! and the cut rule — and [`Batch`] holds the rest: the [`Open`] the placement //! plans by, the [`Prompt`] the batch runs and the [`Keep`] the cut follows. +use std::num::NonZeroUsize; + use bloomery_gpu::head::Head; use bloomery_gpu::host::served::prompt_pass; use bloomery_gpu::host::swap::Residency; -use bloomery_gpu::model::{ChainBody, Rollback}; +use bloomery_gpu::model::{ChainBody, Rollback, Slots}; use bloomery_gpu::weights::Weights; use bloomery_gpu::{Gpu, GpuError, GpuModel}; use bloomery_levers::HostCfg; @@ -24,6 +26,8 @@ use crate::{Keep, Open, Prompt}; #[cfg(feature = "deepseek41")] pub mod deepseek41; +#[cfg(feature = "flint")] +pub mod flint; #[cfg(feature = "glm5next")] pub mod glm5next; #[cfg(feature = "mimo2")] @@ -56,6 +60,8 @@ pub struct HybridCfg { pub residency: Residency, /// The batches a prompt group runs layer by layer. pub group: usize, + /// The resident sequences charged by the plan before any allocation. + pub slots: NonZeroUsize, } /// The plan inputs of a hybrid body: the family's own hyperparameters under @@ -71,6 +77,7 @@ pub trait HybridInputs { machine: &'a Machine, ctx: u64, levers: &PlanLevers, + slots: NonZeroUsize, what: &'static str, ) -> Result, GpuError>; } @@ -85,9 +92,11 @@ impl HybridInputs for PlanInputs { machine: &'a Machine, ctx: u64, levers: &PlanLevers, + slots: NonZeroUsize, what: &'static str, ) -> Result, GpuError> { - PlanInputs::plan(self, machine, ctx, levers).map_err(|e| GpuError::plan(what, e)) + self.plan_with_slots(machine, ctx, levers, slots) + .map_err(|e| GpuError::plan(what, e)) } } @@ -98,6 +107,8 @@ impl HybridInputs for PlanInputs { pub trait Batch: ChainBody + Rollback + Sized { /// The plan inputs the body is loaded by. type Inputs: HybridInputs; + /// The sequence capability the body actually implements. + type Sequences: SequenceSlots; /// The session's name in its refusals. const WHAT: &'static str; @@ -105,6 +116,11 @@ pub trait Batch: ChainBody + Rollback + Sized { /// The plan inputs of `file`. fn read(file: &Split) -> Result; + /// Price configuration-dependent buffers before placement; unchanged for a fixed schedule. + fn configure(_inputs: &mut Self::Inputs, _cfg: &HybridCfg) -> Result<(), GpuError> { + Ok(()) + } + /// The body resident by `plan`, which `inputs` made: the layers the plan /// puts on the card, the host tier over the routed ones. fn open_placed( @@ -113,6 +129,7 @@ pub trait Batch: ChainBody + Rollback + Sized { inputs: &Self::Inputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError>; /// The prompt batch of `ids` from `at`: the head's row after the last. @@ -154,6 +171,10 @@ impl Open for B { B::read(file) } + fn configure_inputs(inputs: &mut B::Inputs, cfg: &HybridCfg) -> Result<(), GpuError> { + B::configure(inputs, cfg) + } + fn layer_count(inputs: &B::Inputs) -> usize { inputs.layers() } @@ -169,8 +190,9 @@ impl Open for B { ctx: usize, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, B::WHAT)?; let ctx = one_card_ctx(B::WHAT, machine, ctx)?; - inputs.plan_at(machine, ctx, &cfg.place, B::WHAT) + inputs.plan_at(machine, ctx, &cfg.place, cfg.slots, B::WHAT) } fn open( @@ -179,7 +201,8 @@ impl Open for B { plan: &Plan<'_>, cfg: &HybridCfg, ) -> Result, GpuError> { - B::open_placed(file, plan, inputs, cfg.host, cfg.residency) + >::check(cfg.slots, B::WHAT)?; + B::open_placed(file, plan, inputs, cfg.host, cfg.residency, cfg.slots) } /// The configuration's group; the units it needs are made here. @@ -222,6 +245,81 @@ impl Keep for B { } } +/// The sequence operations a common hybrid seat may offer. +pub trait SequenceSlots { + /// Whether several resident sequences are backed by body-owned stores. + const RESIDENT: bool; + /// Refuse an unsupported count before planning or allocation. + fn check(slots: NonZeroUsize, what: &'static str) -> Result<(), GpuError>; + /// Park the planned sequences through the session's actual slot operations. + fn add(s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError>; + /// The number of resident sequences the session currently holds. + fn count(s: &crate::Session) -> usize; + /// Select the actual sequence every following call acts on. + fn select(s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError>; +} + +/// A body that holds one sequence; other slots are refused, not simulated. +pub struct SingleSequence; + +impl SequenceSlots for SingleSequence { + const RESIDENT: bool = false; + + fn check(slots: NonZeroUsize, what: &'static str) -> Result<(), GpuError> { + if slots.get() != 1 { + return Err(GpuError::Shape { + what, + detail: format!( + "the body holds one sequence, not {} resident slots", + slots.get() + ), + }); + } + Ok(()) + } + + fn add(_s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError> { + >::check(slots, "single-sequence session").map_err(Into::into) + } + + fn count(_s: &crate::Session) -> usize { + 1 + } + + fn select(_s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError> { + if slot == 0 { + Ok(()) + } else { + Err(crate::SessionError::Refused(format!( + "select_slot {slot}: the body holds one sequence" + ))) + } + } +} + +/// Resident sequences backed by the body's actual parked stores. +pub struct ResidentSequences; + +impl> SequenceSlots for ResidentSequences { + const RESIDENT: bool = true; + + fn check(_slots: NonZeroUsize, _what: &'static str) -> Result<(), GpuError> { + Ok(()) + } + + fn add(s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError> { + s.add_slots(slots.get()) + } + + fn count(s: &crate::Session) -> usize { + s.slots() + } + + fn select(s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError> { + s.select_slot(slot) + } +} + /// `ctx` positions as a plan takes them, for a body whose chain runs on one /// card: refused by name (`what` the body's session) when `machine` puts the /// layers on any other number of cards or `ctx` passes u64. diff --git a/crates/app/src/arch/musespark/mod.rs b/crates/app/src/arch/musespark/mod.rs index 91194dd31..a66a0ca01 100644 --- a/crates/app/src/arch/musespark/mod.rs +++ b/crates/app/src/arch/musespark/mod.rs @@ -47,8 +47,9 @@ impl Open for Body { ctx: usize, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, WHAT)?; let ctx = one_card_ctx(WHAT, machine, ctx)?; - inputs.plan_at(machine, ctx, &cfg.place, WHAT) + inputs.plan_at(machine, ctx, &cfg.place, cfg.slots, WHAT) } /// The model resident by `plan`, on the plan's one card, under the @@ -60,6 +61,7 @@ impl Open for Body { plan: &Plan<'_>, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, WHAT)?; GpuModel::::open_placed_with(file, plan, 0, cfg.host, cfg.residency) } diff --git a/crates/app/src/arch/solar2/mod.rs b/crates/app/src/arch/solar2/mod.rs index 482533bfe..56eac5094 100644 --- a/crates/app/src/arch/solar2/mod.rs +++ b/crates/app/src/arch/solar2/mod.rs @@ -40,6 +40,7 @@ const WHAT: &str = "solar2 session"; impl Batch for Body { type Inputs = PlanInputs; + type Sequences = super::SingleSequence; const WHAT: &'static str = self::WHAT; @@ -53,8 +54,9 @@ impl Batch for Body { inputs: &PlanInputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError> { - Body::open_placed_slots(file, plan, inputs, 0, host, residency, NonZeroUsize::MIN) + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) } fn batch( diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 5eee1b184..df3396b79 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -161,6 +161,12 @@ pub trait Open: ChainBody + Sized { /// Read `file`'s headers. fn inputs(file: &Split) -> Result; + /// Apply load settings that change planned buffers, before the plan borrows its inputs. + /// Bodies with no configuration-dependent buffers keep the header-derived inputs. + fn configure_inputs(_inputs: &mut Self::Inputs, _cfg: &Self::Cfg) -> Result<(), GpuError> { + Ok(()) + } + /// The layer count a placement's machine is laid out for. fn layer_count(inputs: &Self::Inputs) -> usize; @@ -303,7 +309,8 @@ impl Loaded { log: &mut impl OpenLog, edit: impl FnOnce(&B::Inputs, &mut Plan<'_>) -> Result<(), SessionError>, ) -> Result>, SessionError> { - let inputs = B::inputs(&file)?; + let mut inputs = B::inputs(&file)?; + B::configure_inputs(&mut inputs, &args.cfg)?; let machine = (args.machine)(B::layer_count(&inputs)); let mut plan = B::plan(&inputs, &machine, args.ctx, &args.cfg)?; edit(&inputs, &mut plan)?; diff --git a/crates/gpu-deepseek41/src/experts_mxfp4.rs b/crates/gpu-deepseek41/src/experts_mxfp4.rs index 033f5f291..51f15c799 100644 --- a/crates/gpu-deepseek41/src/experts_mxfp4.rs +++ b/crates/gpu-deepseek41/src/experts_mxfp4.rs @@ -182,7 +182,9 @@ mod dflash_kernels { // N_EXPERT = n_expert <= bias.len() by the launch contract; score and // sel_v are block-shared arrays of N_EXPERT entries; the barrier below // publishes them. - unsafe { publish_n::(probs.as_mut_ptr(), Some(bias), e0, score, sel_v, tid, fault) }; + unsafe { + publish_n::(probs.as_mut_ptr(), Some(bias), e0, score, sel_v, tid, fault) + }; thread::sync_threads(); if tid < 32 { diff --git a/crates/gpu-deepseek41/src/prompt_walk.rs b/crates/gpu-deepseek41/src/prompt_walk.rs index 905cba000..5b182c96f 100644 --- a/crates/gpu-deepseek41/src/prompt_walk.rs +++ b/crates/gpu-deepseek41/src/prompt_walk.rs @@ -214,7 +214,14 @@ impl BatchBufs { })?; let stream = gpu.stream(); let sets = group_sets(group); - Ok(Self { + let expected = model::placement::kernels::ffn::PromptWalk { + hidden: embd as u64, + sets: sets as u64, + } + .bytes(t_max as u64) + .and_then(|bytes| usize::try_from(bytes).ok()) + .ok_or_else(|| GpuError::shape("prompt batch buffers", "buffer geometry overflows"))?; + let buffers = Self { group, t_max, units: (0..sets) @@ -225,7 +232,14 @@ impl BatchBufs { stream: PromptStream::default(), facts: None, timing: None, - }) + }; + if buffers.bytes() != expected { + return Err(GpuError::shape( + "prompt batch buffers", + format!("allocated {} bytes, planned {expected}", buffers.bytes()), + )); + } + Ok(buffers) } /// What the next prompt calls move (`BLOOMERY_XSTREAM`), behind the guards every family shares diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index 7eaf38e55..2d9457022 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -4,20 +4,17 @@ //! [`shadow`], [`back`]) or a unit's rows (the batch: [`Routed::front_rows`] and its three). Both feeds leave the same //! bits but the shared expert's rows of a unit of [`GATED_FROM`] rows or more, which its GEMMs write. //! -//! Every layer is a routed layer with one shared expert, so the block has no dense arm. The router, the handoff, the -//! go, the card experts' call and the wait and add around the host leg are the common routed block -//! ([`bloomery_gpu::routed_block`]); this file holds the shared expert and the base the card sum is added to. Its -//! parts around the host leg: -//! - the front: `ffn_norm` RMS of the rows of the residual after the mixer `x1`, and the router over them: the -//! scores, the selection bias (the choice only), the top `R::N_USED`, the weights from the unbiased scores -//! normalized and scaled. The step goes on to the handoff into the host-mapped image and the go; four launches. -//! The batch downloads the rows, weights and ids to the host ([`Routed::download`]). -//! - the shadow: while the host computes, the shared expert, its output summed into `x1` (`x1s`), then the card -//! experts the slot map puts on the card with `x1s` as the base their sum is added to; none of the latter on a -//! layer with no card experts. The base is `x1 + shared` before the card sum is added, in the step and the batch -//! alike. -//! - the back: the wait, then the add of the host's routed sum and the shadow's `card + x1s` (`x1s` alone on a -//! layer with no card experts). One launch after the wait. +//! Dense leading layers run the common dense block; routed layers have one ungated shared expert. The router, +//! handoff, go, card experts and host wait are the common routed block ([`bloomery_gpu::routed_block`]). +//! [`FfnConfig`] resolves the residual association and expert-input range protection once at load: +//! - the front normalizes the attention residual, scores and selects through `R`'s normalization and bias +//! policy, then hands the normed row and picks to the host; +//! - the shadow computes the shared expert and card experts while the host runs. The default saves +//! `residual + shared` as the base. [`ResidualOrder::RoutedSharedResidual`] keeps the residual separate; +//! - the back joins the host and card routed sums. The default adds the host to the card-plus-base rows; +//! the other association computes `(routed + shared) + residual`, after the complete routed sum is ready. +//! +//! The step and prompt batch use the same loaded association, including tier-card joins. //! //! A load whose slot map names an expert tier card ([`TierSide`]) runs a layer the tier holds experts of as a tier //! layer: the front's handoff also writes the tier's image and each slot's tier place and the go is the host's and the @@ -28,15 +25,15 @@ //! [`Routed::join_rows`] (the host leg's [`bloomery_gpu::host::BatchLeg::join_tiered`] hands the tier's rows to the //! join, then [`Routed::back_rows`]). //! -//! What a family fixes is its router instance, its [`CardFamily`] (stack names, counts) and what its description -//! says: the routed experts' and the shared expert's activation ([`rule_of`]: a SwiGLU clamp or SwiGLU-OAI), the -//! weights' scale and widths, which every layer agrees on ([`describe`]). The shared expert's down is Q6_K or Q4_K -//! by layer, so its block plan is read per layer. The shared expert's Q8_0 pair kernel runs the clamp alone, so a -//! SwiGLU-OAI shared expert on Q8_0 weights is refused by name at the first launch. +//! What a family fixes is its router instance, its [`CardFamily`] (stack names, counts), the loaded arithmetic +//! facts, and its description: the routed and shared activations ([`rule_of`]), scales and widths +//! ([`describe`]). Shared and dense block plans are read per layer and support mixed sites, including IQ4_XS. +//! A Q8_0 shared pair runs the clamp alone; a SwiGLU-OAI shared expert on it is refused before launch. use std::sync::Arc; use bloomery_gpu::card_experts::{CardBatch, CardExperts, CardFamily}; +use bloomery_gpu::card_leg::ExpertInput; use bloomery_gpu::gemm::{Gemm32Kernels, GemmInput, GemmKernels, GemmRoute}; use bloomery_gpu::host::leg_tier::TierSide; use bloomery_gpu::host::run::HostRun; @@ -58,9 +55,13 @@ use crate::chain::ffn::FfnBatchKernels; use crate::experts::ExpertKernels; use crate::span::{span, span_mut}; use crate::tier_join::{self, TierBatch, TierSlots}; -use cuda_core::{CudaStream, DeviceBuffer}; +use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; +use cuda_device::float::add_rn_f32; +use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread}; +use cuda_host::cuda_module; use gguf::Split; use model::arch::names; +use model::placement::kernels::ffn; use models::{Act as ActRule, Ffn, LayerSpec, Score}; /// What the refusals of this file name. @@ -70,7 +71,125 @@ const WHAT: &str = "shared routed block"; /// GEMM a site): below it `enqueue_block_cols` runs the step's gemvs on each /// column (at most 8 columns), so a unit of up to 8 rows writes the step's /// bits and from it a launch count of 6 replaces 4 a column. -pub const GATED_FROM: usize = 9; +pub const GATED_FROM: usize = ffn::GATED_FROM as usize; + +/// The association of a routed block's shared output and attention residual. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ResidualOrder { + /// The shared output joins the residual before the routed sum. + #[default] + ResidualSharedRouted, + /// The complete routed sum joins the shared output before the residual. + RoutedSharedResidual, +} + +/// Arithmetic facts resolved once when the routed block is loaded. +#[derive(Clone, Copy, Debug)] +pub struct FfnConfig { + pub residual: ResidualOrder, + pub expert_input: ExpertInput, +} + +impl Default for FfnConfig { + fn default() -> Self { + Self { + residual: ResidualOrder::ResidualSharedRouted, + expert_input: ExpertInput::Direct, + } + } +} + +#[cuda_module] +mod residual_kernels { + use super::*; + + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + host.len() >= n, + card.len() >= n, + shared.len() >= n, + residual.len() >= n, + dest.len() >= n + ) + )] + pub fn routed_shared_residual( + host: &[f32], + card: &[f32], + shared: &[f32], + residual: &[f32], + has_card: u32, + n: u32, + mut dest: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: the launch bounds every input and output by n; i is guarded. + unsafe { + let mut routed = *host.get_unchecked(i); + if has_card != 0 { + routed = add_rn_f32(routed, *card.get_unchecked(i)); + } + let ffn = add_rn_f32(routed, *shared.get_unchecked(i)); + *dest.get_unchecked_mut(i) = add_rn_f32(ffn, *residual.get_unchecked(i)); + } + } +} + +struct ResidualKernels { + module: residual_kernels::LoadedModule, +} + +impl ResidualKernels { + fn load(ctx: &Arc) -> Result { + // SAFETY: this package owns the embedded bundle; enqueue checks its contract. + let module = unsafe { bloomery_gpu::shared_module!(residual_kernels, ctx)? }; + Ok(Self { module }) + } + + fn enqueue( + &self, + stream: &CudaStream, + (host, card): (&DeviceBuffer, Option<&DeviceBuffer>), + (shared, residual): (&DeviceBuffer, &DeviceBuffer), + n: usize, + dest: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + if n == 0 + || host.len() < n + || card.is_some_and(|c| c.len() < n) + || shared.len() < n + || residual.len() < n + || dest.len() < n + { + return Err(GpuError::shape( + WHAT, + "routed residual join outside its buffers", + )); + } + let n = bloomery_gpu::launch_u32(WHAT, "residual values", n)?; + let prep = self + .module + .prepare_routed_shared_residual(LaunchConfig1D::new(n.div_ceil(256), 256, 0))?; + self.module.routed_shared_residual( + stream, + &prep, + host, + card.unwrap_or(host), + shared, + residual, + u32::from(card.is_some()), + n, + dest, + )?; + Ok(()) + } +} /// What the load takes of the model besides the description and the file. #[derive(Clone, Copy, Debug)] @@ -146,16 +265,15 @@ impl DenseFfn { } } -/// Layer `l`'s routed block for the common one: its norm, router and bias -/// tensors (every layer has a bias) and its row in `map`. -fn route_layer(map: &SlotMap, l: usize) -> Result { +/// Layer `l`'s norm, router, optional selection bias and row in `map`. +fn route_layer(map: &SlotMap, l: usize, bias: bool) -> Result { let row_off = map .row_offset(l) .ok_or_else(|| GpuError::shape(WHAT, format!("layer {l}: no row in the slot map")))?; Ok(RouteLayer { norm: names::ffn_norm(l), router: names::ffn_gate_inp(l), - bias: Some(names::exp_probs_b(l)), + bias: bias.then(|| names::exp_probs_b(l)), row_off, }) } @@ -169,6 +287,8 @@ struct Kernels { experts: ExpertKernels, /// The K-quant block's kernels: the shared expert is K-quant in this file. block: Option, + /// Loaded only for the routed-plus-shared, then residual association. + residual: Option, } /// The step's buffers: the router's outputs and each slot's place, and the @@ -191,6 +311,31 @@ struct SharedWide { u: DeviceBuffer, } +fn wide_layout( + (gate_up, down): (&[SiteTy], &[SiteTy]), + (embd, ff): (usize, usize), +) -> ffn::WideBlock { + let forms = |types: &[SiteTy]| { + types.iter().fold(ffn::WideForms::default(), |forms, ty| { + forms.with(ty.reads()) + }) + }; + ffn::WideBlock { + hidden: embd as u64, + ff: ff as u64, + input: forms(gate_up).without_f32(), + down: forms(down), + } +} + +fn device_forms(forms: ffn::WideForms) -> site::WideForms { + site::WideForms { + q128: forms.quant.q128, + q32: forms.quant.q32, + f32: forms.f32, + } +} + impl SharedWide { /// Buffers for up to `cap` rows over a block of width `ff` on `embd` values, whose gate and up read /// the types `gate_up` and whose down reads the types `down`. Load-time only. @@ -200,11 +345,12 @@ impl SharedWide { (cap, embd, ff): (usize, usize, usize), ) -> Result { let stream = gpu.stream(); + let layout = wide_layout((gate_up, down), (embd, ff)); Ok(SharedWide { gemm: GemmKernels::load(gpu.context())?, route: GemmRoute::new(stream, cap, 1)?, - x: WideActs::new(stream, WideActs::wants_of(gate_up).without_f32(), cap, embd)?, - h: WideActs::new(stream, WideActs::wants_of(down), cap, ff)?, + x: WideActs::new(stream, device_forms(layout.input), cap, embd)?, + h: WideActs::new(stream, device_forms(layout.down), cap, ff)?, g: DeviceBuffer::zeroed(stream, cap * ff)?, u: DeviceBuffer::zeroed(stream, cap * ff)?, }) @@ -325,7 +471,21 @@ impl BatchBufs { let z = |n: usize| DeviceBuffer::::zeroed(stream, n); let (shared_gate_up, shared_down) = block_types(per.iter().flatten().map(|l| &l.plan)); let (dense_gate_up, dense_down) = block_types(dense.iter().flatten().map(|l| &l.plan)); - Ok(BatchBufs { + let layout = ffn::RoutedBatch { + hidden: embd as u64, + experts: R::N_EXPERT as u64, + used: R::N_USED as u64, + shared: wide_layout((&shared_gate_up, &shared_down), (embd, ff)), + dense: (dense_ff > 0) + .then(|| wide_layout((&dense_gate_up, &dense_down), (embd, dense_ff))), + card: (card.any() || card.any_tier()).then(|| card.batch_layout(cap)), + tiered, + }; + let expected = layout + .bytes(cap as u64) + .and_then(|bytes| usize::try_from(bytes).ok()) + .ok_or_else(|| GpuError::shape(WHAT, "routed batch scratch byte overflow"))?; + let buffers = BatchBufs { cap, normed: z(cap * embd)?, probs: z(cap * R::N_EXPERT)?, @@ -348,7 +508,14 @@ impl BatchBufs { tier: tiered .then(|| TierBatch::new(stream, cap, R::N_USED)) .transpose()?, - }) + }; + if buffers.bytes() != expected { + return Err(GpuError::shape( + WHAT, + "routed batch allocation differs from its common scratch descriptor", + )); + } + Ok(buffers) } fn bytes(&self) -> usize { @@ -448,8 +615,8 @@ struct Described { } /// The description of `layers` as the compiled router, handoff and shared -/// block run it: `R`'s experts and picks, sigmoid scores with a bias and -/// normalized weights, a SwiGLU or SwiGLU-OAI shared expert without a gate. +/// block run it: `R`'s experts, picks, normalization and bias policy; sigmoid scores and a +/// SwiGLU or SwiGLU-OAI shared expert without a gate. /// Anything else is refused by name with its layer. fn describe(layers: &[LayerSpec]) -> Result { let (mut scale, mut expert_ff, mut shared_ff) = (None, None, None); @@ -469,8 +636,8 @@ fn describe(layers: &[LayerSpec]) -> Result || m.top_k as usize != R::N_USED || m.latent.is_some() || r.score != Score::Sigmoid - || !r.bias - || r.norm != models::RouterNorm::SelectedSum + || (!r.bias && !R::OPTIONAL_BIAS) + || r.norm != R::NORM || r.hash || shared.sigmoid_gate { @@ -478,9 +645,11 @@ fn describe(layers: &[LayerSpec]) -> Result WHAT, format!( "layer {l}: {m:?} is not the {}-expert top-{} sigmoid router \ - with a bias, normalized weights and an ungated shared expert", + with {:?} normalization, optional bias={} and an ungated shared expert", R::N_EXPERT, - R::N_USED + R::N_USED, + R::NORM, + R::OPTIONAL_BIAS ), )); } @@ -530,6 +699,28 @@ impl Routed { (layers, map, slots): (&[LayerSpec], &SlotMap, &Arc>), d: RoutedDims, card: CardFamily, + ) -> Result, GpuError> { + Self::new_configured( + gpu, + file, + w, + (layers, map, slots), + d, + card, + FfnConfig::default(), + ) + } + + /// Load the same common block with explicit arithmetic facts. The default keeps the legacy + /// association and expert input unchanged; the selected policies apply to every routed layer. + pub fn new_configured( + gpu: &Gpu, + file: &Split, + w: &Weights, + (layers, map, slots): (&[LayerSpec], &SlotMap, &Arc>), + d: RoutedDims, + card: CardFamily, + config: FfnConfig, ) -> Result, GpuError> { if (card.n_expert, card.n_used) != (R::N_EXPERT, R::N_USED) { return Err(GpuError::shape( @@ -564,7 +755,11 @@ impl Routed { }) .collect::, GpuError>>()?; let routes = (0..layers.len()) - .map(|l| routed(l).then(|| route_layer(map, l)).transpose()) + .map(|l| { + moe_of(layers, l) + .map(|m| route_layer(map, l, m.router.bias)) + .transpose() + }) .collect::, _>>()?; let route_dims = RouteDims { embd: d.embd, @@ -577,6 +772,9 @@ impl Routed { let k = Kernels { g32: Gemm32Kernels::load(ctx)?, experts: ExpertKernels::load(ctx)?, + residual: (config.residual == ResidualOrder::RoutedSharedResidual) + .then(|| ResidualKernels::load(ctx)) + .transpose()?, block: per .iter() .flatten() @@ -606,10 +804,12 @@ impl Routed { }) .transpose()?; let dims = (d.embd, desc.expert_ff); - let card = match tier { + let mut card = match tier { Some(_) => CardExperts::tiered(gpu, w, card, layers, map, dims, 1)?, None => CardExperts::new(gpu, w, card, layers, map, dims, 1)?, }; + card.set_expert_input(gpu, config.expert_input)?; + card.set_add_base(config.residual == ResidualOrder::ResidualSharedRouted); let dense_ff = dense.iter().flatten().map(|l| l.ff).max().unwrap_or(0); let batch = (d.batch_rows > 1) .then(|| { @@ -703,9 +903,8 @@ impl Routed { self.step.route.ids() } - /// Layer `l`'s selection bias, the values the router adds to the scores - /// before it ranks them: the ranking a reference engine's picks are read - /// at. + /// Layer `l`'s selection bias for reference-pick inspection. A layer with no bias is refused by + /// name; execution handles it through the router's optional-bias policy. pub fn route_bias<'a>( &'a self, w: &'a Weights, @@ -766,10 +965,8 @@ fn shared_launch<'a>( (kernels, blk) } -/// Enqueue layer `l`'s shared expert and card experts of the step in its host -/// leg's shadow, after its [`front`]: `x1s` is `x1` plus the shared expert's -/// output, and the card sum plus `x1s` goes to the card experts' `pre` where -/// the layer has any. +/// Enqueue layer `l`'s shared and card experts after [`front`], while the host runs. `x1s` holds the +/// legacy residual-plus-shared base, or the saved residual when the loaded association joins it last. pub fn shadow( gpu: &Gpu, w: &Weights, @@ -786,8 +983,13 @@ pub fn shadow( y: &mut r.step.sh_y, }; enqueue_block_act(gpu, w, &kernels, &mut r.block, (&blk, r.rules.1), io)?; - gpu.elem() - .enqueue_add(stream, x1, &r.step.sh_y, r.d.embd, &mut r.step.x1s)?; + if r.k.residual.is_some() { + span_mut(WHAT, &mut r.step.x1s, 0, r.d.embd)? + .copy_from_device_async(&*span(WHAT, x1, 0, r.d.embd)?, stream)?; + } else { + gpu.elem() + .enqueue_add(stream, x1, &r.step.sh_y, r.d.embd, &mut r.step.x1s)?; + } let rule = r.rules.0; r.route.shadow( (gpu, w), @@ -812,6 +1014,19 @@ pub fn back( ) -> Result<(), GpuError> { let n_tier = r.card.tier_k(l); if n_tier == 0 { + if let Some(join) = &r.k.residual { + hybrid.boundary().enqueue_back_of(gpu.stream(), 0, l)?; + return join.enqueue( + gpu.stream(), + ( + hybrid.boundary().hsum_of(0)?, + r.card.has(l).then(|| r.card.acc()), + ), + (&r.step.sh_y, &r.step.x1s), + r.d.embd, + dest, + ); + } return r.route.back(gpu, hybrid, &r.card, l, (&r.step.x1s, dest)); } const W: &str = "shared routed back (tier layer)"; @@ -827,6 +1042,22 @@ pub fn back( n_tier, n_used: R::N_USED, }; + if let Some(residual) = &r.k.residual { + hybrid.enqueue_tier_back(gpu.stream(), l, 0)?; + let mut join = r.card.step_join(r.step.route.sel()); + tier_join::enqueue_acc( + (gpu.stream(), kernels), + (&mut join, &slots), + (hybrid.tier_rows(0)?, 1), + )?; + return residual.enqueue( + gpu.stream(), + (hybrid.boundary().hsum_of(0)?, Some(join.acc)), + (&r.step.sh_y, &r.step.x1s), + r.d.embd, + dest, + ); + } tier_join::enqueue_step_join( gpu, kernels, @@ -961,10 +1192,8 @@ impl Routed { self.card_rows(gw, lm, (&[], None)) } - /// Layer `l`'s shared expert over the `m` rows [`Routed::front_rows`] - /// wrote: the shared expert's rows (per column below [`GATED_FROM`] rows, - /// on `site::gated` from it) and `x1s = x1 + shared`, the base - /// [`Routed::card_rows`] adds the card sum to. + /// Layer `l`'s shared expert over the `m` normed rows, per column below [`GATED_FROM`] and on + /// `site::gated` from it. Keep the residual-plus-shared base or the separate residual, as loaded. pub fn shared_rows( &mut self, (gpu, w): (&Gpu, &Weights), @@ -1007,8 +1236,14 @@ impl Routed { (&b.normed, &mut b.sh), )?; } - gpu.elem() - .enqueue_add(gpu.stream(), x1, &b.sh, m * n, &mut b.x1s) + if k.residual.is_some() { + span_mut(WHAT, &mut b.x1s, 0, m * n)? + .copy_from_device_async(&*span(WHAT, x1, 0, m * n)?, gpu.stream())?; + Ok(()) + } else { + gpu.elem() + .enqueue_add(gpu.stream(), x1, &b.sh, m * n, &mut b.x1s) + } } /// Layer `l`'s card experts, where the slot map puts any, over the `m` @@ -1097,6 +1332,13 @@ impl Routed { }); }; let slots = tb.slots(n, (n_card, n_tier, R::N_USED)); + if self.k.residual.is_some() { + return tier_join::enqueue_acc( + (gpu.stream(), kernels), + (&mut cb.join(), &slots), + (trows, m), + ); + } tier_join::enqueue_join( gpu, (gpu.stream(), kernels), @@ -1106,10 +1348,8 @@ impl Routed { ) } - /// `x = hsum +` the shadow's rows (the card sum plus `x1s`, or `x1s` on a - /// layer with no card experts) over the unit's `m` rows, after - /// [`Routed::shadow_rows`] and the host sums' upload; on a tier layer after - /// [`Routed::join_rows`], which wrote the sum. + /// Join the uploaded host sums and shadow results over `m` rows with the loaded association. + /// On a tier layer, [`Routed::join_rows`] first forms the raw routed sum over both cards. pub fn back_rows( &self, gpu: &Gpu, @@ -1118,6 +1358,20 @@ impl Routed { ) -> Result<(), GpuError> { let b = self.batch.as_ref().ok_or_else(no_batch)?; b.check(l, m)?; + if let Some(join) = &self.k.residual { + let card = b + .card + .as_ref() + .filter(|_| self.card.sums(l)) + .map(CardBatch::acc); + return join.enqueue( + gpu.stream(), + (hsum, card), + (&b.sh, &b.x1s), + m * self.d.embd, + x, + ); + } let shadowed = match b.card.as_ref().filter(|_| self.card.sums(l)) { Some(cb) => cb.pre(), None => &b.x1s, diff --git a/crates/gpu-deepseek41/src/tier_join.rs b/crates/gpu-deepseek41/src/tier_join.rs index e02e500ba..ddaff2392 100644 --- a/crates/gpu-deepseek41/src/tier_join.rs +++ b/crates/gpu-deepseek41/src/tier_join.rs @@ -53,6 +53,18 @@ pub fn enqueue_join( (join, slots): (&mut TierJoin<'_>, &TierSlots<'_>), (trows, m): (&DeviceBuffer, usize), base: &DeviceBuffer, +) -> Result<(), GpuError> { + enqueue_acc((stream, kernels), (&mut *join, slots), (trows, m))?; + gpu.elem() + .enqueue_add(stream, join.acc, base, m * slots.n, join.pre) +} + +/// Sum both cards' routed slots without adding a base. A caller whose shared output and residual +/// join after the complete routed sum uses this result directly. +pub fn enqueue_acc( + (stream, kernels): (&CudaStream, &FfnBatchKernels), + (join, slots): (&mut TierJoin<'_>, &TierSlots<'_>), + (trows, m): (&DeviceBuffer, usize), ) -> Result<(), GpuError> { let a = CardAccTier { down: join.down, @@ -66,9 +78,7 @@ pub fn enqueue_join( n_tier: slots.n_tier, n_used: slots.n_used, }; - kernels.enqueue_card_acc_tier(stream, &a, join.acc)?; - gpu.elem() - .enqueue_add(stream, join.acc, base, m * slots.n, join.pre) + kernels.enqueue_card_acc_tier(stream, &a, join.acc) } /// Layer `l`'s back of the step on a tier layer at the page's row `row`: the wait for the host and diff --git a/crates/gpu-flint/Cargo.toml b/crates/gpu-flint/Cargo.toml new file mode 100644 index 000000000..2e1678700 --- /dev/null +++ b/crates/gpu-flint/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bloomery-gpu-flint" +version = "0.1.0" +edition = "2024" +publish = false +license = "MIT" + +[lints] +workspace = true + +[dependencies] +cuda-core.workspace = true +bloomery-gpu = { path = "../gpu" } +bloomery-gpu-deepseek41 = { path = "../gpu-deepseek41" } +bloomery-gguf = { path = "../gguf" } +bloomery-levers = { path = "../levers" } +bloomery-model = { path = "../model" } +bloomery-models = { path = "../models" } +bloomery-runtime = { path = "../runtime" } diff --git a/crates/gpu-flint/src/body.rs b/crates/gpu-flint/src/body.rs new file mode 100644 index 000000000..27b7654db --- /dev/null +++ b/crates/gpu-flint/src/body.rs @@ -0,0 +1,583 @@ +//! Checked load, sequence ownership and common model seam for Flint. + +use std::num::NonZeroUsize; +use std::ops::Range; +use std::sync::Arc; + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch as AttnScratch}; +use bloomery_gpu::card_experts::CardFamily; +use bloomery_gpu::card_leg::ExpertInput; +use bloomery_gpu::head::Head; +use bloomery_gpu::host::leg_tier::{Hang, LegOpen}; +use bloomery_gpu::host::refuse_tier_count; +use bloomery_gpu::host::run::{HostRun, HostWidths}; +use bloomery_gpu::host::served::{ResidencyParts, TierBody}; +use bloomery_gpu::host::shell::TierShell; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::host::swap_source::{ + DEADLINE, LIVE_DELAY, ResidencyGlue, ResidencySpec, hosted_stacks, +}; +use bloomery_gpu::host::tier::TierOpen; +use bloomery_gpu::hybrid::{Chain, HostResidency, Hybrid, SlotMap}; +use bloomery_gpu::model::{ + CardBytes, ChainBody, HostServed, Rollback, Slots, StepMode, hosted_hybrid_tiered, + refuse_reserve_past, refuse_scratch_past, stage_run, +}; +use bloomery_gpu::rope_table::{RopeRows, RopeSpec, RopeTable}; +use bloomery_gpu::route_body::sigmoid_floor_256_6::RouterKernels; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{DeviceTensor, Gpu, GpuError, GpuModel}; +use bloomery_gpu_deepseek41::prompt_walk::{self, BatchBufs}; +use bloomery_gpu_deepseek41::shared_routed::{FfnConfig, ResidualOrder, Routed, RoutedDims}; +use bloomery_levers::{ChurnHosting, HostCfg}; +use cuda_core::{CudaStream, DeviceBuffer}; +use gguf::Split; +use model::arch::Arch; +use model::arch::flint::{host, place::PlanInputs}; +use model::arch::names; +use model::embed::EmbedTable; +use model::placement::Plan; +use model::placement::accepts::Program; +use models::{Ffn, Mixer, RopeMode}; +use runtime::layer::{Layer, MixerKind, ResidualKind, hosted}; + +pub(crate) const WHAT: &str = "flint Body"; +pub type FlintModel = GpuModel; +pub(crate) type FfnBody = Routed; + +/// The per-replay position. The embedding row is staged before refresh. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StepInput { + pub pos: u32, +} + +pub(crate) struct StepRows { + pub x: DeviceBuffer, + pub x1: DeviceBuffer, + pub xn: DeviceBuffer, + pub delta: DeviceBuffer, + pub pos: DeviceBuffer, + pub n_keys: DeviceBuffer, +} + +impl StepRows { + fn new(stream: &CudaStream, hidden: usize) -> Result { + Ok(Self { + x: DeviceBuffer::zeroed(stream, hidden)?, + x1: DeviceBuffer::zeroed(stream, hidden)?, + xn: DeviceBuffer::zeroed(stream, hidden)?, + delta: DeviceBuffer::zeroed(stream, hidden)?, + pos: DeviceBuffer::zeroed(stream, 1)?, + n_keys: DeviceBuffer::zeroed(stream, 1)?, + }) + } + + fn bytes(&self) -> usize { + [&self.x, &self.x1, &self.xn, &self.delta] + .into_iter() + .map(DeviceBuffer::num_bytes) + .sum::() + + self.pos.num_bytes() + + self.n_keys.num_bytes() + } +} + +/// Weights, kernels and scratch belong to the load; only caches and their high-water mark belong to a sequence. +pub struct Body { + pub(crate) shell: TierShell, + pub(crate) cfg: Vec, + pub(crate) bindings: Vec, + pub(crate) stores: Vec, + pub(crate) k: Kernels, + pub(crate) attention: AttnScratch, + pub(crate) batch_attention: AttnScratch, + pub(crate) s: StepRows, + pub(crate) rope: RopeRows, + pub(crate) ffn: FfnBody, + pub(crate) units: BatchBufs, + pub(crate) embed: EmbedTable, + row: Vec, + pub(crate) slots: Arc>, + pub(crate) at: u32, + pub(crate) ctx: usize, + pub(crate) hidden: usize, + eps: f32, + slots_planned: usize, + slots_made: usize, + prompt_reserve: u64, +} + +impl Body { + /// A single-sequence placed load with fixed expert residency. + pub fn open_placed( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + card: usize, + host: HostCfg, + ) -> Result { + Self::open_placed_slots( + file, + plan, + inputs, + card, + host, + Residency::Off, + NonZeroUsize::MIN, + ) + } + + /// Load exactly the number of resident sequences counted by the placement. + #[allow( + clippy::too_many_arguments, + reason = "the placed load's file, plan, card and host policy, residency and sequence count" + )] + pub fn open_placed_slots( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + card: usize, + host: HostCfg, + residency: Residency, + slots: NonZeroUsize, + ) -> Result { + let tiers = TierOpen::of_machine(plan.machine)?; + refuse_tier_count(WHAT, tiers.len())?; + let (layers, _) = stage_run(WHAT, plan, card, inputs.spec.layers.len())?; + let one = inputs.kv.bytes(layers, plan.ctx_max); + refuse_store_count(plan, card, one, slots)?; + let run = hosted(&inputs.spec.layers).map_err(|e| shape(e.to_string()))?; + let residency = ResidencySpec { + lever: residency, + pool: ChurnHosting::Hosted, + delay: LIVE_DELAY, + deadline: DEADLINE, + top_k: 6, + max_rows: 1, + stacks: Arc::new(hosted_stacks(&inputs.model, WHAT, (run.len(), true))?), + }; + GpuModel::load_placed_with( + file, + plan, + card, + host, + residency, + |_, _, _, _| Ok(()), + |gpu, file, w, set, glue| { + Self::load( + gpu, + file, + w, + (plan, inputs, card), + host, + (set, glue), + (&tiers, slots), + ) + }, + ) + } + + #[allow( + clippy::too_many_arguments, + reason = "the checked placed load's disjoint common loader parts" + )] + fn load( + gpu: &Gpu, + file: &Arc, + w: &Weights, + (plan, inputs, card): (&Plan<'_>, &PlanInputs, usize), + host_cfg: HostCfg, + (set, glue): (HostResidency, ResidencyGlue), + (tiers, seqs): (&[TierOpen], NonZeroUsize), + ) -> Result { + gpu.context().bind_to_thread()?; + let spec = &inputs.spec; + let (layers, ctx) = stage_run(WHAT, plan, card, spec.layers.len())?; + let hidden = spec.hidden as usize; + let cfg: Vec<_> = spec.layers.iter().map(Layer::of).collect(); + for (l, kind) in cfg.iter().enumerate() { + if kind.mixer != MixerKind::Gqa || kind.residual != ResidualKind::Plain { + return Err(shape(format!( + "layer {l}: {kind:?}, not plain-residual GQA" + ))); + } + match &spec.layers[l].ffn { + Ffn::Moe(m) if m.experts != 256 || m.top_k != 6 || m.expert_ff != 512 => { + return Err(shape(format!( + "layer {l}: expert geometry {m:?}, not the compiled floor256top6 block" + ))); + } + _ => {} + } + } + let run = hosted(&spec.layers).map_err(|e| shape(e.to_string()))?; + let tier_cards: Vec<_> = tiers.iter().map(|t| t.card).collect(); + let map = SlotMap::of_plan_tiers(plan, card, &tier_cards, run.clone(), 256)?; + let bindings = spec + .layers + .iter() + .enumerate() + .map(|(l, s)| Binding::load(file, w, l, s, hidden, spec.rms_eps, ctx)) + .collect::, _>>()?; + let stores = bindings + .iter() + .map(|b| Cache::new(gpu, b)) + .collect::, _>>()?; + let held: usize = stores.iter().map(Cache::bytes).sum(); + let planned = inputs.kv.bytes(layers, plan.ctx_max); + if held as u64 != planned { + return Err(shape(format!( + "stores hold {held} bytes; the plan counts {planned}" + ))); + } + for (l, b) in bindings.iter().enumerate() { + let counted = inputs.kv.rows_at(l, plan.ctx_max); + if b.capacity() as u64 != counted { + return Err(shape(format!( + "layer {l}: {} physical rows; the plan counts {counted}", + b.capacity() + ))); + } + } + let rope_base = spec + .layers + .iter() + .find_map(|s| match &s.mixer { + Mixer::Gqa(g) if g.rope.mode == RopeMode::Neox => Some(g.rope.base), + _ => None, + }) + .ok_or_else(|| shape("no NeoX layer in the validated description"))?; + let table = RopeTable::new(&RopeSpec::window(rope_base, 128))?; + let rope = RopeRows::new(gpu.stream(), &table, 128, ctx)?; + let slots = Arc::new(DeviceTensor::upload( + gpu.stream(), + &map.stage_view(), + run.len(), + 256, + )?); + let ffn = FfnBody::new_configured( + gpu, + file, + w, + (&spec.layers, &map, &slots), + RoutedDims { + embd: hidden, + rms_eps: spec.rms_eps, + batch_rows: crate::prefill::T_MAX, + }, + CardFamily { + what: WHAT, + n_expert: 256, + n_used: 6, + stacks: names::routed_stacks, + program: Program::FlintBody, + }, + FfnConfig { + residual: ResidualOrder::RoutedSharedResidual, + expert_input: ExpertInput::AmaxF16, + }, + )?; + let hang = Hang { + tiers, + family: &crate::tier::NODE_FAMILY, + open: LegOpen { + n_expert: 256, + layers: cfg.len(), + hidden, + n_used: 6, + rows: 1, + }, + cols: crate::prefill::T_MAX, + card_dontneed: host_cfg.card_dontneed, + bytes: CardBytes::File, + }; + let mut hybrid = hosted_hybrid_tiered( + (gpu, file, plan), + (run.clone(), map), + (host_cfg, set, &glue), + ( + HostWidths { + embd: hidden, + ff: 512, + n_used: 6, + }, + 1, + ), + Some(&hang), + |src| { + run.clone() + .map(|l| host::layer(src, spec, l)) + .collect::, _>>() + }, + )?; + prompt_walk::prepare_host(gpu, &mut hybrid, crate::prefill::T_MAX)?; + let embed = EmbedTable::new(Arc::clone(file), &names::token_embd(), hidden)?; + if embed.n_vocab() != spec.vocab as usize { + return Err(shape(format!( + "embedding vocabulary {} differs from {}", + embed.n_vocab(), + spec.vocab + ))); + } + let prompt_reserve = inputs + .prompt_batch_bytes(plan.machine) + .map_err(|e| GpuError::plan(WHAT, e))?; + let mut body = Self { + shell: TierShell::new(hybrid, glue, hidden, WHAT), + k: Kernels::load(gpu)?, + attention: AttnScratch::new(gpu, &bindings, 1)?, + batch_attention: AttnScratch::new(gpu, &bindings, crate::prefill::T_MAX)?, + s: StepRows::new(gpu.stream(), hidden)?, + units: BatchBufs::new(gpu, hidden, crate::prefill::T_MAX, 1)?, + cfg, + bindings, + stores, + rope, + ffn, + embed, + row: vec![0.0; hidden], + slots, + at: 0, + ctx, + hidden, + eps: spec.rms_eps, + slots_planned: seqs.get(), + slots_made: 1, + prompt_reserve, + }; + refuse_scratch_past( + WHAT, + plan, + card, + body.scratch_bytes(), + "step rows, attention, routed blocks, rope, host boundary and slot map", + )?; + body.check_prompt_reserve()?; + body.start_residency(gpu)?; + Ok(body) + } + + pub fn ctx(&self) -> usize { + self.ctx + } + pub fn kinds(&self) -> Vec { + self.cfg.clone() + } + pub fn store_bytes(&self) -> usize { + self.stores.iter().map(Cache::bytes).sum() + } + pub fn scratch_bytes(&self) -> usize { + self.s.bytes() + + self.attention.bytes() + + self.k.bytes() + + self.ffn.bytes() + + self.rope.table.num_bytes() + + self.shell.hybrid.boundary().device_bytes() + + self.slots.buf().num_bytes() + } + pub fn prompt_bytes(&self) -> usize { + self.batch_attention.bytes() + self.units.bytes() + self.ffn.batch_bytes() + } + pub(crate) fn check_prompt_reserve(&self) -> Result<(), GpuError> { + refuse_reserve_past( + WHAT, + "prompt batch buffers", + self.prompt_reserve, + self.prompt_bytes() as u64, + "attention rows, common walk units and shared routed batch buffers", + ) + } + + /// Rollback never restores a ring. Its depth is measured from the greatest written end, not the last cut. + pub fn cut_keeps(&self, pos: u32, n: u32) -> bool { + let end = self.at.max(pos); + n <= end + && self.bindings.iter().all(|b| { + runtime::stores::cut_depth_keeps( + u64::from(end - n), + b.capacity() as u64, + self.ctx as u64, + b.window() as u64, + ) + }) + } + pub fn keepable_at(&self, pos: u32, n: u32) -> u32 { + if self.cut_keeps(pos, n) { n } else { 0 } + } + pub fn stand_for_cut(&mut self, pos: u32) { + self.at = self.at.max(pos); + } + pub fn hybrid(&self) -> &Hybrid { + self.shell.hybrid() + } + pub fn hybrid_mut(&mut self) -> &mut Hybrid { + self.shell.hybrid_mut() + } + pub fn taps(&self, gpu: &Gpu) -> Result, GpuError> { + self.shell.taps(gpu) + } +} + +impl ChainBody for Body { + type Input = StepInput; + type Host = Body; + const UNHOSTED_POOL: bloomery_levers::UnhostedPool = bloomery_levers::UnhostedPool::NotCarried; + fn arch() -> Arch { + Arch::Flint + } + fn decode_input(&mut self, token: u32, pos: u32) -> Result { + if pos as usize >= self.ctx { + return Err(shape(format!("position {pos} past context {}", self.ctx))); + } + self.embed.row_into(token, &mut self.row)?; + Ok(StepInput { pos }) + } + fn refresh(&mut self, stream: &CudaStream, input: &StepInput) -> Result<(), GpuError> { + self.s.x.copy_from_host(stream, &self.row)?; + self.s.pos.copy_from_host(stream, &[input.pos])?; + self.s.n_keys.copy_from_host(stream, &[input.pos + 1])?; + self.at = self.at.max(input.pos + 1); + Ok(()) + } + fn enqueue_chain(&mut self, gpu: &Gpu, w: &Weights, head: &mut Head) -> Result<(), GpuError> { + crate::step::enqueue(gpu, w, self, head) + } + fn reset(&mut self, gpu: &Gpu) -> Result<(), GpuError> { + self.shell.hybrid.settle(gpu.stream())?; + self.at = 0; + Ok(()) + } + fn head_eps(&self) -> f32 { + self.eps + } + fn resident_bytes(&self) -> usize { + self.store_bytes() + + self.scratch_bytes() + + self.prompt_bytes() + + self + .shell + .taps + .as_ref() + .map_or(0, |t| t.iter().map(DeviceBuffer::num_bytes).sum()) + } + fn layers(&self) -> Range { + 0..self.cfg.len() + } + fn host(&mut self) -> Option<&mut Body> { + Some(self) + } +} + +impl TierBody for Body { + type Experts = HostRun; + fn hybrid(&self) -> &Hybrid { + self.shell.hybrid() + } + fn hybrid_mut(&mut self) -> &mut Hybrid { + self.shell.hybrid_mut() + } + fn residency_parts(&mut self) -> Option> { + self.shell.residency_parts(&self.slots) + } + fn serve_chain(&mut self, chain: Chain) -> Result<(), GpuError> { + self.shell.hybrid.serve_captured_of(chain) + } +} + +impl Rollback for Body { + fn rollback(&mut self, pos: u32) -> Result<(), GpuError> { + if !self.cut_keeps(self.at, pos) { + return Err(shape(format!( + "rollback to {pos} from written end {} loses ring keys; reset and refeed", + self.at + ))); + } + Ok(()) + } +} + +/// One resident sequence's device stores and greatest written end. +pub struct Seq { + stores: Vec, + at: u32, +} + +impl Slots for Body { + type Seq = Seq; + fn new_seq(&mut self, gpu: &Gpu) -> Result { + if self.slots_made >= self.slots_planned { + return Err(shape(format!( + "sequence {} exceeds {} planned resident sequences", + self.slots_made + 1, + self.slots_planned + ))); + } + let need = self.store_bytes(); + let free = gpu.alloc_room()?; + if need > free { + return Err(shape(format!( + "sequence needs {need} store bytes, card has {free}" + ))); + } + let stores = self + .bindings + .iter() + .map(|b| Cache::new(gpu, b)) + .collect::, _>>()?; + self.slots_made += 1; + Ok(Seq { stores, at: 0 }) + } + fn swap_seq(&mut self, _gpu: &Gpu, seq: &mut Seq) -> Result<(), GpuError> { + if self.stores.len() != seq.stores.len() + || self + .stores + .iter() + .zip(&seq.stores) + .any(|(a, b)| a.bytes() != b.bytes()) + { + return Err(shape("parked sequence has another store layout")); + } + std::mem::swap(&mut self.stores, &mut seq.stores); + std::mem::swap(&mut self.at, &mut seq.at); + Ok(()) + } + fn seq_bytes(&self) -> usize { + self.store_bytes() + } +} + +impl Drop for Body { + fn drop(&mut self) { + self.shell.hybrid.stop_swap(); + } +} + +pub fn set_taps(m: &mut FlintModel, on: bool) -> Result<(), GpuError> { + m.set_mode(StepMode::Eager); + let (gpu, _, body) = m.body_parts(WHAT)?; + body.shell.set_taps(gpu, body.cfg.len(), on) +} + +fn refuse_store_count( + plan: &Plan<'_>, + card: usize, + one: u64, + slots: NonZeroUsize, +) -> Result<(), GpuError> { + let want = one.saturating_mul(slots.get() as u64); + let actual = plan + .cards + .get(card) + .ok_or_else(|| shape(format!("no card {card} in plan")))? + .kv_bytes; + if want != actual { + return Err(shape(format!( + "{slots} resident sequences need {want} store bytes; plan counts {actual}" + ))); + } + Ok(()) +} + +pub(crate) fn shape(detail: impl Into) -> GpuError { + GpuError::shape(WHAT, detail) +} diff --git a/crates/gpu-flint/src/lib.rs b/crates/gpu-flint/src/lib.rs new file mode 100644 index 000000000..57ceacb09 --- /dev/null +++ b/crates/gpu-flint/src/lib.rs @@ -0,0 +1,13 @@ +//! Flint's composition of the common head128 attention, shared routed block and host tier. +//! Device-linked consumers build this crate with `cargo oxide`. + +pub mod body; +pub mod prefill; +mod step; +pub mod tier; + +pub use body::{Body, FlintModel, Seq, StepInput, set_taps}; +pub use prefill::{prefill, set_prefill_group}; + +#[cfg(test)] +mod tests; diff --git a/crates/gpu-flint/src/prefill.rs b/crates/gpu-flint/src/prefill.rs new file mode 100644 index 000000000..a0e50c733 --- /dev/null +++ b/crates/gpu-flint/src/prefill.rs @@ -0,0 +1,128 @@ +//! Flint attention plugged into the common prompt walk and shared FFN. + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch}; +use bloomery_gpu::head::Head; +use bloomery_gpu::rope_table::RopeRows; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError}; +use bloomery_gpu_deepseek41::prompt_walk::{Batch, BatchAttention, UnitRows}; +use runtime::layer::{FfnKind, Layer}; + +use crate::body::{Body, FlintModel, shape}; + +/// The plan and the ring both hold one common prompt unit. +pub const T_MAX: usize = model::arch::flint::place::PROMPT_ROWS; +const _: () = assert!(T_MAX as u64 == runtime::stores::PROMPT_BATCH); + +/// Run a prompt through real attention and dense/routed blocks, then the final head. +pub fn prefill( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, + ids: &[u32], + at: u32, +) -> Result { + let to = + runtime::prompt::call_end(at, ids.len(), body.ctx).map_err(|e| shape(e.to_string()))?; + for &id in ids { + if id as usize >= body.embed.n_vocab() { + return Err(shape(format!( + "prompt token {id} past vocabulary {}", + body.embed.n_vocab() + ))); + } + } + // A failed call may already have overwritten ring rows. Reset, not a lower logical position, repairs that state. + body.at = body.at.max(to); + let Body { + shell, + cfg, + bindings, + stores, + k, + batch_attention, + rope, + units, + ffn, + embed, + hidden, + ctx, + .. + } = body; + let mut attention = Attention { + gpu, + w, + cfg, + bindings, + stores, + k, + scratch: batch_attention, + rope, + }; + let mut walk = Batch { + gpu, + w, + bufs: units, + attention: &mut attention, + ffn, + hybrid: &mut shell.hybrid, + embed, + head, + taps: shell.taps.as_deref_mut(), + embd: *hidden, + ctx: *ctx, + layers: cfg.len(), + }; + let mut cursor = at; + walk.prompt(ids, &mut cursor) +} + +struct Attention<'a> { + gpu: &'a Gpu, + w: &'a Weights, + cfg: &'a [Layer], + bindings: &'a [Binding], + stores: &'a mut [Cache], + k: &'a Kernels, + scratch: &'a mut Scratch, + rope: &'a RopeRows, +} + +impl BatchAttention for Attention<'_> { + fn attend( + &mut self, + l: usize, + _u: usize, + m: usize, + rows: UnitRows<'_>, + ) -> Result<(), GpuError> { + // SAFETY: the common walk stages consecutive positions and each count is position + 1. + unsafe { + self.bindings[l].enqueue( + self.gpu, + self.w, + self.k, + self.scratch, + &mut self.stores[l], + rows.x, + rows.x1, + rows.pos, + rows.n_keys, + m, + self.rope, + ) + } + } + fn ffn(&self, l: usize) -> FfnKind { + self.cfg[l].ffn + } +} + +/// Change prompt grouping between calls; allocation stays within the plan's reserve. +pub fn set_prefill_group(m: &mut FlintModel, group: usize) -> Result { + let (gpu, _, body) = m.body_parts("flint prompt group")?; + let made = body.units.set_group(gpu, body.hidden, group)?; + body.check_prompt_reserve()?; + Ok(made) +} diff --git a/crates/gpu-flint/src/step.rs b/crates/gpu-flint/src/step.rs new file mode 100644 index 000000000..a20320782 --- /dev/null +++ b/crates/gpu-flint/src/step.rs @@ -0,0 +1,183 @@ +//! One-token walk: common attention, then the dense or routed block, and the output head. + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch}; +use bloomery_gpu::head::Head; +use bloomery_gpu::host::StepLeg; +use bloomery_gpu::host::run::HostRun; +use bloomery_gpu::rope_table::RopeRows; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError}; +use bloomery_gpu_deepseek41::shared_routed; +use cuda_core::DeviceBuffer; +use runtime::layer::{FfnKind, Layer}; +use runtime::sched::{self, At, LayerProgram, Overlap, PortKind}; + +use crate::body::{Body, FfnBody, StepRows, shape}; + +pub(crate) fn enqueue( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, +) -> Result<(), GpuError> { + let Body { + shell, + cfg, + bindings, + stores, + k, + attention, + s, + rope, + ffn, + .. + } = body; + let mut port = StepLeg::new(gpu.stream(), &mut shell.hybrid); + let mut program = Step { + gpu, + w, + cfg, + bindings, + stores, + k, + attention, + s, + rope, + ffn, + taps: shell.taps.as_deref_mut(), + head, + }; + sched::walk( + Overlap { + units: 1, + cols: 1, + port: PortKind::Step, + }, + program.cfg.len(), + &mut port, + &mut program, + ) +} + +struct Step<'a> { + gpu: &'a Gpu, + w: &'a Weights, + cfg: &'a [Layer], + bindings: &'a [Binding], + stores: &'a mut [Cache], + k: &'a Kernels, + attention: &'a mut Scratch, + s: &'a mut StepRows, + rope: &'a RopeRows, + ffn: &'a mut FfnBody, + taps: Option<&'a mut [DeviceBuffer]>, + head: &'a mut Head, +} + +impl Step<'_> { + fn tap(&mut self, l: usize) -> Result<(), GpuError> { + let last = l + 1 == self.cfg.len(); + if let Some(taps) = self.taps.as_deref_mut() { + let out = if last { + &*self.head.input_mut() + } else { + &self.s.x + }; + taps[l].copy_from_device_async(out, self.gpu.stream())?; + } + Ok(()) + } +} + +impl<'a> LayerProgram for Step<'a> { + type Port = StepLeg<'a, HostRun>; + + fn begin(&mut self, unit: usize) -> Result<(), GpuError> { + one(unit) + } + + fn front(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + let l = at.layer; + // SAFETY: refresh stages the one position and its count, position + 1. + unsafe { + self.bindings[l].enqueue( + self.gpu, + self.w, + self.k, + self.attention, + &mut self.stores[l], + &self.s.x, + &mut self.s.x1, + &self.s.pos, + &self.s.n_keys, + 1, + self.rope, + )?; + } + match self.cfg[l].ffn { + FfnKind::Moe => { + shared_routed::front(self.gpu, self.w, self.ffn, &self.s.x1, (port.hybrid(), l)) + } + FfnKind::Dense => { + let dest = if l + 1 == self.cfg.len() { + self.head.input_mut() + } else { + &mut self.s.x + }; + shared_routed::dense( + self.gpu, + self.w, + self.ffn, + l, + (&self.s.x1, &mut self.s.xn, &mut self.s.delta), + dest, + )?; + self.tap(l) + } + } + } + + fn shadow(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + if !self.cfg[at.layer].host_leg() { + return Ok(()); + } + shared_routed::shadow( + self.gpu, + self.w, + self.ffn, + &self.s.x1, + (port.hybrid().boundary(), at.layer), + ) + } + + fn back(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + let l = at.layer; + if !self.cfg[l].host_leg() { + return Ok(()); + } + let dest = if l + 1 == self.cfg.len() { + self.head.input_mut() + } else { + &mut self.s.x + }; + shared_routed::back(self.gpu, self.ffn, port.hybrid(), l, dest)?; + self.tap(l)?; + port.hybrid().row_enqueued(l, at.unit) + } + + fn end(&mut self, unit: usize) -> Result<(), GpuError> { + one(unit)?; + self.head.enqueue(self.gpu, self.w) + } +} + +fn one(unit: usize) -> Result<(), GpuError> { + if unit == 0 { + Ok(()) + } else { + Err(shape(format!("step walk has one unit, not unit {unit}"))) + } +} diff --git a/crates/gpu-flint/src/tests.rs b/crates/gpu-flint/src/tests.rs new file mode 100644 index 000000000..5be7729eb --- /dev/null +++ b/crates/gpu-flint/src/tests.rs @@ -0,0 +1,252 @@ +//! Full-file consumer transitions, compared against an untouched resident sequence. + +use std::num::NonZeroUsize; + +use bloomery_gpu::GpuError; +use bloomery_gpu::host::served::prompt_pass; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::model::StepMode; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::PlanInputs; +use model::placement::{PlanLevers, workstation}; + +use crate::{Body, FlintModel}; + +const CTX: usize = 1536; +const PREFIX: usize = 512; +const WRITTEN: usize = 1025; +const SLOTS: usize = 3; +const WHAT: &str = "flint full-file transitions"; + +type Error = Box; + +struct Observation { + token: u32, + pos: u32, + logits: Vec, +} + +fn observe(m: &FlintModel, token: u32) -> Result { + let logits = m.logits()?; + assert!( + logits.iter().all(|x| x.is_finite()), + "{WHAT}: finite final logits" + ); + Ok(Observation { + token, + pos: m.pos(), + logits: logits.into_iter().map(f32::to_bits).collect(), + }) +} + +fn equal(label: &str, actual: &Observation, expected: &Observation) { + assert_eq!(actual.pos, expected.pos, "{label}: sequence position"); + assert_eq!(actual.token, expected.token, "{label}: next-token argmax"); + assert_eq!( + actual.logits.len(), + expected.logits.len(), + "{label}: vocabulary" + ); + let mismatch = actual + .logits + .iter() + .zip(&expected.logits) + .position(|(a, b)| a != b); + assert!( + mismatch.is_none(), + "{label}: final logit bits first differ at {mismatch:?}" + ); +} + +fn prompt(m: &mut FlintModel, ids: &[u32]) -> Result { + prompt_pass(m, |m| { + m.run_rows(ids.len(), WHAT, |gpu, w, body, head, at| { + crate::prefill::prefill(gpu, w, body, head, ids, at) + }) + })? + .ok_or(GpuError::State { + what: WHAT, + missing: "the prompt's final token", + }) +} + +fn stream(vocab: u32, seed: u32) -> Vec { + let mut state = seed; + (0..=WRITTEN) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + state % vocab + }) + .collect() +} + +/// Two decode observations straddle the same warmed context in every arm. The remainder wraps +/// the 1024-row window ring, while leaving a saved decode result exactly at its rollback boundary. +fn feed(m: &mut FlintModel, ids: &[u32]) -> Result<([Observation; 2], Observation), GpuError> { + prompt(m, &ids[..PREFIX])?; + let first = m.step(&ids[PREFIX..PREFIX + 1])?; + let first = observe(m, first)?; + let second = m.step(&ids[PREFIX + 1..PREFIX + 2])?; + let second = observe(m, second)?; + let last = prompt(m, &ids[PREFIX + 2..WRITTEN])?; + Ok(([first, second], observe(m, last)?)) +} + +#[test] +#[ignore = "needs a CUDA card and the complete Flint GGUF in BLOOMERY_MODEL"] +fn hw_full_file_capture_slots_reset_and_ring_boundary() -> Result<(), Error> { + let path = std::env::var_os("BLOOMERY_MODEL") + .ok_or("flint full-file transitions: BLOOMERY_MODEL must name the complete Flint GGUF")?; + let file = Split::open(path)?; + let inputs = PlanInputs::read(&file)?; + assert_eq!( + inputs.spec.layers.len(), + 49, + "the released full model, not a small fixture" + ); + assert_eq!( + inputs.spec.hidden, 2048, + "the released full model residual width" + ); + let ids = stream(inputs.spec.vocab, 0x4f1a_29c3); + let foreign = stream(inputs.spec.vocab, 0xe72d_810b); + assert_ne!( + &ids[..PREFIX], + &foreign[..PREFIX], + "the interleaved slot has a distinct history" + ); + let census = bloomery_gpu::census()?; + let [device] = census.as_slice() else { + return Err(format!( + "{WHAT}: exactly one CUDA device must be visible, found {}", + census.len() + ) + .into()); + }; + let machine = workstation::plan_on( + workstation::spec_of_device(device), + inputs.spec.layers.len(), + ); + let slots = NonZeroUsize::new(SLOTS).expect("three resident sequences"); + let plan = inputs.plan_with_slots(&machine, CTX as u64, &PlanLevers::default(), slots)?; + let host = HostCfg { + populate: false, + lock: false, + card_dontneed: false, + r8: false, + route_trace: false, + }; + let mut m = Body::open_placed_slots(file, &plan, &inputs, 0, host, Residency::Off, slots)?; + m.add_slots(SLOTS)?; + + // Slot 2 is a fresh sequence run without interleaving: its outputs are the control, not a guessed numerical band. + m.select_slot(2)?; + m.set_mode(StepMode::Eager); + let (control_decode, control_end) = feed(&mut m, &ids)?; + let control_next = m.step(&ids[WRITTEN..WRITTEN + 1])?; + let control_next = observe(&m, control_next)?; + + m.select_slot(0)?; + m.set_mode(StepMode::Graph); + let (captured_decode, captured_end) = feed(&mut m, &ids)?; + for (row, (actual, expected)) in captured_decode.iter().zip(&control_decode).enumerate() { + equal( + &format!("eager versus captured decode row {row}"), + actual, + expected, + ); + } + equal( + "eager versus captured after ring wrap", + &captured_end, + &control_end, + ); + assert!(m.has_capture(), "slot 0 actually captured a decode chain"); + let capture_nodes = m.step_graph_nodes()?.len(); + + // A second captured slot must not redirect the first slot's graph or its cache handles. + m.select_slot(1)?; + feed(&mut m, &foreign)?; + assert!(m.has_capture(), "slot 1 captured its own decode chain"); + m.select_slot(0)?; + assert_eq!( + m.pos(), + WRITTEN as u32, + "switching preserved slot 0's position" + ); + assert!( + m.has_capture(), + "slot 0's capture remained parked with its stores" + ); + assert_eq!( + m.step_graph_nodes()?.len(), + capture_nodes, + "slot 0's captured chain survived the switch" + ); + let resumed = m.step(&ids[WRITTEN..WRITTEN + 1])?; + equal( + "resumed slot versus untouched control", + &observe(&m, resumed)?, + &control_next, + ); + + // Reset after overwrite, then refeed through the retained capture, must reproduce a fresh sequence. + m.reset()?; + assert_eq!(m.pos(), 0, "reset rewound the selected sequence"); + assert!(m.has_capture(), "reset retains the reusable decode capture"); + let (refed_decode, refed_end) = feed(&mut m, &ids)?; + for (actual, expected) in refed_decode.iter().zip(&control_decode) { + equal( + "reset/refeed decode versus fresh sequence", + actual, + expected, + ); + } + equal( + "reset/refeed wrapped state versus fresh sequence", + &refed_end, + &control_end, + ); + + // At written end 1025, the ring holds positions 1..1025. Replaying position 513 + // needs 1..513, but position 512 needs overwritten position 0: depth 512, not 513. + let safe = WRITTEN as u32 - 512; + let lost = safe - 1; + assert!( + m.body(WHAT)?.cut_keeps(m.pos(), safe), + "depth 512 is retained" + ); + assert!( + !m.body(WHAT)?.cut_keeps(m.pos(), lost), + "depth 513 requires reset/refeed" + ); + m.rollback(safe)?; + assert_eq!( + m.pos(), + safe, + "retained rollback moved the logical position" + ); + let refusal = m.rollback(lost); + assert!( + matches!(&refusal, Err(GpuError::Shape { what, detail }) + if *what == "flint Body" && detail.contains("ring keys") && detail.contains("reset and refeed")), + "lost-window rollback must name the ring and recovery, got {refusal:?}" + ); + assert_eq!(m.pos(), safe, "refused rollback moved no sequence position"); + let replayed = m.step(&ids[safe as usize..safe as usize + 1])?; + equal( + "retained boundary replay", + &observe(&m, replayed)?, + &control_decode[1], + ); + + // Reset/refeed of slot 0 did not erase the parked control's position or continuation. + m.select_slot(2)?; + assert_eq!( + m.pos(), + (WRITTEN + 1) as u32, + "reset did not rewind a parked sequence" + ); + Ok(()) +} diff --git a/crates/gpu-flint/src/tier.rs b/crates/gpu-flint/src/tier.rs new file mode 100644 index 000000000..6286348fe --- /dev/null +++ b/crates/gpu-flint/src/tier.rs @@ -0,0 +1,65 @@ +//! Flint's checked description and host views for the common card-leg tier. + +use std::ops::Range; + +use bloomery_gpu::GpuError; +use bloomery_gpu::host::leg_tier::{self, LegFamily, LegWidths}; +use bloomery_gpu::host::node::NodeFamily; +use gguf::Split; +use model::ModelError; +use model::arch::flint::{host, place::PlanInputs}; +use model::moe::HostLayer; +use model::placement::ModelTensors; +use model::placement::accepts::Program; +use model::r8file::R8Source; +use models::LayerSpec; + +/// Maximum tokens per expert-tier launch; the placement reserves this capacity. +pub const CHUNK: usize = model::arch::flint::place::TIER_CHUNK as usize; + +struct Flint; + +impl LegFamily for Flint { + const WHAT: &'static str = "flint tier"; + const WORD: &'static str = "flint"; + const ROWS: usize = 1; + const CHUNK: usize = CHUNK; + type Inputs = PlanInputs; + + fn describe(file: &Split) -> Result { + PlanInputs::describe(file).map_err(|e| GpuError::plan(Self::WHAT, e)) + } + + fn into_model(inputs: PlanInputs) -> ModelTensors { + inputs.model + } + + fn layers(inputs: &PlanInputs) -> &[LayerSpec] { + &inputs.spec.layers + } + + fn widths(inputs: &PlanInputs) -> LegWidths { + LegWidths { + n_expert: 256, + n_used: 6, + n_embd: inputs.spec.hidden as usize, + ff: 512, + program: Program::FlintBody, + } + } + + fn expert_input(_inputs: &PlanInputs) -> bloomery_gpu::card_leg::ExpertInput { + bloomery_gpu::card_leg::ExpertInput::AmaxF16 + } + + fn routed( + src: R8Source<'_>, + inputs: &PlanInputs, + run: Range, + ) -> Result, ModelError> { + run.map(|l| host::layer(src, &inputs.spec, l)).collect() + } +} + +/// The tier service uses the same Program and host arithmetic as the stage card. +pub const NODE_FAMILY: NodeFamily = leg_tier::node_family::(); diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index 172d3c255..e2019422b 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -60,6 +60,7 @@ bloomery-gpu-mimo2 = { path = "../gpu-mimo2", optional = true } bloomery-gpu-solar2 = { path = "../gpu-solar2", optional = true } # The MiniMax-M3 program, reached only through the `minimaxm3` feature (its sparse-attention gate). bloomery-gpu-minimaxm3 = { path = "../gpu-minimaxm3", optional = true } +bloomery-gpu-flint = { path = "../gpu-flint", optional = true } # The EmbeddingGemma 2 encoder, reached only through the `gemma-embedding2` feature (its end-to-end # gate). bloomery-gpu-gemma-embedding2 = { path = "../gpu-gemma-embedding2", optional = true } @@ -152,6 +153,7 @@ glm5next = ["deepseek41", "dep:bloomery-gpu-glm5next", "bloomery-app/glm5next"] mimo2 = ["gpu", "dep:bloomery-gpu-mimo2", "bloomery-app/mimo2"] solar2 = ["gpu", "dep:bloomery-gpu-solar2", "bloomery-app/solar2"] minimaxm3 = ["gpu", "dep:bloomery-gpu-minimaxm3", "bloomery-app/minimaxm3"] +flint = ["gpu", "dep:bloomery-gpu-flint", "bloomery-app/flint"] gemma-embedding2 = ["gpu", "gemma4a", "dep:bloomery-gpu-vision", "dep:bloomery-gpu-gemma-embedding2", "dep:bloomery-serve"] gemma4a = ["gpu", "dep:bloomery-gpu-gemma4a", "dep:bloomery-vision"] @@ -171,7 +173,7 @@ required-features = ["deepseek41"] [[bin]] name = "bloomery-serve" path = "src/bin/bloomery_serve.rs" -required-features = ["glm5next", "clef", "mimo2", "solar2", "minimaxm3", "gemma-embedding2", "hf"] +required-features = ["glm5next", "clef", "mimo2", "solar2", "minimaxm3", "flint", "gemma-embedding2", "hf"] [[bin]] name = "bloomery-serve-ds41" diff --git a/crates/gpu-gates/src/bin/bloomery_serve.rs b/crates/gpu-gates/src/bin/bloomery_serve.rs index 06012fad1..f28a1e413 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve.rs @@ -1,5 +1,5 @@ //! `bloomery-serve` — one server binary, one seat a model: `--model -//! ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide` picks the seat +//! ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide` picks the seat //! (`serve_seats`, the shared seat modules), every other flag the seat's own, //! parsed by the seat's own parser, so each seat's flag set is exactly its //! per-model binary's. The @@ -7,7 +7,7 @@ //! picks the seat, and a bare start or `--help` prints [`drive::MODELS`], //! one line a seat with a working `--hf` example. //! -//! bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide] +//! bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide] //! [-m PATH | --model-file PATH | --hf [:]] //! [--head [--head-config ]] //! @@ -40,7 +40,8 @@ //! glm5-next the glm seat, qwen4exp the qwen38 seat, qwen3moe and qwen35moe //! the qwen3 seat, mimo2 the mimo2 seat, solar-open2 the solar2 seat, minimax-m3 //! the minimaxm3 seat, -//! nemotron_h and nemotron_h_moe the nemotron seat, raon1 the raon1 seat, muse-spark the musespark seat; +//! nemotron_h and nemotron_h_moe the nemotron seat, raon1 the raon1 seat, muse-spark the musespark seat, +//! and flint the flint seat; //! a dense qwen35 file generates through `--model qwen3`. With no model //! word it keeps the decision selection rules: a file with no head is //! refused by name, saying what would seat it (`--head`, a row's `--hf` @@ -164,19 +165,20 @@ mod drive { use serve::preflight::Driver; use crate::serve_seats::{ - decide, ds41, glm, mimo2, minimaxm3, musespark, nemotron, qwen3, qwen38, raon1, solar2, + decide, ds41, flint, glm, mimo2, minimaxm3, musespark, nemotron, qwen3, qwen38, raon1, + solar2, }; const NAME: &str = "bloomery-serve"; - const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide] \ + const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide] \ [-m PATH | --model-file PATH | --hf [:]] [--head \ [--head-config ]] [--embd-model FILE | --embd-hf \ [:]] [--embd-ctx N] [--embd-cache-bytes N] [--embd-mmproj FILE | \ --embd-no-mmproj] [--embd-image-min-tokens N] [--embd-image-max-tokens N] \ [llama-server's placement flags: -dev, -sm, -mg, -ts, -ngl, \ --cpu-moe, -fit, -fitt, -devd] | --list-devices | --version"; @@ -196,6 +198,7 @@ models (the --model word is optional with a model file: the file's architecture nemotron Nemotron 3.5 Bolt -m raon1 Raon-1-mini -m musespark Muse Spark Open 816B -m + flint Flint -m decide a decision model by its head --hf bartowski/Cloudflare_clef-flash-GGUF:Q5_K_M decide a decision model in its file --hf ggml-org/lev-GGUF:Q4_K_M"; @@ -221,6 +224,7 @@ models (the --model word is optional with a model file: the file's architecture Nemotron, Raon1, MuseSpark, + Flint, } /// A `--model` word: a generative seat's, or the decide seat's. @@ -243,9 +247,10 @@ models (the --model word is optional with a model file: the file's architecture "nemotron" => Ok(Word::Seat(Model::Nemotron)), "raon1" => Ok(Word::Seat(Model::Raon1)), "musespark" => Ok(Word::Seat(Model::MuseSpark)), + "flint" => Ok(Word::Seat(Model::Flint)), serve::decide::WORD => Ok(Word::Decide), other => Err(format!( - "--model is ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark or {}, not {other}", + "--model is ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, flint or {}, not {other}", serve::decide::WORD ) .into()), @@ -262,7 +267,7 @@ models (the --model word is optional with a model file: the file's architecture impl Model { /// Every generative seat. - const ALL: [Model; 10] = [ + const ALL: [Model; 11] = [ Model::Ds41, Model::Qwen38, Model::Glm, @@ -273,6 +278,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron, Model::Raon1, Model::MuseSpark, + Model::Flint, ]; /// The seat's word, as errors and the usage name it. @@ -288,6 +294,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron => "nemotron", Model::Raon1 => "raon1", Model::MuseSpark => "musespark", + Model::Flint => "flint", } } @@ -310,6 +317,7 @@ models (the --model word is optional with a model file: the file's architecture ), Model::Raon1 => matches!(Arch::from_name(arch), Ok(Arch::Raon1)), Model::MuseSpark => matches!(Arch::from_name(arch), Ok(Arch::MuseSpark)), + Model::Flint => matches!(Arch::from_name(arch), Ok(Arch::Flint)), } } } @@ -327,7 +335,7 @@ models (the --model word is optional with a model file: the file's architecture } let v = it.next().ok_or_else(|| { format!( - "--model needs a value (ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark or {}): {USAGE}", + "--model needs a value (ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, flint or {}): {USAGE}", serve::decide::WORD ) })?; @@ -602,6 +610,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron => nemotron::run(&rest, &side), Model::Raon1 => raon1::run(&rest, &side), Model::MuseSpark => musespark::run(&rest, &side), + Model::Flint => flint::run(&rest, &side), }; ended(r) } diff --git a/crates/gpu-gates/src/bin/gate_flint_attn.rs b/crates/gpu-gates/src/bin/gate_flint_attn.rs index 486bb3ed6..0c2ad26d6 100644 --- a/crates/gpu-gates/src/bin/gate_flint_attn.rs +++ b/crates/gpu-gates/src/bin/gate_flint_attn.rs @@ -25,10 +25,12 @@ fn main() -> std::process::ExitCode { #[cfg(feature = "gpu")] mod gate { + use bloomery_gpu::attention_h128::Kernels as LoadedAttention; use bloomery_gpu::fault::{Fault, FaultSite}; use bloomery_gpu::flash_gqa::{GROUP, HEAD, window_segments}; use bloomery_gpu::flash_gqa_h128_swa::{H128SwaArgs, H128SwaKernels}; use bloomery_gpu::qk_rms::{HEAD as RMS_HEAD, QkRmsArgs, QkRmsKernels}; + use bloomery_gpu::rope_neox::NeoxArgs; use bloomery_gpu::{Gpu, GpuError, LAYER_NONE}; use bloomery_gpu_gates::rounding::butterfly; use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, verdict}; @@ -631,6 +633,315 @@ mod gate { Ok(ok) } + fn sample(pos: usize, head: usize, dim: usize, salt: usize) -> f32 { + ((pos * 17 + head * 13 + dim * 7 + salt) % 97) as f32 / 97.0 - 0.5 + } + + fn turn_at(values: &[f32], table: &[f32], pos: usize, rotate: bool) -> Vec { + let mut out = values.to_vec(); + if rotate { + for d in 0..HEAD / 2 { + let (x, y) = (values[d], values[d + HEAD / 2]); + let (c, s) = (table[pos * HEAD + 2 * d], table[pos * HEAD + 2 * d + 1]); + out[d] = x.mul_add(c, -(y * s)); + out[d + HEAD / 2] = x.mul_add(s, y * c); + } + } + out + } + + /// Append across a wrap before a causal multirow pass. The full logical planes are the + /// independent oracle; only the device ring maps absolute rows modulo its capacity. + fn loaded_case( + gpu: &Gpu, + loaded: &LoadedAttention, + rotate: bool, + window: usize, + rows: usize, + m: usize, + ) -> Result { + let stream = gpu.stream(); + let ctx = 3072usize; + let start = 2047usize; + let kv = 2; + let heads = kv * GROUP; + let gq_h: Vec = (0..HEAD).map(|d| 0.6 + d as f32 * 0.001).collect(); + let gk_h: Vec = (0..HEAD).map(|d| 0.8 - d as f32 * 0.001).collect(); + let mut table_h = vec![0.0f32; ctx * HEAD]; + for p in 0..ctx { + for d in 0..HEAD / 2 { + let angle = p as f64 * 10000.0f64.powf(-((2 * d) as f64) / HEAD as f64); + table_h[p * HEAD + 2 * d] = angle.cos() as f32; + table_h[p * HEAD + 2 * d + 1] = angle.sin() as f32; + } + } + let mut logical_k = vec![0u16; kv * ctx * HEAD]; + let mut logical_v = vec![0u16; kv * ctx * HEAD]; + for kh in 0..kv { + for p in 0..start + m { + let raw: Vec = (0..HEAD).map(|d| sample(p, kh, d, 11)).collect(); + let key = turn_at(&rms_head(&raw, &gk_h, EPS), &table_h, p, rotate); + for d in 0..HEAD { + logical_k[(kh * ctx + p) * HEAD + d] = f32_to_f16_bits(key[d]); + logical_v[(kh * ctx + p) * HEAD + d] = + f32_to_f16_bits(sample(p, kh, d, 29) * 4.0); + } + } + } + let guard = 2 * HEAD; + let mut cache_k_h = vec![SENTINEL; kv * rows * HEAD + guard]; + let mut cache_v_h = cache_k_h.clone(); + for kh in 0..kv { + for p in start.saturating_sub(rows)..start { + let physical = (kh * rows + p % rows) * HEAD; + let logical = (kh * ctx + p) * HEAD; + cache_k_h[physical..physical + HEAD] + .copy_from_slice(&logical_k[logical..logical + HEAD]); + cache_v_h[physical..physical + HEAD] + .copy_from_slice(&logical_v[logical..logical + HEAD]); + } + } + let mut expect_k = cache_k_h.clone(); + let mut expect_v = cache_v_h.clone(); + let mut q_h = Vec::new(); + let mut raw_k = Vec::new(); + let mut raw_v = Vec::new(); + let mut expect_q = Vec::new(); + for t in 0..m { + let p = start + t; + for h in 0..heads { + let raw: Vec = (0..HEAD).map(|d| sample(p, h, d, 3)).collect(); + expect_q.extend(turn_at(&rms_head(&raw, &gq_h, EPS), &table_h, p, rotate)); + q_h.extend(raw); + } + for kh in 0..kv { + raw_k.extend((0..HEAD).map(|d| sample(p, kh, d, 11))); + raw_v.extend((0..HEAD).map(|d| sample(p, kh, d, 29) * 4.0)); + let physical = (kh * rows + p % rows) * HEAD; + let logical = (kh * ctx + p) * HEAD; + expect_k[physical..physical + HEAD] + .copy_from_slice(&logical_k[logical..logical + HEAD]); + expect_v[physical..physical + HEAD] + .copy_from_slice(&logical_v[logical..logical + HEAD]); + } + } + let pos_h: Vec = (start..start + m).map(|p| p as u32).collect(); + let counts_h: Vec = pos_h.iter().map(|p| p + 1).collect(); + let mut q = DeviceBuffer::from_host(stream, &q_h)?; + let mut kd = DeviceBuffer::from_host(stream, &raw_k)?; + let v = DeviceBuffer::from_host(stream, &raw_v)?; + let gq = DeviceBuffer::from_host(stream, &gq_h)?; + let gk = DeviceBuffer::from_host(stream, &gk_h)?; + let table = DeviceBuffer::from_host(stream, &table_h)?; + let pos = DeviceBuffer::from_host(stream, &pos_h)?; + let counts = DeviceBuffer::from_host(stream, &counts_h)?; + let sinks_h: Vec = (0..heads).map(|h| 7.0 + h as f32 * 0.3).collect(); + let sinks = DeviceBuffer::from_host(stream, &sinks_h)?; + let mut ck = DeviceBuffer::from_host(stream, &cache_k_h)?; + let mut cv = DeviceBuffer::from_host(stream, &cache_v_h)?; + let segs = window_segments(window); + let mut pv = DeviceBuffer::zeroed(stream, m * heads * segs * HEAD)?; + let mut pm = DeviceBuffer::zeroed(stream, m * heads * segs * 2)?; + let mut out = DeviceBuffer::from_host(stream, &vec![12345.0f32; m * heads * HEAD + HEAD])?; + gpu.take_fault()?; + // SAFETY: consecutive positions, distinct modulo rows, and m <= rows-window+1 + // for rings. All readable historical keys above were initialized. + unsafe { + loaded.enqueue_append( + stream, + NeoxArgs { + q: &mut q, + k: &mut kd, + v: &v, + gq: &gq, + gk: &gk, + table: &table, + pos: &pos, + eps: EPS, + n_head: heads, + n_kv: kv, + ctx, + m, + fault: gpu.unlabelled_sink(), + cache_k: &mut ck, + cache_v: &mut cv, + }, + rows, + rotate, + )?; + } + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &counts, + scale: score_scale(), + n_kv: kv, + ctx, + m, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + )?; + stream.synchronize()?; + let got_q = q.to_host_vec(stream)?; + let got = out.to_host_vec(stream)?; + let cache_same = ck.to_host_vec(stream)? == expect_k && cv.to_host_vec(stream)? == expect_v; + let query_same = bits_equal(&got_q, &expect_q); + let mut error = 0.0f32; + let mut no_sink_gap = 0.0f32; + for t in 0..m { + let end = start + t + 1; + for (h, &sink) in sinks_h.iter().enumerate() { + let b = (t * heads + h) * HEAD; + let base = (h / GROUP) * ctx * HEAD; + let keys = &logical_k[base..base + ctx * HEAD]; + let values = &logical_v[base..base + ctx * HEAD]; + let reference = attend( + &expect_q[b..b + HEAD], + keys, + values, + window_first(end, window), + end, + score_scale(), + Some(sink), + ); + error = error.max(max_abs(&got[b..b + HEAD], &reference)); + if t == 0 && h == 0 { + let no_sink = attend( + &expect_q[b..b + HEAD], + keys, + values, + window_first(end, window), + end, + score_scale(), + None, + ); + no_sink_gap = max_abs(&reference, &no_sink); + } + } + } + let canary = got[m * heads * HEAD..].iter().all(|x| *x == 12345.0); + let clean = gpu.take_fault()?.is_none(); + let mut ok = clause( + "loaded attention logical/ring", + query_same && cache_same && canary && clean && error <= BAND, + &format!( + "rotate={rotate} window={window} rows={rows} m={m} q={query_same} cache={cache_same} canary={canary} clean={clean} max_abs={error:.3e}" + ), + ); + ok &= clause( + "loaded attention sink witness", + no_sink_gap > 1e-5, + &format!("max_abs without sink={no_sink_gap:.3e}"), + ); + if rows < ctx { + ok &= refused( + "ring unsafe append-then-attend chunk", + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &counts, + scale: score_scale(), + n_kv: kv, + ctx, + m: rows - window + 2, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + ), + "invalid ring geometry", + ); + let bad_counts = DeviceBuffer::from_host(stream, &[0u32, ctx as u32 + 1])?; + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &bad_counts, + scale: score_scale(), + n_kv: kv, + ctx, + m: 2, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + )?; + stream.synchronize()?; + let invalid = out.to_host_vec(stream)?; + ok &= clause( + "ring invalid logical count", + invalid[..2 * heads * HEAD].iter().all(|v| v.is_nan()) + && gpu.take_fault()? == Some(Fault::at(LAYER_NONE, FaultSite::KeyCount)), + "zero and count past logical ctx are NaN with key_count", + ); + let invalid_pos = DeviceBuffer::from_host(stream, &[ctx as u32])?; + let before_k = ck.to_host_vec(stream)?; + let before_v = cv.to_host_vec(stream)?; + // SAFETY: the out-of-range position is never appended. + unsafe { + loaded.enqueue_append( + stream, + NeoxArgs { + q: &mut q, + k: &mut kd, + v: &v, + gq: &gq, + gk: &gk, + table: &table, + pos: &invalid_pos, + eps: EPS, + n_head: heads, + n_kv: kv, + ctx, + m: 1, + fault: gpu.unlabelled_sink(), + cache_k: &mut ck, + cache_v: &mut cv, + }, + rows, + rotate, + )?; + } + stream.synchronize()?; + ok &= clause( + "ring invalid absolute position", + q.to_host_vec(stream)?[..heads * HEAD] + .iter() + .all(|v| v.is_nan()) + && ck.to_host_vec(stream)? == before_k + && cv.to_host_vec(stream)? == before_v + && gpu.take_fault()? == Some(Fault::at(LAYER_NONE, FaultSite::CachePos)), + "position at logical ctx is NaN and preserves all physical rows", + ); + } + Ok(ok) + } + pub fn run() -> Result<(), GateError> { bloomery_levers::at_main(&[])?; let gpu = Gpu::new()?; @@ -653,6 +964,13 @@ mod gate { ok &= flash_fault(&gpu, &flash, 9, 8, "flash count past ctx")?; ok &= flash_refusals(&gpu, &flash)?; ok &= rms_case(&gpu, &rms)?; + let loaded = LoadedAttention::load(&gpu)?; + for rotate in [false, true] { + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 1024, 3)?; + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 1024, 512)?; + ok &= loaded_case(&gpu, &loaded, rotate, 0, 3072, 3)?; + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 3072, 3)?; + } if ok { Ok(()) } else { Err(checks_failed()) } } } diff --git a/crates/gpu-gates/src/bin/gate_flint_router.rs b/crates/gpu-gates/src/bin/gate_flint_router.rs index fbaee13b9..c6052fd15 100644 --- a/crates/gpu-gates/src/bin/gate_flint_router.rs +++ b/crates/gpu-gates/src/bin/gate_flint_router.rs @@ -11,8 +11,8 @@ fn main() -> std::process::ExitCode { mod gate { use bloomery_gpu::gemm::Gemm32Kernels; use bloomery_gpu::route_body::sigmoid_floor_256_6::{RouterKernels, RouterOut}; - use bloomery_gpu::routed_block::{RouteIn, RoutedRows, RouterEntry}; use bloomery_gpu::route_core::sigmoid; + use bloomery_gpu::routed_block::{RouteIn, RoutedRows, RouterEntry}; use bloomery_gpu::{DeviceTensor, Fault, FaultSite, Gpu, LAYER_NONE}; use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, verdict}; use cuda_core::DeviceBuffer; @@ -54,9 +54,13 @@ mod gate { // Independent contract expression: do not reuse the device divisor. fn weights_of(scores: &[f32], ids: &[u32], scale: f32) -> Vec { - let sum = ids.iter().fold(0.0f64, |sum, &e| sum + f64::from(scores[e as usize])); + let sum = ids + .iter() + .fold(0.0f64, |sum, &e| sum + f64::from(scores[e as usize])); let div = (sum as f32).max(FLOOR); - ids.iter().map(|&e| scores[e as usize] / div * scale).collect() + ids.iter() + .map(|&e| scores[e as usize] / div * scale) + .collect() } fn matrix(logits: &[f32]) -> Vec { @@ -89,7 +93,12 @@ mod gate { pass } - fn valid_case(cx: &Cx, name: &str, logits: &[f32], bias: Option<&[f32]>) -> Result { + fn valid_case( + cx: &Cx, + name: &str, + logits: &[f32], + bias: Option<&[f32]>, + ) -> Result { let s = cx.gpu.stream(); let mut wh = matrix(logits); if name == "dense" { @@ -111,7 +120,12 @@ mod gate { } let x = DeviceBuffer::from_host(s, &xh)?; let b = bias.map(|b| DeviceBuffer::from_host(s, b)).transpose()?; - let input = RouteIn { w: &w, bias: b.as_ref(), scale: SCALE, fault: cx.gpu.unlabelled_sink() }; + let input = RouteIn { + w: &w, + bias: b.as_ref(), + scale: SCALE, + fault: cx.gpu.unlabelled_sink(), + }; let mut out = RouterOut::new(s)?; let mut single = Pick::default(); let mut first = Pick::default(); @@ -134,103 +148,190 @@ mod gate { let mut ids = DeviceBuffer::zeroed(s, USED * TOKENS)?; let mut weights = DeviceBuffer::zeroed(s, USED * TOKENS)?; cx.router.route_rows( - (s, &cx.gpu, &cx.wide), &x, &input, TOKENS, - RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, + (s, &cx.gpu, &cx.wide), + &x, + &input, + TOKENS, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, )?; s.synchronize()?; let batch = Pick { - scores: probs.to_host_vec(s)?, ids: ids.to_host_vec(s)?, weights: weights.to_host_vec(s)?, + scores: probs.to_host_vec(s)?, + ids: ids.to_host_vec(s)?, + weights: weights.to_host_vec(s)?, }; let clean = cx.gpu.take_fault()?.is_none(); let mut expected = true; for t in 0..TOKENS { let scores = &single.scores[t * EXPERTS..(t + 1) * EXPERTS]; - let keys: Vec = scores.iter().enumerate() - .map(|(e, &p)| bias.map_or(p, |b| p + b[e])).collect(); + let keys: Vec = scores + .iter() + .enumerate() + .map(|(e, &p)| bias.map_or(p, |b| p + b[e])) + .collect(); let want_ids = top_k(&keys, USED); expected &= single.ids[t * USED..(t + 1) * USED] == want_ids; if name != "dense" { - let planted_keys: Vec = logits.iter().enumerate().map(|(e, &logit)| { - let p = sigmoid(logit * factors[t]); - bias.map_or(p, |b| p + b[e]) - }).collect(); + let planted_keys: Vec = logits + .iter() + .enumerate() + .map(|(e, &logit)| { + let p = sigmoid(logit * factors[t]); + bias.map_or(p, |b| p + b[e]) + }) + .collect(); expected &= single.ids[t * USED..(t + 1) * USED] == top_k(&planted_keys, USED); } expected &= bits_equal( - &single.weights[t * USED..(t + 1) * USED], &weights_of(scores, &want_ids, SCALE), + &single.weights[t * USED..(t + 1) * USED], + &weights_of(scores, &want_ids, SCALE), ); } let mut ok = clause( &format!("{name} selection and unbiased floor weights"), expected && clean && tickets, - &format!("bias={} clean={clean} tickets_zero={tickets}", bias.is_some()), + &format!( + "bias={} clean={clean} tickets_zero={tickets}", + bias.is_some() + ), + ); + ok &= clause( + &format!("{name} token-major rows"), + single.same(&batch), + "bit_identical", ); - ok &= clause(&format!("{name} token-major rows"), single.same(&batch), "bit_identical"); if name.starts_with("tiny") { let sum: f32 = first.ids.iter().map(|&e| first.scores[e as usize]).sum(); - let legacy: Vec = first.ids.iter() - .map(|&e| first.scores[e as usize] / (sum + 1e-20) * SCALE).collect(); - ok &= clause("tiny floor active", sum > 0.0 && sum < FLOOR - && !bits_equal(&first.weights, &legacy) - && first.weights.iter().sum::() < SCALE, "not epsilon renormalization"); + let legacy: Vec = first + .ids + .iter() + .map(|&e| first.scores[e as usize] / (sum + 1e-20) * SCALE) + .collect(); + ok &= clause( + "tiny floor active", + sum > 0.0 + && sum < FLOOR + && !bits_equal(&first.weights, &legacy) + && first.weights.iter().sum::() < SCALE, + "not epsilon renormalization", + ); } if name == "zero" || name == "ties" { - ok &= clause(&format!("{name} descending-id ties"), - first.ids == [255, 254, 253, 252, 251, 250], "six distinct ids"); + ok &= clause( + &format!("{name} descending-id ties"), + first.ids == [255, 254, 253, 252, 251, 250], + "six distinct ids", + ); if name == "zero" { - ok &= clause("zero selected sum", first.weights.iter().all(|&w| w == 0.0), "finite zero weights"); + ok &= clause( + "zero selected sum", + first.weights.iter().all(|&w| w == 0.0), + "finite zero weights", + ); } } if let Some(bias) = bias { - let keys: Vec = first.scores.iter().zip(bias).map(|(&p, &b)| p + b).collect(); - ok &= clause("bias changes selection not weight source", + let keys: Vec = first + .scores + .iter() + .zip(bias) + .map(|(&p, &b)| p + b) + .collect(); + ok &= clause( + "bias changes selection not weight source", first.ids != top_k(&first.scores, USED) && !bits_equal(&first.weights, &weights_of(&keys, &first.ids, SCALE)), - "unbiased selected scores consumed"); + "unbiased selected scores consumed", + ); } else if name == "normal" { let top7 = top_k(&first.scores, 7); - ok &= clause("six not seven selected scores", - !bits_equal(&first.weights, &weights_of(&first.scores, &top7, SCALE)[..USED]), - "top-seven renormalization differs"); + ok &= clause( + "six not seven selected scores", + !bits_equal( + &first.weights, + &weights_of(&first.scores, &top7, SCALE)[..USED], + ), + "top-seven renormalization differs", + ); } let x1 = DeviceBuffer::from_host(s, &xh[..K])?; - let graph = cx.gpu.capture(|st| cx.router.route(st, &x1, &input, &mut out))?; + let graph = cx + .gpu + .capture(|st| cx.router.route(st, &x1, &input, &mut out))?; let mut replay = graph.node_count() == 1; for _ in 0..2 { graph.launch(s)?; s.synchronize()?; replay &= read_one(cx, &out)?.same(&first) && out.tickets(s)? == 0; } - ok &= clause(&format!("{name} single capture"), replay, "one node, two exact replays, zero tickets"); - let graph = cx.gpu.capture(|st| cx.router.route_rows( - (st, &cx.gpu, &cx.wide), &x, &input, TOKENS, - RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, - ))?; + ok &= clause( + &format!("{name} single capture"), + replay, + "one node, two exact replays, zero tickets", + ); + let graph = cx.gpu.capture(|st| { + cx.router.route_rows( + (st, &cx.gpu, &cx.wide), + &x, + &input, + TOKENS, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, + ) + })?; let mut replay = graph.node_count() == 3; for _ in 0..2 { graph.launch(s)?; s.synchronize()?; let now = Pick { - scores: probs.to_host_vec(s)?, ids: ids.to_host_vec(s)?, weights: weights.to_host_vec(s)?, + scores: probs.to_host_vec(s)?, + ids: ids.to_host_vec(s)?, + weights: weights.to_host_vec(s)?, }; replay &= now.same(&single); } replay &= cx.gpu.take_fault()?.is_none(); - ok &= clause(&format!("{name} rows capture"), replay, "three nodes, two exact replays, no fault"); + ok &= clause( + &format!("{name} rows capture"), + replay, + "three nodes, two exact replays, no fault", + ); Ok(ok) } - fn invalid_case(cx: &Cx, name: &str, logit: Option, bias: Option, scale: f32) -> Result { + fn invalid_case( + cx: &Cx, + name: &str, + logit: Option, + bias: Option, + scale: f32, + ) -> Result { let s = cx.gpu.stream(); let mut logits = vec![0.0; EXPERTS]; - if let Some(logit) = logit { logits[40] = logit; } + if let Some(logit) = logit { + logits[40] = logit; + } let w = DeviceTensor::upload(s, &matrix(&logits), EXPERTS, K)?; let mut bh = vec![0.0; EXPERTS]; - if let Some(bias) = bias { bh[40] = bias; } + if let Some(bias) = bias { + bh[40] = bias; + } let b = bias.map(|_| DeviceBuffer::from_host(s, &bh)).transpose()?; let x = DeviceBuffer::from_host(s, &activations(&[1.0, 1.0]))?; - let input = RouteIn { w: &w, bias: b.as_ref(), scale, fault: cx.gpu.unlabelled_sink() }; + let input = RouteIn { + w: &w, + bias: b.as_ref(), + scale, + fault: cx.gpu.unlabelled_sink(), + }; let mut out = RouterOut::new(s)?; cx.gpu.take_fault()?; cx.router.route(s, &x, &input, &mut out)?; @@ -242,15 +343,28 @@ mod gate { let mut ids = DeviceBuffer::zeroed(s, USED * 2)?; let mut weights = DeviceBuffer::zeroed(s, USED * 2)?; cx.router.route_rows( - (s, &cx.gpu, &cx.wide), &x, &input, 2, - RoutedRows { probs: &mut probs, ids: &mut ids, weights: &mut weights }, + (s, &cx.gpu, &cx.wide), + &x, + &input, + 2, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, )?; s.synchronize()?; let rows = cx.gpu.take_fault()?; let rows_poisoned = weights.to_host_vec(s)?.iter().all(|w| !w.is_finite()); let want = Some(Fault::at(LAYER_NONE, FaultSite::Router)); - Ok(clause(name, one == want && rows == want && one_poisoned && rows_poisoned && tickets == 0, - &format!("one={one:?} rows={rows:?} poisoned={} tickets={tickets}", one_poisoned && rows_poisoned))) + Ok(clause( + name, + one == want && rows == want && one_poisoned && rows_poisoned && tickets == 0, + &format!( + "one={one:?} rows={rows:?} poisoned={} tickets={tickets}", + one_poisoned && rows_poisoned + ), + )) } fn host_boundaries(cx: &Cx) -> Result { @@ -260,20 +374,68 @@ mod gate { let short_bias = DeviceBuffer::zeroed(s, EXPERTS - 1)?; let mut out = RouterOut::new(s)?; let sink = cx.gpu.unlabelled_sink(); - let short = cx.router.enqueue_router(s, &w, &x, Some(&short_bias), SCALE, &mut out, sink).is_err(); + let short = cx + .router + .enqueue_router(s, &w, &x, Some(&short_bias), SCALE, &mut out, sink) + .is_err(); let mut probs = DeviceBuffer::zeroed(s, EXPERTS)?; let mut ids = DeviceBuffer::zeroed(s, USED)?; let mut weights = DeviceBuffer::zeroed(s, USED)?; - let empty = cx.router.enqueue_router_rows(s, &cx.gpu, &cx.wide, &w, &x, None, - SCALE, 0, &mut probs, &mut ids, &mut weights, sink).is_err(); - let short_rows = cx.router.enqueue_router_rows(s, &cx.gpu, &cx.wide, &w, &x, None, - SCALE, 2, &mut probs, &mut ids, &mut weights, sink).is_err(); - let legacy = bloomery_gpu::route_body::sigmoid_256_6::RouterKernels::load(cx.gpu.context())?; + let empty = cx + .router + .enqueue_router_rows( + s, + &cx.gpu, + &cx.wide, + &w, + &x, + None, + SCALE, + 0, + &mut probs, + &mut ids, + &mut weights, + sink, + ) + .is_err(); + let short_rows = cx + .router + .enqueue_router_rows( + s, + &cx.gpu, + &cx.wide, + &w, + &x, + None, + SCALE, + 2, + &mut probs, + &mut ids, + &mut weights, + sink, + ) + .is_err(); + let legacy = + bloomery_gpu::route_body::sigmoid_256_6::RouterKernels::load(cx.gpu.context())?; let mut legacy_out = bloomery_gpu::route_body::sigmoid_256_6::RouterOut::new(s)?; - let missing = legacy.route(s, &x, - &RouteIn { w: &w, bias: None, scale: SCALE, fault: sink }, &mut legacy_out).is_err(); - Ok(clause("host boundaries", short && empty && short_rows && missing, - "short bias, empty/short rows, legacy missing bias refused")) + let missing = legacy + .route( + s, + &x, + &RouteIn { + w: &w, + bias: None, + scale: SCALE, + fault: sink, + }, + &mut legacy_out, + ) + .is_err(); + Ok(clause( + "host boundaries", + short && empty && short_rows && missing, + "short bias, empty/short rows, legacy missing bias refused", + )) } pub fn run() -> Result<(), GateError> { @@ -283,10 +445,16 @@ mod gate { let router = RouterKernels::load(gpu.context())?; let wide = Gemm32Kernels::load(gpu.context())?; let cx = Cx { gpu, router, wide }; - let normal: Vec = (0..EXPERTS).map(|e| if e < 16 { 2.0 - e as f32 * 0.25 } else { -20.0 }).collect(); - let tiny: Vec = (0..EXPERTS).map(|e| -30.0 - (e % 16) as f32 * 0.25).collect(); + let normal: Vec = (0..EXPERTS) + .map(|e| if e < 16 { 2.0 - e as f32 * 0.25 } else { -20.0 }) + .collect(); + let tiny: Vec = (0..EXPERTS) + .map(|e| -30.0 - (e % 16) as f32 * 0.25) + .collect(); let mut bias = vec![0.0; EXPERTS]; - for (e, b) in bias.iter_mut().enumerate().skip(240) { *b = 2.0 + e as f32 / 256.0; } + for (e, b) in bias.iter_mut().enumerate().skip(240) { + *b = 2.0 + e as f32 / 256.0; + } let mut ok = valid_case(&cx, "normal", &normal, None)?; ok &= valid_case(&cx, "dense", &normal, None)?; ok &= valid_case(&cx, "biased", &normal, Some(&bias))?; @@ -294,9 +462,19 @@ mod gate { ok &= valid_case(&cx, "tiny-biased", &tiny, Some(&bias))?; ok &= valid_case(&cx, "zero", &vec![-200.0; EXPERTS], None)?; ok &= valid_case(&cx, "ties", &vec![0.0; EXPERTS], None)?; - for (name, value) in [("NaN", f32::NAN), ("+infinity", f32::INFINITY), ("-infinity", f32::NEG_INFINITY)] { + for (name, value) in [ + ("NaN", f32::NAN), + ("+infinity", f32::INFINITY), + ("-infinity", f32::NEG_INFINITY), + ] { ok &= invalid_case(&cx, &format!("{name} logit"), Some(value), None, SCALE)?; - ok &= invalid_case(&cx, &format!("{name} selection bias"), None, Some(value), SCALE)?; + ok &= invalid_case( + &cx, + &format!("{name} selection bias"), + None, + Some(value), + SCALE, + )?; } ok &= invalid_case(&cx, "nonfinite weight scale", None, None, f32::INFINITY)?; ok &= host_boundaries(&cx)?; diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index 9bcd34bc7..df267bdaf 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -298,6 +298,7 @@ mod gate { host: levers.host(), residency: Residency::Off, group: 1, + slots: std::num::NonZeroUsize::MIN, }; let pass = bits(&file, &cfg, &inputs)?; println!("{NAME}: {}", tier::tally_line()); diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs index 495f265e3..0a91647cc 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs @@ -1,6 +1,6 @@ -//! The seat of a body that runs one chain on one card, fed in prompt batches: -//! the one owner of the open, the load record, the seat's calls and the think -//! tags, for the family whose body holds one sequence ([`solar2`](super::solar2)). +//! A common hybrid HTTP seat over one chain on one card: opening, planning, +//! resident sequence selection, load records and prompt calls. Its body's +//! sequence capability preserves named refusals for single-sequence families. //! //! A seat gives its family the facts the run needs — the architectures it //! accepts, its plan inputs, the words its refusals name and the records it @@ -13,8 +13,8 @@ use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, Instant}; -use app::arch::HybridCfg; -use app::{Keep, Loaded, OpenArgs, OpenLog, Prompt, Session, SessionError}; +use app::arch::{Batch, HybridCfg, SequenceSlots}; +use app::{Loaded, OpenArgs, OpenLog, Prompt, Session, SessionError}; use bloomery_gpu::GpuModel; use bloomery_gpu::model::{ChainBody, StepMode}; use bloomery_gpu_gates::GateError; @@ -40,7 +40,7 @@ use super::tiered::{self, Open, Tiered}; pub trait Family: Sized + 'static { /// The body the seat serves: one chain on one card, fed in prompt /// batches. - type Body: ChainBody + app::Open, Cfg = HybridCfg> + Prompt + Keep; + type Body: Batch>; /// The family's hyperparameters: what its plan inputs are read for. type H; @@ -64,6 +64,11 @@ pub trait Family: Sized + 'static { /// The file's plan inputs, its own reading. fn read(split: &Split) -> Result, PlacementError>; + /// Price the chosen prompt group before the common context probes. + fn configure(_inputs: &mut PlanInputs, _group: usize) -> Result<(), GateError> { + Ok(()) + } + /// The records the seat prints, registered once at `main`. fn register(); @@ -118,6 +123,7 @@ struct Log { opened: Duration, ctx: usize, group: usize, + slots: usize, host_experts: u64, card_experts: u64, graph_nodes: usize, @@ -160,7 +166,7 @@ impl OpenLog for Log { } impl Log { - /// The `load` record of `m`, the one slot ready. + /// The `load` record of `m`, with every planned slot ready. fn record(&self, m: &GpuModel) { if let Some((card, rate)) = &self.lane { record::lane(card, rate).eprint(); @@ -170,7 +176,7 @@ impl Log { .w("arch", ::arch().name()) .u("resident_bytes", m.resident_bytes()) .u("ctx", self.ctx) - .u("slots", 1) + .u("slots", self.slots) .u("layers", m.layers().len()) .u("host_experts", self.host_experts) .u("card_experts", self.card_experts) @@ -196,6 +202,7 @@ fn open_session(a: Open<()>) -> Result, GateError> { opened: Duration::ZERO, ctx: a.ctx, group: a.group, + slots: a.slots.get(), pin_main: a.pin_main, pinned, host_experts: 0, @@ -220,16 +227,26 @@ fn open_session(a: Open<()>) -> Result, GateError> { host: a.host, residency: a.residency, group: a.group, + slots: a.slots, }, }; - let s = Loaded::::open(file, args, &mut log)? + let mut s = Loaded::::open(file, args, &mut log)? .ok_or("the open stopped at its plan")? .ready(&mut log)?; + ::Sequences::add(&mut s, a.slots)?; log.record(s.model()); Ok(Hybrid { s, ctx: a.ctx }) } impl Seat for Hybrid { + fn slots(&self) -> usize { + ::Sequences::count(&self.s) + } + + fn select(&mut self, slot: usize) -> Result<(), GateError> { + Ok(::Sequences::select(&mut self.s, slot)?) + } + fn pos(&self) -> usize { self.s.model().pos() as usize } @@ -309,7 +326,8 @@ impl Tiered for Hybrid { const ALIAS: &'static str = F::ALIAS; const FAMILY: &'static str = F::FAMILY; const SERVES: &'static str = F::SERVES; - const RESIDENT: bool = false; + const RESIDENT: bool = <::Sequences as SequenceSlots>::RESIDENT; + const ONE_PASS: bool = false; fn accepts(arch: Arch) -> bool { F::accepts(arch) @@ -319,6 +337,10 @@ impl Tiered for Hybrid { F::read(split) } + fn configure(inputs: &mut PlanInputs, group: usize) -> Result<(), GateError> { + F::configure(inputs, group) + } + fn flags(_flag: &str, _value: &str, _out: &mut ()) -> Result { Ok(false) } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs index b5b52bb18..edf2105a2 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs @@ -37,6 +37,8 @@ mod drafted; // DSpark draft, its slot rule and the draft word (`served::ServedDraft`). #[cfg(feature = "deepseek41")] pub mod ds41; +#[cfg(all(feature = "deepseek41", feature = "flint"))] +pub mod flint; #[cfg(feature = "glm5next")] pub mod glm; #[cfg(feature = "deepseek41")] @@ -80,7 +82,7 @@ pub mod minimaxm3_folder; pub mod musespark; // The seat of a body that runs one chain on one card, fed in prompt batches // (`solar2`): its open, its load record, its calls and its think tags. -#[cfg(all(feature = "deepseek41", feature = "solar2"))] +#[cfg(all(feature = "deepseek41", any(feature = "solar2", feature = "flint")))] pub mod hybrid; // The nemotron seat sits behind `deepseek41` for the same reason: the server // surface it binds through is scoped to it, and it runs no V4.1 code. The diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs index 696f42120..0ba36629b 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs @@ -228,6 +228,12 @@ pub trait Tiered: Seat + Sized + 'static { /// The file's plan inputs, its own `place::read`. fn read(split: &Split) -> Result, PlacementError>; + /// Price the selected prompt schedule before any context probe or allocation. + /// A family whose schedule has fixed buffers keeps its header-derived inputs. + fn configure(_inputs: &mut PlanInputs, _group: usize) -> Result<(), GateError> { + Ok(()) + } + /// One of the seat's own flags (`SeatFlags::parse`'s `own`): `Ok(true)` took it. fn flags(flag: &str, value: &str, out: &mut Self::Flags) -> Result; @@ -403,7 +409,7 @@ fn tier_experts( levers: &PlanLevers, ctx: Option, ) -> Result { - let machine = S::machine(offer, S::tier_batch(offer, inputs), inputs.model.layers)?; + let machine = S::machine(offer, S::tier_batch(offer, inputs)?, inputs.model.layers)?; let ctx = ctx.map_or(Ok(1), u64::try_from)?; let plan = inputs.plan(&machine, ctx, levers)?; Ok(plan.tier_n_l.iter().flatten().sum()) @@ -487,7 +493,8 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result Result { place::tier_batch(&inputs.hp, T_MAX as u64) } diff --git a/crates/gpu-minimaxm3/src/tier.rs b/crates/gpu-minimaxm3/src/tier.rs index b937c9291..ca916c671 100644 --- a/crates/gpu-minimaxm3/src/tier.rs +++ b/crates/gpu-minimaxm3/src/tier.rs @@ -20,8 +20,8 @@ use gguf::Split; use model::arch::minimaxm3::host as minimax_host; use model::arch::minimaxm3::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::{ModelTensors, PlacementError}; use model::placement::workstation::TierBatchBytes; +use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::{LayerSpec, Moe}; @@ -38,7 +38,6 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// The tier's prompt-batch bytes the plan reserves on the tier card and the host for the file /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. -#[must_use] pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(inputs, T_MAX as u64) } diff --git a/crates/gpu/src/arch/qwen3moe/tier_leg.rs b/crates/gpu/src/arch/qwen3moe/tier_leg.rs index f5b383b89..d0393f193 100644 --- a/crates/gpu/src/arch/qwen3moe/tier_leg.rs +++ b/crates/gpu/src/arch/qwen3moe/tier_leg.rs @@ -50,7 +50,6 @@ const _: () = assert!(model::arch::qwen35moe::place::LEG_TIER_CHUNK == LEG_CHUNK /// Threads per block of both join entries. const THREADS: u32 = 256; - /// The routed activation the tier runs: `silu(g)·u`. A clamped SwiGLU, and /// any other activation, is refused by name. fn qwen_act(moe: &Moe) -> Result { @@ -70,7 +69,6 @@ fn qwen_act(moe: &Moe) -> Result { } } - /// The host views of the routed layers `run`, the refusal naming the run as /// `word`'s. `n_layer` is the description's layer count, so a trunk layer is /// not refused for sitting past a next-token block. @@ -155,7 +153,6 @@ impl LegFamily for Qwen3 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - } /// Qwen3.6's facts for the common tier node service. The description's @@ -210,7 +207,6 @@ impl LegFamily for Qwen35 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - } /// Qwen3-30B's tier node service ([`leg_tier::node_family`]). diff --git a/crates/gpu/src/attention_h128.rs b/crates/gpu/src/attention_h128.rs new file mode 100644 index 000000000..ee8fbd1ca --- /dev/null +++ b/crates/gpu/src/attention_h128.rs @@ -0,0 +1,617 @@ +//! Loaded head-128 GQA: independent rotation and readable-window policies over f16 KV stores. +//! +//! KV retention belongs to the runtime's Store policy. This binding owns no logical cursor: +//! the caller supplies absolute positions and causal counts, and exchanges only cache planes +//! between slots. Ring appends are cut to `capacity - window + 1` rows before attention, so +//! a later row never overwrites a key still readable by the first row of the chunk. + +use crate::attention_h128_ring; +use crate::flash_gqa::{GROUP, HEAD, window_segments}; +use crate::flash_gqa_h128_swa::{H128SwaArgs, H128SwaKernels}; +use crate::gemm::{GEMM_MAX_SLOTS, Gemm32Kernels, GemmInput, GemmKernels, GemmRoute}; +use crate::qk_rms::{QkRmsArgs, QkRmsKernels}; +use crate::rope_neox::{NeoxArgs, RopeNeoxKernels}; +use crate::rope_table::RopeRows; +use crate::site::{self, SiteTy, SiteWeight, WideActs, WideForms, WideKernels}; +use crate::weights::Weights; +use crate::{Gpu, GpuError}; +use cuda_core::DeviceBuffer; +use gguf::Split; +use model::arch::models::{LayerSpec, Mixer, OutGate, RopeMode}; +use model::arch::names; +use model::placement::kernels::attention::{Forms, Head128Scratch}; + +const WHAT: &str = "attention_h128"; +const SITES: &[SiteTy] = &[ + SiteTy::Q3K, + SiteTy::Q4K, + SiteTy::Q5K, + SiteTy::Q6K, + SiteTy::Iq4Xs, + SiteTy::Q5_0, + SiteTy::Q8_0, + SiteTy::F32, + SiteTy::Mxfp8, + SiteTy::Fp8B128, + SiteTy::Bf16, +]; + +struct Projection { + name: String, + ty: SiteTy, + rows: usize, +} + +impl Projection { + fn load( + file: &Split, + w: &Weights, + name: String, + rows: usize, + cols: usize, + ) -> Result { + let ty = site::file_site(file, WHAT, &name, (rows, cols), SITES)?; + site::site(w, WHAT, &name, (rows, cols), ty)?; + Ok(Self { name, ty, rows }) + } + + #[allow( + clippy::too_many_arguments, + reason = "one typed projection's borrowed launch state (rust-quality R8)" + )] + fn enqueue( + &self, + gpu: &Gpu, + w: &Weights, + k: &WideKernels<'_>, + a: &WideActs, + x: &DeviceBuffer, + route: &GemmRoute, + m: usize, + y: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + site::gemm( + gpu, + k, + (self.ty, w, SiteWeight::Resident(&self.name)), + self.rows, + &a.input(x), + (route, GemmInput::PerSlot), + m, + y, + ) + } +} + +/// A layer's loaded sites, gains, sinks, and independent attention policies. +pub struct Binding { + layer: usize, + hidden: usize, + heads: usize, + kv: usize, + ctx: usize, + capacity: usize, + window: usize, + rotate: bool, + eps: f32, + norm: String, + qnorm: String, + knorm: String, + sinks: String, + q: Projection, + k: Projection, + v: Projection, + o: Projection, +} + +fn vector(file: &Split, w: &Weights, name: String, len: usize) -> Result { + let (_, t) = file + .find(&name) + .ok_or_else(|| GpuError::tensor(WHAT, &name, "a vector"))?; + if t.dims != [len as u64] { + return Err(GpuError::shape( + WHAT, + format!("{name} has shape {:?}; need [{len}]", t.dims), + )); + } + let mut values = vec![0.0; len]; + gguf::quant::dequant_row(t.ty, file.data(&name)?, &mut values) + .map_err(::model::ModelError::from)?; + if let Some(i) = values.iter().position(|x| !x.is_finite()) { + return Err(GpuError::shape(WHAT, format!("{name}[{i}] is not finite"))); + } + site::vector(w, WHAT, name, len) +} + +impl Binding { + /// Bind actual file and resident formats once. No architecture-name dispatch is involved. + #[allow( + clippy::too_many_arguments, + reason = "the loaded layer and its geometry (rust-quality R8)" + )] + pub fn load( + file: &Split, + w: &Weights, + layer: usize, + spec: &LayerSpec, + hidden: usize, + eps: f32, + ctx: usize, + ) -> Result { + let Mixer::Gqa(g) = &spec.mixer else { + return Err(GpuError::shape(WHAT, "a GQA layer is required")); + }; + let heads = g.heads as usize; + let kv = g.kv_heads as usize; + let rotate = match g.rope.mode { + RopeMode::None if g.rope.dims == 0 && g.rope.yarn.is_none() => false, + RopeMode::Neox + if g.rope.dims as usize == HEAD + && g.rope.yarn.is_none() + && g.rope.base.is_finite() + && g.rope.base > 0.0 => + { + true + } + _ => return Err(GpuError::shape(WHAT, "need NoPE or whole-head plain NeoX")), + }; + let window = g.window.map_or(0, |n| n as usize); + if hidden == 0 + || ctx == 0 + || u32::try_from(ctx).is_err() + || kv == 0 + || kv.checked_mul(GROUP) != Some(heads) + || g.head_dim as usize != HEAD + || g.value_dim as usize != HEAD + || !g.qk_norm + || !g.sinks + || g.o_norm + || g.out_gate != OutGate::None + || g.select.is_some() + || g.value_scale.is_some() + || !eps.is_finite() + || eps <= 0.0 + || g.window == Some(0) + { + return Err(GpuError::shape( + WHAT, + format!("unsupported head128 layer {layer}: {g:?}"), + )); + } + // The same capacity owner sizes the plan and the physical allocation. + let capacity = + runtime::stores::Rows::Ring.count(g.window.map(u64::from), ctx as u64) as usize; + let qwidth = heads + .checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "query width overflow"))?; + let kvwidth = kv + .checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "KV width overflow"))?; + Ok(Self { + layer, + hidden, + heads, + kv, + ctx, + capacity, + window, + rotate, + eps, + norm: vector(file, w, names::attn_norm(layer), hidden)?, + qnorm: vector(file, w, names::attn_q_norm(layer), HEAD)?, + knorm: vector(file, w, names::attn_k_norm(layer), HEAD)?, + sinks: vector(file, w, names::attn_sinks(layer), heads)?, + q: Projection::load(file, w, names::attn_q(layer), qwidth, hidden)?, + k: Projection::load(file, w, names::attn_k(layer), kvwidth, hidden)?, + v: Projection::load(file, w, names::attn_v(layer), kvwidth, hidden)?, + o: Projection::load(file, w, names::attn_output(layer), hidden, qwidth)?, + }) + } + + #[must_use] + pub fn capacity(&self) -> usize { + self.capacity + } + #[must_use] + pub fn window(&self) -> usize { + self.window + } + /// Maximum append-then-attend chunk. Plane rows have no overwrite hazard. + #[must_use] + pub fn chunk_rows(&self) -> usize { + if self.capacity == self.ctx { + self.ctx + } else { + self.capacity - self.window + 1 + } + } + #[must_use] + pub fn projection_types(&self) -> [SiteTy; 4] { + [self.q.ty, self.k.ty, self.v.ty, self.o.ty] + } + + /// Full attention sub-layer: RMS, Q/K/V, head RMS and optional NeoX, append, + /// sink-folded attention, O projection, residual. Every scratch cell read is written. + /// + /// # Safety + /// `pos[0..m]` are consecutive absolute positions and `n_keys[t] == pos[t] + 1`. + /// Cache rows for earlier readable positions are initialized. The runtime Store owner + /// has established retention before a rewind; a wrapped ring cannot restore overwritten keys. + #[allow( + clippy::too_many_arguments, + reason = "borrowed layer launch state (rust-quality R8)" + )] + pub unsafe fn enqueue( + &self, + gpu: &Gpu, + w: &Weights, + k: &Kernels, + s: &mut Scratch, + cache: &mut Cache, + x: &DeviceBuffer, + x1: &mut DeviceBuffer, + pos: &DeviceBuffer, + n_keys: &DeviceBuffer, + m: usize, + rope: &RopeRows, + ) -> Result<(), GpuError> { + if m == 0 + || m > s.rows + || m > self.chunk_rows() + || s.hidden != self.hidden + || s.heads != self.heads + || s.kv != self.kv + || cache.capacity != self.capacity + || cache.kv != self.kv + || x.len() < m * self.hidden + || x1.len() < m * self.hidden + || pos.len() < m + || n_keys.len() < m + || (self.rotate && (rope.width != HEAD || rope.rows() < self.ctx)) + { + return Err(GpuError::shape( + WHAT, + format!( + "layer {} incompatible scratch/cache/rows: m={m}", + self.layer + ), + )); + } + let stream = gpu.stream(); + let fault = gpu.layer_sink(self.layer)?; + let wide = WideKernels { + gemm: &k.gemm, + g32: Some(&k.g32), + }; + gpu.elem().enqueue_rms_norm( + stream, + x, + w.f32_buf(WHAT, &self.norm)?, + self.eps, + self.hidden, + m, + &mut s.xn, + )?; + k.gemm.enqueue_route_dense(stream, m, &mut s.route, fault)?; + s.input.quantize( + gpu, + &wide, + fault, + &s.xn, + m, + &[self.q.ty, self.k.ty, self.v.ty], + )?; + self.q + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.q)?; + self.k + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.k)?; + self.v + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.v)?; + let gq = w.f32_buf(WHAT, &self.qnorm)?; + let gk = w.f32_buf(WHAT, &self.knorm)?; + // SAFETY: positions and initialized retained rows are the caller's contract; + // chunk_rows prevents wrapped appends from destroying an earlier row's window. + unsafe { + k.enqueue_append( + stream, + NeoxArgs { + q: &mut s.q, + k: &mut s.k, + v: &s.v, + gq, + gk, + table: &rope.table, + pos, + eps: self.eps, + n_head: self.heads, + n_kv: self.kv, + ctx: self.ctx, + m, + fault, + cache_k: &mut cache.k, + cache_v: &mut cache.v, + }, + self.capacity, + self.rotate, + )?; + } + k.enqueue_attention( + stream, + H128SwaArgs { + q: &s.q, + kc: &cache.k, + vc: &cache.v, + n_keys, + scale: 1.0 / (HEAD as f32).sqrt(), + n_kv: self.kv, + ctx: self.ctx, + m, + window: self.window, + sinks: Some(w.f32_buf(WHAT, &self.sinks)?), + part_v: &mut s.part_v, + part_ms: &mut s.part_ms, + fault, + y: &mut s.attended, + }, + self.heads, + self.capacity, + )?; + s.output + .quantize(gpu, &wide, fault, &s.attended, m, &[self.o.ty])?; + self.o.enqueue( + gpu, + w, + &wide, + &s.output, + &s.attended, + &s.route, + m, + &mut s.delta, + )?; + gpu.elem() + .enqueue_add(stream, x, &s.delta, m * self.hidden, x1) + } +} + +/// Only the physical cache travels with a slot. Logical retention stays in runtime::Store. +pub struct Cache { + pub k: DeviceBuffer, + pub v: DeviceBuffer, + capacity: usize, + kv: usize, +} + +impl Cache { + pub fn new(gpu: &Gpu, binding: &Binding) -> Result { + let n = binding + .kv + .checked_mul(binding.capacity) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "cache allocation overflow"))?; + Ok(Self { + k: DeviceBuffer::zeroed(gpu.stream(), n)?, + v: DeviceBuffer::zeroed(gpu.stream(), n)?, + capacity: binding.capacity, + kv: binding.kv, + }) + } + #[must_use] + pub fn bytes(&self) -> usize { + self.k.num_bytes() + self.v.num_bytes() + } +} + +/// Loaded launch modules. Their module handles allocate no device scratch. +pub struct Kernels { + gemm: GemmKernels, + g32: Gemm32Kernels, + qk: QkRmsKernels, + rope: RopeNeoxKernels, + flash: H128SwaKernels, + ring: attention_h128_ring::Kernels, +} + +impl Kernels { + pub fn load(gpu: &Gpu) -> Result { + Ok(Self { + gemm: GemmKernels::load(gpu.context())?, + g32: Gemm32Kernels::load(gpu.context())?, + qk: QkRmsKernels::load(gpu.context())?, + rope: RopeNeoxKernels::load(gpu.context())?, + flash: H128SwaKernels::load(gpu.context())?, + ring: attention_h128_ring::Kernels::load(gpu.context())?, + }) + } + #[must_use] + pub fn bytes(&self) -> usize { + 0 + } + + /// Head RMS and append, with independent rotation and physical capacity. + /// # Safety + /// Valid absolute positions are distinct modulo `rows`. + pub unsafe fn enqueue_append( + &self, + stream: &cuda_core::CudaStream, + args: NeoxArgs<'_>, + rows: usize, + rotate: bool, + ) -> Result<(), GpuError> { + if rows != args.ctx { + // SAFETY: this method's caller guarantees distinct physical rows. + return unsafe { self.ring.append(stream, args, rows, rotate) }; + } + if rotate { + // SAFETY: distinct modulo the full plane means distinct plane positions. + unsafe { self.rope.enqueue_head_norm_neox_append(stream, args) } + } else { + let NeoxArgs { + q, + k, + v, + gq, + gk, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + .. + } = args; + self.qk.enqueue( + stream, + QkRmsArgs { + q, + k, + v, + gq, + gk, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + }, + ) + } + } + + pub fn enqueue_attention( + &self, + stream: &cuda_core::CudaStream, + args: H128SwaArgs<'_>, + heads: usize, + rows: usize, + ) -> Result<(), GpuError> { + if rows == args.ctx { + self.flash.enqueue(stream, args, heads) + } else { + self.ring.attention(stream, args, heads, rows) + } + } +} + +/// Reused step/prompt storage. No allocation occurs during enqueue. +pub struct Scratch { + rows: usize, + hidden: usize, + heads: usize, + kv: usize, + xn: DeviceBuffer, + q: DeviceBuffer, + k: DeviceBuffer, + v: DeviceBuffer, + attended: DeviceBuffer, + delta: DeviceBuffer, + part_v: DeviceBuffer, + part_ms: DeviceBuffer, + input: WideActs, + output: WideActs, + route: GemmRoute, +} + +fn layout( + bindings: &[Binding], + rows: usize, +) -> Result<(usize, usize, usize, usize, WideForms, WideForms), GpuError> { + let first = bindings + .first() + .ok_or_else(|| GpuError::shape(WHAT, "at least one binding required"))?; + if !(1..=GEMM_MAX_SLOTS).contains(&rows) + || bindings + .iter() + .any(|b| b.hidden != first.hidden || b.heads != first.heads || b.kv != first.kv) + { + return Err(GpuError::shape( + WHAT, + format!("scratch needs 1..={GEMM_MAX_SLOTS} rows and equal layer widths"), + )); + } + let mut input = WideForms::NONE; + let mut output = WideForms::NONE; + let mut segs = 1; + for b in bindings { + input = input.union(WideActs::wants_of(&[b.q.ty, b.k.ty, b.v.ty])); + output = output.union(WideActs::wants(b.o.ty)); + segs = segs.max(window_segments(b.window)); + } + // The f32 source buffers are already owned above; no duplicate form is needed. + input.f32 = false; + output.f32 = false; + Ok((first.hidden, first.heads, first.kv, segs, input, output)) +} + +impl Scratch { + pub fn new(gpu: &Gpu, bindings: &[Binding], rows: usize) -> Result { + let (hidden, heads, kv, segs, input, output) = layout(bindings, rows)?; + let _ = Self::planned_bytes(bindings, rows)?; + let stream = gpu.stream(); + Ok(Self { + rows, + hidden, + heads, + kv, + xn: DeviceBuffer::zeroed(stream, rows * hidden)?, + q: DeviceBuffer::zeroed(stream, rows * heads * HEAD)?, + k: DeviceBuffer::zeroed(stream, rows * kv * HEAD)?, + v: DeviceBuffer::zeroed(stream, rows * kv * HEAD)?, + attended: DeviceBuffer::zeroed(stream, rows * heads * HEAD)?, + delta: DeviceBuffer::zeroed(stream, rows * hidden)?, + part_v: DeviceBuffer::zeroed(stream, rows * heads * segs * HEAD)?, + part_ms: DeviceBuffer::zeroed(stream, rows * heads * segs * 2)?, + input: WideActs::new(stream, input, rows, hidden)?, + output: WideActs::new(stream, output, rows, heads * HEAD)?, + route: GemmRoute::new(stream, rows, 1)?, + }) + } + /// Exact device allocation, including typed activation forms and the dense route table. + pub fn planned_bytes(bindings: &[Binding], rows: usize) -> Result { + let (hidden, heads, kv, segs, input, output) = layout(bindings, rows)?; + let shape = Head128Scratch { + hidden: hidden as u64, + query: (heads * HEAD) as u64, + key_value: (kv * HEAD) as u64, + heads: heads as u64, + segments: segs as u64, + input: Forms { + q128: input.q128, + q32: input.q32, + }, + output: Forms { + q128: output.q128, + q32: output.q32, + }, + }; + shape + .bytes(rows as u64) + .and_then(|n| usize::try_from(n).ok()) + .ok_or_else(|| GpuError::shape(WHAT, "scratch allocation overflow")) + } + #[must_use] + pub fn bytes(&self) -> usize { + [ + &self.xn, + &self.q, + &self.k, + &self.v, + &self.attended, + &self.delta, + &self.part_v, + &self.part_ms, + ] + .iter() + .map(|b| b.num_bytes()) + .sum::() + + self.input.bytes() + + self.output.bytes() + + self.route.bytes() + } +} diff --git a/crates/gpu/src/attention_h128_ring.rs b/crates/gpu/src/attention_h128_ring.rs new file mode 100644 index 000000000..bda87f776 --- /dev/null +++ b/crates/gpu/src/attention_h128_ring.rs @@ -0,0 +1,880 @@ +//! Ring-addressed head-128 primitives. Logical positions index RoPE; only KV addresses wrap. + +use crate::GpuError; +use crate::elem::rms_scale_of_mean_checked; +use crate::fault::{FaultSink, FaultSite}; +use crate::flash::f32_to_f16_bits; +use crate::flash::{MERGE_BATCH, half_bits_to_f32, online_fold}; +use crate::flash_gqa::{ + GROUP, HEAD, KEY_TILE, SEG_KEYS, SEGMENTS, seg_span, window_cut, window_segments, +}; +use crate::flash_gqa_h128_swa::{H128SwaArgs, fold_tile}; +use crate::launch_u32; +use crate::rope_neox::NeoxArgs; +use crate::rope_neox::neox_pair; +use cuda_core::{CudaContext, CudaStream, LaunchConfig1D}; +use cuda_device::{ + DisjointSlice, SharedArray, kernel, launch_bounds, launch_contract, thread, warp, +}; +use cuda_host::cuda_module; +use std::sync::Arc; + +/// u32 words of one key row, and the padded stride the key tile is staged at. +const ROW_WORDS: usize = HEAD / 2; +const K_STRIDE: usize = ROW_WORDS + 1; +/// u64 words of one value row. +const ROW_QWORDS: usize = HEAD / 4; +const ILP: usize = 4; +const THREADS: usize = GROUP * 32; +const THREADS_U32: u32 = THREADS as u32; +const MERGE_THREADS: u32 = HEAD as u32; +const _: () = assert!(THREADS_U32 as usize == THREADS && MERGE_THREADS as usize == HEAD); +const _: () = assert!(ROW_QWORDS == 32 && ROW_WORDS.is_multiple_of(ILP) && K_STRIDE % 2 == 1); +// The launch's window: 513 positions is nine 64-key segments, and a row of +// 600 live keys starts at key 87. +const _: () = { + assert!(window_segments(0) == SEGMENTS && window_segments(513) == 9); + let (first, n_w) = window_cut(600, 513); + assert!(first == 87 && n_w == 513 && first + n_w == 600); +}; + +#[inline(always)] +fn live_keys(count: u32, ctx: usize) -> usize { + let c = count as usize; + if c > ctx { 0 } else { c } +} + +#[cuda_module] +mod append_kernels { + use super::*; + const THREADS: usize = 64; + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(64)] + #[launch_contract( + domain = 1, + block = (64, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + q.len() >= m * n_head * 128, + k.len() >= m * n_kv * 128, + v.len() >= m * n_kv * 128, + gq.len() >= 128, + gk.len() >= 128, + pos.len() >= m, + cache_k.len() >= n_kv * rows * 128, + cache_v.len() >= n_kv * rows * 128 + ) + )] + pub fn qk_rms_append_ring( + gq: &[f32], + gk: &[f32], + pos: &[u32], + v: &[f32], + eps: f32, + n_head: u32, + n_kv: u32, + ctx: u32, + rows: u32, + m: u32, + fault: FaultSink, + mut q: DisjointSlice, + mut k: DisjointSlice, + mut cache_k: DisjointSlice, + mut cache_v: DisjointSlice, + ) { + static mut WSUM: SharedArray = SharedArray::UNINIT; + + let heads = (n_head + n_kv) as usize; + let b = thread::blockIdx_x() as usize; + if b >= m as usize * heads { + return; // block-uniform + } + let t = b / heads; + let h = b - t * heads; + let tid = thread::threadIdx_x() as usize; + let is_q = h < n_head as usize; + let kh = h.wrapping_sub(n_head as usize); + let base = if is_q { + (t * n_head as usize + h) * HEAD + } else { + (t * n_kv as usize + kh) * HEAD + }; + // SAFETY: t < m <= pos.len() by the launch contract. + let p = unsafe { *pos.get_unchecked(t) } as usize; + + // SAFETY: base + tid + 64 is inside the head's buffer by the launch + // contract. One thread per value pair. + let (x0, x1) = unsafe { + if is_q { + ( + *q.get_unchecked_mut(base + tid), + *q.get_unchecked_mut(base + tid + THREADS), + ) + } else { + ( + *k.get_unchecked_mut(base + tid), + *k.get_unchecked_mut(base + tid + THREADS), + ) + } + }; + let mut acc = f64::from(x0 * x0) + f64::from(x1 * x1); + acc += warp::shuffle_xor_f64(acc, 16); + acc += warp::shuffle_xor_f64(acc, 8); + acc += warp::shuffle_xor_f64(acc, 4); + acc += warp::shuffle_xor_f64(acc, 2); + acc += warp::shuffle_xor_f64(acc, 1); + // SAFETY: WSUM is this block's own shared allocation. Two slots, one + // per warp, written before the barrier that publishes them. + let ws = unsafe { SharedArray::as_raw_mut_ptr(&raw mut WSUM) }; + if warp::lane_id() == 0 { + // SAFETY: tid / 32 < 2. One lane per warp writes its slot. + unsafe { *ws.add(tid / 32) = acc }; + } + thread::sync_threads(); + // SAFETY: both slots were written before the barrier above. + let sum = unsafe { *ws.add(0) + *ws.add(1) }; + let mean = (sum / HEAD as f64) as f32; + let scale = rms_scale_of_mean_checked(mean, eps, fault, tid == 0); + + if p >= ctx as usize { + if tid == 0 { + fault.raise(FaultSite::CachePos); + } + // SAFETY: the positions read above. This thread owns them. + unsafe { + if is_q { + *q.get_unchecked_mut(base + tid) = f32::NAN; + *q.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } else { + *k.get_unchecked_mut(base + tid) = f32::NAN; + *k.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } + } + return; // block-uniform: p is the token's + } + // SAFETY: tid + 64 < 128 <= the gain's length. + let (g0, g1) = unsafe { + let g = if is_q { gq } else { gk }; + (*g.get_unchecked(tid), *g.get_unchecked(tid + THREADS)) + }; + let n0 = (scale * g0) * x0; + let n1 = (scale * g1) * x1; + + if is_q { + // SAFETY: the positions read above. This thread owns them. + unsafe { + *q.get_unchecked_mut(base + tid) = n0; + *q.get_unchecked_mut(base + tid + THREADS) = n1; + } + return; + } + // SAFETY: the positions read above. This thread owns them. + unsafe { + *k.get_unchecked_mut(base + tid) = n0; + *k.get_unchecked_mut(base + tid + THREADS) = n1; + } + let row = (kh * rows as usize + p % rows as usize) * HEAD; + // SAFETY: kh < n_kv and p % rows < rows, so the row is inside both rings. + // base + tid + 64 is inside v. One thread per pair. The tokens of + // one launch hold distinct positions, so no two blocks write one row. + unsafe { + *cache_k.get_unchecked_mut(row + tid) = f32_to_f16_bits(n0); + *cache_k.get_unchecked_mut(row + tid + THREADS) = f32_to_f16_bits(n1); + *cache_v.get_unchecked_mut(row + tid) = f32_to_f16_bits(*v.get_unchecked(base + tid)); + *cache_v.get_unchecked_mut(row + tid + THREADS) = + f32_to_f16_bits(*v.get_unchecked(base + tid + THREADS)); + } + } + + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(64)] + #[launch_contract( + domain = 1, + block = (64, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + q.len() >= m * n_head * 128, + k.len() >= m * n_kv * 128, + v.len() >= m * n_kv * 128, + gq.len() >= 128, + gk.len() >= 128, + table.len() >= ctx * 128, + pos.len() >= m, + cache_k.len() >= n_kv * rows * 128, + cache_v.len() >= n_kv * rows * 128 + ) + )] + pub fn head_norm_neox_append_ring( + gq: &[f32], + gk: &[f32], + table: &[f32], + pos: &[u32], + v: &[f32], + eps: f32, + n_head: u32, + n_kv: u32, + ctx: u32, + rows: u32, + m: u32, + fault: FaultSink, + mut q: DisjointSlice, + mut k: DisjointSlice, + mut cache_k: DisjointSlice, + mut cache_v: DisjointSlice, + ) { + static mut WSUM: SharedArray = SharedArray::UNINIT; + + let heads = (n_head + n_kv) as usize; + let b = thread::blockIdx_x() as usize; + if b >= m as usize * heads { + return; // block-uniform + } + let t = b / heads; + let h = b - t * heads; + let tid = thread::threadIdx_x() as usize; + let is_q = h < n_head as usize; + let kh = h.wrapping_sub(n_head as usize); + let base = if is_q { + (t * n_head as usize + h) * HEAD + } else { + (t * n_kv as usize + kh) * HEAD + }; + // SAFETY: t < m <= pos.len() by the launch contract. + let p = unsafe { *pos.get_unchecked(t) } as usize; + + // SAFETY: base + tid + 64 < (t·n + h + 1)·128 <= m·n·128, inside the + // head's buffer by the launch contract; one thread per value pair. + let (x0, x1) = unsafe { + if is_q { + ( + *q.get_unchecked_mut(base + tid), + *q.get_unchecked_mut(base + tid + THREADS), + ) + } else { + ( + *k.get_unchecked_mut(base + tid), + *k.get_unchecked_mut(base + tid + THREADS), + ) + } + }; + let mut acc = f64::from(x0 * x0) + f64::from(x1 * x1); + acc += warp::shuffle_xor_f64(acc, 16); + acc += warp::shuffle_xor_f64(acc, 8); + acc += warp::shuffle_xor_f64(acc, 4); + acc += warp::shuffle_xor_f64(acc, 2); + acc += warp::shuffle_xor_f64(acc, 1); + // SAFETY: WSUM is this block's own shared allocation; the raw form is + // the only way to reach it without a reference to a `static mut`. + // Two slots, one per warp, written before the barrier that publishes + // them. + let ws = unsafe { SharedArray::as_raw_mut_ptr(&raw mut WSUM) }; + if warp::lane_id() == 0 { + // SAFETY: tid / 32 < 2; one lane per warp writes its slot. + unsafe { *ws.add(tid / 32) = acc }; + } + thread::sync_threads(); + // SAFETY: both slots were written before the barrier above. + let sum = unsafe { *ws.add(0) + *ws.add(1) }; + let mean = (sum / HEAD as f64) as f32; + let scale = rms_scale_of_mean_checked(mean, eps, fault, tid == 0); + + if p >= ctx as usize { + if tid == 0 { + fault.raise(FaultSite::CachePos); + } + // SAFETY: the positions read above; this thread owns them. + unsafe { + if is_q { + *q.get_unchecked_mut(base + tid) = f32::NAN; + *q.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } else { + *k.get_unchecked_mut(base + tid) = f32::NAN; + *k.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } + } + return; // block-uniform: p is the token's + } + // SAFETY: tid + 64 < 128 <= the gain's length; p < ctx and 2·tid + 1 + // < 128, so the table pair is inside row p of the table's ctx rows. + let (g0, g1, c, s) = unsafe { + let g = if is_q { gq } else { gk }; + ( + *g.get_unchecked(tid), + *g.get_unchecked(tid + THREADS), + *table.get_unchecked(p * HEAD + 2 * tid), + *table.get_unchecked(p * HEAD + 2 * tid + 1), + ) + }; + let n0 = (scale * g0) * x0; + let n1 = (scale * g1) * x1; + let (y0, y1) = neox_pair(n0, n1, c, s); + + if is_q { + // SAFETY: the positions read above; this thread owns them. + unsafe { + *q.get_unchecked_mut(base + tid) = y0; + *q.get_unchecked_mut(base + tid + THREADS) = y1; + } + return; + } + // SAFETY: the positions read above; this thread owns them. + unsafe { + *k.get_unchecked_mut(base + tid) = y0; + *k.get_unchecked_mut(base + tid + THREADS) = y1; + } + let row = (kh * rows as usize + p % rows as usize) * HEAD; + // SAFETY: kh < n_kv and p % rows < rows, so row + 127 < n_kv·rows·128 <= the + // planes' lengths; base + tid + 64 < m·n_kv·128 <= v.len(). One + // thread per pair of the row; the tokens of one launch hold distinct + // positions (the launch contract; the engine's embedding launch + // writes consecutive ones), so no two blocks write one row. + unsafe { + *cache_k.get_unchecked_mut(row + tid) = f32_to_f16_bits(y0); + *cache_k.get_unchecked_mut(row + tid + THREADS) = f32_to_f16_bits(y1); + *cache_v.get_unchecked_mut(row + tid) = f32_to_f16_bits(*v.get_unchecked(base + tid)); + *cache_v.get_unchecked_mut(row + tid + THREADS) = + f32_to_f16_bits(*v.get_unchecked(base + tid + THREADS)); + } + } +} + +#[cuda_module] +mod flash_kernels { + use super::*; + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + n_keys_buf.len() >= m, + segs >= 1, + q.len() >= m * n_kv * 8 * 128, + kc.len() >= n_kv * rows * 128, + vc.len() >= n_kv * rows * 128, + part_v.len() >= m * n_kv * 8 * segs * 128, + part_ms.len() >= m * n_kv * 8 * segs * 2 + ) + )] + pub fn gqa_flash_seg_h128_swa_ring( + q: &[f32], + kc: &[u16], + vc: &[u16], + n_keys_buf: &[u32], + scale: f32, + n_kv: u32, + ctx: u32, + segs: u32, + seg_keys: u32, + rows: u32, + m: u32, + window: u32, + mut part_v: DisjointSlice, + mut part_ms: DisjointSlice, + ) { + static mut QS: SharedArray = SharedArray::UNINIT; + static mut KS: SharedArray = SharedArray::UNINIT; + static mut VS: SharedArray = SharedArray::UNINIT; + static mut WS: SharedArray = SharedArray::UNINIT; + + let b = thread::blockIdx_x() as usize; + let nkv = n_kv as usize; + let n_seg = segs as usize; + let tokens = m as usize; + if b >= tokens * nkv * n_seg { + return; // block-uniform + } + let t = b % tokens; + let sk = b / tokens; + let seg = sk / nkv; + let kh = sk - seg * nkv; + let tid = thread::threadIdx_x() as usize; + let w = tid / 32; + let lane = warp::lane_id() as usize; + let n_head = nkv * GROUP; + let h = kh * GROUP + w; + let idx = (t * n_head + h) * n_seg + seg; + let ctx = ctx as usize; + // SAFETY: t < m <= n_keys_buf.len() by the launch contract. + let limit = live_keys(unsafe { *n_keys_buf.get_unchecked(t) }, ctx); + let (first, n_w) = window_cut(limit, window as usize); + let span = seg_span(n_w, n_seg, seg_keys as usize); + let lo = first + seg * span; + if lo >= first + n_w { + if lane == 0 { + // SAFETY: idx < m·n_kv·GROUP·segs, both slots inside part_ms. + unsafe { + *part_ms.get_unchecked_mut(2 * idx) = f32::NEG_INFINITY; + *part_ms.get_unchecked_mut(2 * idx + 1) = 0.0; + } + } + return; // block-uniform: the window cut is the block's + } + let hi = (lo + span).min(first + n_w); + + // SAFETY: each `static mut` is this block's own shared allocation. + // The raw form reaches it without a reference. Every index below is + // inside its array, and every write is ordered before its reads by a + // block barrier (or, for the weights, by the warp sync in `fold_tile`). + let (qs, ks, vs, ws) = unsafe { + ( + SharedArray::as_raw_mut_ptr(&raw mut QS), + SharedArray::as_raw_mut_ptr(&raw mut KS), + SharedArray::as_raw_mut_ptr(&raw mut VS), + SharedArray::as_raw_mut_ptr(&raw mut WS), + ) + }; + + let qb = (t * n_head + kh * GROUP) * HEAD; + let mut i = 0usize; + while i < 4 { + // SAFETY: qb + 4·tid + i is inside row t's group, and that row is + // inside q by the launch contract. The shared index is inside + // GROUP·HEAD. + unsafe { *qs.add(4 * tid + i) = *q.get_unchecked(qb + 4 * tid + i) }; + i += 1; + } + + let plane = kh * rows as usize * HEAD; + let k64 = kc.as_ptr() as *const u64; + let v64 = vc.as_ptr() as *const u64; + let mut mx = f32::NEG_INFINITY; + let mut s_sum = 0.0f32; + let mut acc = [0.0f32; 4]; + let mut t0 = lo; + while t0 < hi { + thread::sync_threads(); + let mut i = 0usize; + while i < 4 { + let e = tid + THREADS * i; + let key = e / ROW_QWORDS; + let word = e - key * ROW_QWORDS; + let (kw, vw) = if t0 + key < hi { + // SAFETY: t0 + key < hi <= limit <= ctx, so the row is + // mapped modulo rows inside key head kh's ring. A row is 32 u64 and the + // planes start 8-byte aligned (a device allocation). + unsafe { + let r = (plane + ((t0 + key) % rows as usize) * HEAD) / 4 + word; + (*k64.add(r), *v64.add(r)) + } + } else { + (0u64, 0u64) + }; + // SAFETY: key < 32 and word < 32, inside the staged tiles. + unsafe { + *ks.add(key * K_STRIDE + 2 * word) = kw as u32; + *ks.add(key * K_STRIDE + 2 * word + 1) = (kw >> 32) as u32; + *vs.add(key * ROW_QWORDS + word) = vw; + } + i += 1; + } + thread::sync_threads(); + + let live = t0 + lane < hi; + let mut a = [0.0f32; ILP]; + let mut wd = 0usize; + while wd < ROW_WORDS { + let mut p = 0usize; + while p < ILP { + // SAFETY: lane < 32 and wd + p < ROW_WORDS, inside the + // staged key tile. The query index is inside GROUP·HEAD. + let (kw, q0, q1) = unsafe { + ( + *ks.add(lane * K_STRIDE + wd + p), + *qs.add(w * HEAD + 2 * (wd + p)), + *qs.add(w * HEAD + 2 * (wd + p) + 1), + ) + }; + let k0 = half_bits_to_f32(kw as u16); + let k1 = half_bits_to_f32((kw >> 16) as u16); + a[p] = f32::mul_add(q0, k0, a[p]); + a[p] = f32::mul_add(q1, k1, a[p]); + p += 1; + } + wd += ILP; + } + let dot = (a[0] + a[1]) + (a[2] + a[3]); + let sc = if live { dot * scale } else { f32::NEG_INFINITY }; + + // SAFETY: the weight and value tiles are this block's, the value + // tile was published by the barrier above, and w < GROUP. + unsafe { fold_tile(sc, live, &mut mx, &mut s_sum, &mut acc, ws, vs, w, lane) }; + t0 += KEY_TILE; + } + + // SAFETY: idx is inside the partials. The four dims 4·lane .. +3 are + // this lane's alone, and lane 0 writes (max, sum). + unsafe { + let o = idx * HEAD + 4 * lane; + *part_v.get_unchecked_mut(o) = acc[0]; + *part_v.get_unchecked_mut(o + 1) = acc[1]; + *part_v.get_unchecked_mut(o + 2) = acc[2]; + *part_v.get_unchecked_mut(o + 3) = acc[3]; + if lane == 0 { + *part_ms.get_unchecked_mut(2 * idx) = mx; + *part_ms.get_unchecked_mut(2 * idx + 1) = s_sum; + } + } + } + + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + n_keys_buf.len() >= m, + segs >= 1, + sinks.len() >= n_head, + part_v.len() >= m * n_head * segs * 128, + part_ms.len() >= m * n_head * segs * 2, + y.len() >= m * n_head * 128 + ) + )] + pub fn gqa_flash_merge_h128_swa_ring( + part_v: &[f32], + part_ms: &[f32], + n_keys_buf: &[u32], + sinks: &[f32], + n_head: u32, + ctx: u32, + segs: u32, + seg_keys: u32, + m: u32, + window: u32, + fault: FaultSink, + mut y: DisjointSlice, + ) { + let b = thread::blockIdx_x() as usize; + let nh = n_head as usize; + if b >= m as usize * nh { + return; // block-uniform + } + let t = b / nh; + let h = b - t * nh; + let d = thread::threadIdx_x() as usize; + let n_seg = segs as usize; + // SAFETY: t < m <= n_keys_buf.len() by the launch contract. + let limit = live_keys(unsafe { *n_keys_buf.get_unchecked(t) }, ctx as usize); + if limit == 0 { + if d == 0 { + fault.raise(FaultSite::KeyCount); + } + // SAFETY: b·HEAD + d is inside y; thread d owns it. + unsafe { *y.get_unchecked_mut(b * HEAD + d) = f32::NAN }; + return; // block-uniform: the row is the block's + } + let (_, n_w) = window_cut(limit, window as usize); + let live = n_w.div_ceil(seg_span(n_w, n_seg, seg_keys as usize)); + let mut mx = f32::NEG_INFINITY; + let mut s = 0.0f32; + let mut acc = 0.0f32; + let mut j = 0usize; + while j < live { + let mut ms = [0.0f32; 2 * MERGE_BATCH]; + let mut vs = [0.0f32; MERGE_BATCH]; + for i in 0..MERGE_BATCH { + cuda_device::thread::__unroll_config::<0>(); + if j + i < live { + let idx = b * n_seg + j + i; + // SAFETY: idx is inside the row's segments, so both + // partials and the value dim are inside the buffers. + unsafe { + ms[2 * i] = *part_ms.get_unchecked(2 * idx); + ms[2 * i + 1] = *part_ms.get_unchecked(2 * idx + 1); + vs[i] = *part_v.get_unchecked(idx * HEAD + d); + } + } + } + for i in 0..MERGE_BATCH { + cuda_device::thread::__unroll_config::<0>(); + if j + i < live && ms[2 * i + 1] != 0.0 { + (mx, s, acc) = online_fold(mx, s, acc, ms[2 * i], ms[2 * i + 1], vs[i]); + } + } + j += MERGE_BATCH; + } + // SAFETY: h < n_head <= sinks.len() by the launch contract. + let sink = unsafe { *sinks.get_unchecked(h) }; + // The sink joins last, after every live segment. + let (_, s, acc) = online_fold(mx, s, acc, sink, 1.0, 0.0); + let out = if sink.is_nan() || sink == f32::INFINITY { + f32::NAN + } else { + acc * (1.0 / s) + }; + // SAFETY: b·HEAD + d is inside y; thread d owns it. + unsafe { *y.get_unchecked_mut(b * HEAD + d) = out }; + } +} + +/// New ring entries; legacy plane entries stay in their original modules. +pub(crate) struct Kernels { + append: append_kernels::LoadedModule, + flash: flash_kernels::LoadedModule, +} + +impl Kernels { + pub(crate) fn load(ctx: &Arc) -> Result { + // SAFETY: both embedded modules belong to this crate; every launch is checked below. + let append = unsafe { crate::shared_module!(append_kernels, ctx)? }; + // SAFETY: this crate owns this embedded module too. + let flash = unsafe { crate::shared_module!(flash_kernels, ctx)? }; + Ok(Self { append, flash }) + } + + /// # Safety + /// Valid positions are distinct modulo `rows`. + pub(crate) unsafe fn append( + &self, + stream: &CudaStream, + args: NeoxArgs<'_>, + rows: usize, + rotate: bool, + ) -> Result<(), GpuError> { + const WHAT: &str = "attention_h128::ring_append"; + let NeoxArgs { + q, + k, + v, + gq, + gk, + table, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + } = args; + if n_head == 0 || n_kv == 0 || ctx == 0 || rows == 0 || rows > ctx || m == 0 || m > rows { + return Err(GpuError::shape( + WHAT, + format!( + "need heads/kv >=1, 1<=rows<=ctx and 1<=m<=rows: heads={n_head} kv={n_kv} ctx={ctx} rows={rows} m={m}" + ), + )); + } + let plane = n_kv + .checked_mul(rows) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "cache shape overflow"))?; + let q_len = m + .checked_mul(n_head) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "query shape overflow"))?; + let kv_len = m + .checked_mul(n_kv) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "key shape overflow"))?; + for (name, got, need) in [ + ("q", q.len(), q_len), + ("k", k.len(), kv_len), + ("v", v.len(), kv_len), + ("gq", gq.len(), HEAD), + ("gk", gk.len(), HEAD), + ("pos", pos.len(), m), + ("cache_k", cache_k.len(), plane), + ("cache_v", cache_v.len(), plane), + ( + "table", + table.len(), + if rotate { + ctx.checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "table shape overflow"))? + } else { + 0 + }, + ), + ] { + if got < need { + return Err(GpuError::shape( + WHAT, + format!("{name}.len() {got} < {need}"), + )); + } + } + let grid = launch_u32( + WHAT, + "grid", + m.checked_mul( + n_head + .checked_add(n_kv) + .ok_or_else(|| GpuError::shape(WHAT, "head count overflow"))?, + ) + .ok_or_else(|| GpuError::shape(WHAT, "grid overflow"))?, + )?; + let nh = launch_u32(WHAT, "heads", n_head)?; + let nkv = launch_u32(WHAT, "kv", n_kv)?; + let ctx = launch_u32(WHAT, "ctx", ctx)?; + let rows = launch_u32(WHAT, "rows", rows)?; + let m = launch_u32(WHAT, "m", m)?; + if rotate { + let prep = self + .append + .prepare_head_norm_neox_append_ring(LaunchConfig1D::new(grid, 64, 0))?; + self.append.head_norm_neox_append_ring( + stream, &prep, gq, gk, table, pos, v, eps, nh, nkv, ctx, rows, m, fault, q, k, + cache_k, cache_v, + )?; + } else { + let prep = self + .append + .prepare_qk_rms_append_ring(LaunchConfig1D::new(grid, 64, 0))?; + self.append.qk_rms_append_ring( + stream, &prep, gq, gk, pos, v, eps, nh, nkv, ctx, rows, m, fault, q, k, cache_k, + cache_v, + )?; + } + Ok(()) + } + + pub(crate) fn attention( + &self, + stream: &CudaStream, + args: H128SwaArgs<'_>, + n_head: usize, + rows: usize, + ) -> Result<(), GpuError> { + const WHAT: &str = "attention_h128::ring_attention"; + let H128SwaArgs { + q, + kc, + vc, + n_keys, + scale, + n_kv, + ctx, + m, + window, + sinks, + part_v, + part_ms, + fault, + y, + } = args; + if n_kv == 0 + || ctx == 0 + || rows == 0 + || rows > ctx + || m == 0 + || n_kv.checked_mul(GROUP) != Some(n_head) + || !scale.is_finite() + || window == 0 + || window > rows + || m > rows - window + 1 + { + return Err(GpuError::shape( + WHAT, + format!( + "invalid ring geometry: heads={n_head} kv={n_kv} ctx={ctx} rows={rows} m={m} window={window} scale={scale}" + ), + )); + } + let sinks = sinks.ok_or_else(|| GpuError::shape(WHAT, "per-head sinks required"))?; + let segs = window_segments(window); + let heads = m + .checked_mul(n_head) + .ok_or_else(|| GpuError::shape(WHAT, "heads overflow"))?; + let partials = heads + .checked_mul(segs) + .ok_or_else(|| GpuError::shape(WHAT, "partials overflow"))?; + for (name, got, need) in [ + ("q", q.len(), heads.checked_mul(HEAD)), + ( + "kc", + kc.len(), + n_kv.checked_mul(rows).and_then(|n| n.checked_mul(HEAD)), + ), + ( + "vc", + vc.len(), + n_kv.checked_mul(rows).and_then(|n| n.checked_mul(HEAD)), + ), + ("n_keys", n_keys.len(), Some(m)), + ("sinks", sinks.len(), Some(n_head)), + ("part_v", part_v.len(), partials.checked_mul(HEAD)), + ("part_ms", part_ms.len(), partials.checked_mul(2)), + ("y", y.len(), heads.checked_mul(HEAD)), + ] { + let need = + need.ok_or_else(|| GpuError::shape(WHAT, format!("{name} shape overflow")))?; + if got < need { + return Err(GpuError::shape( + WHAT, + format!("{name}.len() {got} < {need}"), + )); + } + } + let grid = launch_u32( + WHAT, + "grid", + m.checked_mul(n_kv) + .and_then(|n| n.checked_mul(segs)) + .ok_or_else(|| GpuError::shape(WHAT, "grid overflow"))?, + )?; + let nkv = launch_u32(WHAT, "kv", n_kv)?; + let ctx = launch_u32(WHAT, "ctx", ctx)?; + let rows = launch_u32(WHAT, "rows", rows)?; + let ns = launch_u32(WHAT, "segments", segs)?; + let sk = launch_u32(WHAT, "segment keys", SEG_KEYS)?; + let mu = launch_u32(WHAT, "m", m)?; + let window = launch_u32(WHAT, "window", window)?; + let nh = launch_u32(WHAT, "heads", n_head)?; + let prep = self + .flash + .prepare_gqa_flash_seg_h128_swa_ring(LaunchConfig1D::new(grid, THREADS_U32, 0))?; + self.flash.gqa_flash_seg_h128_swa_ring( + stream, + &prep, + q, + kc, + vc, + n_keys, + scale, + nkv, + ctx, + ns, + sk, + rows, + mu, + window, + &mut *part_v, + &mut *part_ms, + )?; + let prep = self + .flash + .prepare_gqa_flash_merge_h128_swa_ring(LaunchConfig1D::new( + launch_u32(WHAT, "merge grid", heads)?, + MERGE_THREADS, + 0, + ))?; + self.flash.gqa_flash_merge_h128_swa_ring( + stream, &prep, part_v, part_ms, n_keys, sinks, nh, ctx, ns, sk, mu, window, fault, y, + )?; + Ok(()) + } +} diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 468a422bd..5811138e5 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -47,6 +47,7 @@ use model::placement::kernels::{self, DownForms}; use model::arch::place_hybrid::RingForms; use model::placement::kernels::DownAct; +use crate::card_leg::ExpertInput; use crate::card_leg::{ self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, }; @@ -86,6 +87,7 @@ struct CardKernels { leg: CardLeg, places: CardSlotKernels, arms: LegArms, + add_base: bool, } impl CardFamily { @@ -173,6 +175,12 @@ impl CardBatch { &self.bufs.pre } + /// The raw routed sum, before shared experts and residuals. + #[must_use] + pub fn acc(&self) -> &DeviceBuffer { + &self.bufs.acc + } + /// The buffers a tier layer's join reads and writes over the batch. pub fn join(&mut self) -> TierJoin<'_> { self.bufs.join(&self.sel) @@ -217,22 +225,6 @@ impl CardLayer { } } -/// The form the down stack of `layer`, of type `ty`, reads ([`kernels::down`]); -/// a type the table names no down entry for is refused by name (`what`). -fn down_act( - what: &'static str, - layer: &CardLayer, - ty: gguf::GgmlType, -) -> Result { - kernels::down(ty) - .map(|(_, act)| act) - .ok_or_else(|| GpuError::Tensor { - what, - name: layer.names[2].clone(), - need: "a down stack of a type the kernel table names a down entry for", - }) -} - /// The forms a load's leg buffers hold: the forms `downs` the card layers' /// downs read, or the 128-value [`DownAct::Q8Act`] on a load with no card /// layer (a tier card's join reads only the buffers' down rows). @@ -279,6 +271,7 @@ pub struct CardExperts { n: usize, /// The union of the resident layers' selected down activation forms. downs: DownForms, + expert_input: ExpertInput, /// The one-token step's buffers. step: CardBufs, /// The buffers of a pass of several rows, on a load that runs one. @@ -392,12 +385,14 @@ impl CardExperts { leg: CardLeg::load(gpu)?, places: CardSlotKernels::load(gpu.context())?, arms, + add_base: true, }, layers: per, tier: tier_k, ff, n, downs, + expert_input: ExpertInput::Direct, step: CardBufs::new(gpu.stream(), &family, (n, ff, 1), downs)?, pass: (rows > 1) .then(|| CardBufs::new(gpu.stream(), &family, (n, ff, rows), downs)) @@ -405,6 +400,21 @@ impl CardExperts { }) } + /// Select the loaded down-input policy before making prompt batches or captures. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + self.step.leg.set_expert_input(gpu, policy)?; + if let Some(pass) = &mut self.pass { + pass.leg.set_expert_input(gpu, policy)?; + } + self.expert_input = policy; + Ok(()) + } + + /// Leave only the raw routed sum when the caller owns residual association. + pub fn set_add_base(&mut self, add_base: bool) { + self.k.add_base = add_base; + } + /// Layer `l` computes experts on the card. #[must_use] pub fn has(&self, l: usize) -> bool { @@ -460,6 +470,12 @@ impl CardExperts { &self.step.pre } + /// The raw routed sum, before shared experts and residuals. + #[must_use] + pub fn acc(&self) -> &DeviceBuffer { + &self.step.acc + } + /// The buffers a tier layer's join reads and writes over the step's row, /// the slots' places `sel` as the handoff wrote them. pub fn step_join<'a>(&'a mut self, sel: &'a DeviceBuffer) -> TierJoin<'a> { @@ -475,6 +491,14 @@ impl CardExperts { }) } + /// The raw routed sum of the last pass. + pub fn pass_acc(&self) -> Result<&DeviceBuffer, GpuError> { + self.pass.as_ref().map(|p| &p.acc).ok_or(GpuError::State { + what: self.family.what, + missing: "the buffers of a pass of several rows", + }) + } + /// Device bytes of the step's buffers and a pass's. #[must_use] pub fn bytes(&self) -> usize { @@ -485,8 +509,10 @@ impl CardExperts { /// first-prompt only. pub fn batch(&self, gpu: &Gpu, cap: usize) -> Result { let stream = gpu.stream(); + let mut bufs = CardBufs::new(stream, &self.family, (self.n, self.ff, cap), self.downs)?; + bufs.leg.set_expert_input(gpu, self.expert_input)?; Ok(CardBatch { - bufs: CardBufs::new(stream, &self.family, (self.n, self.ff, cap), self.downs)?, + bufs, sel: DeviceBuffer::zeroed(stream, cap * self.family.n_used)?, ring: None, }) @@ -518,28 +544,31 @@ impl CardExperts { )); } let shape = (cb.bufs.cap(), self.family.n_used, self.n, self.ff); - let ring = RingGemm::new(gpu, shape, half_slots, RingForms::BLOCKS32)?; + let mut ring = RingGemm::new(gpu, shape, half_slots, RingForms::BLOCKS32)?; + ring.set_expert_input(gpu, self.expert_input)?; cb.ring = Some(ring); Ok(()) } - /// Device bytes of [`CardExperts::batch`]'s buffers for `cap` tokens - /// ([`CardBatch::bytes`]'s formula): the card leg's in the card layers' - /// down forms ([`kernels::leg_bytes`]), the card sum and the sum plus the - /// base, and the places. + /// The common pure descriptor of a prompt batch's leg and selected input protection. + #[must_use] + pub fn batch_layout(&self, cap: usize) -> model::placement::workstation::TierLegScratch { + model::placement::workstation::TierLegScratch { + n_embd: self.n as u64, + ff: self.ff as u64, + n_used: self.family.n_used as u64, + chunk: cap as u64, + downs: self.downs, + input_scales: self.expert_input == ExpertInput::AmaxF16, + } + } + + /// Device bytes of [`CardExperts::batch`] before an expert stream arms its ring. #[must_use] pub fn batch_bytes(&self, cap: usize) -> usize { - let shape = self.family.shape(self.n, self.ff, cap); - let leg = kernels::leg_bytes( - shape.cap as u64, - shape.slots_per_col as u64, - shape.n as u64, - shape.ff as u64, - self.downs, - ); - usize::try_from(leg).expect("the leg's bytes fit a usize") - + 2 * size_of::() * cap * self.n - + size_of::() * cap * self.family.n_used + let bytes = kernels::ffn::card_batch_bytes(self.batch_layout(cap)) + .expect("the card batch's dimensions have representable bytes"); + usize::try_from(bytes).expect("the card batch's bytes fit a usize") } /// Layer `l`'s card slots of the step (module doc, `m = 1`), the slots' @@ -795,6 +824,9 @@ impl CardExperts { self.k .leg .sum_rows(gpu, leg, &*sel, (l, io.m, n_card), sum)?; + if !self.k.add_base { + return Ok(()); + } gpu.elem() .enqueue_add(stream, acc, io.x1, io.m * self.n, pre) })(); @@ -838,7 +870,7 @@ impl CardKernels { acc: &mut bufs.acc, }); self.leg.enqueue(gpu, &mut bufs.leg, &run, sum)?; - if tiered { + if tiered || !self.add_base { return Ok(()); } let stream = gpu.stream(); @@ -992,11 +1024,12 @@ mod tests { vec![(0..CARD as u32).collect(); triples.len()], ) .expect("the tier set"); - let mut tier = LegTier::new( - &gpu, &specs, &set, &w, shape, FAMILY.program, - |moe| Ok(Act::SwigluClamp { limit: moe.act.swiglu_limit() }), - ) - .expect("the tier wrapper"); + let mut tier = LegTier::new(&gpu, &specs, &set, &w, shape, FAMILY.program, |moe| { + Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit(), + }) + }) + .expect("the tier wrapper"); assert_eq!(tier.block_bytes(), counted(shape.cap)); let host_x: Vec = (0..CAP * N).map(|i| (i % 71) as f32 / 71.0 - 0.5).collect(); let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); @@ -1163,8 +1196,15 @@ mod tests { assert!( matches!( LegTier::new( - &gpu, &specs, &set, &w, family.shape(N, FF, 2), program, - |moe| Ok(Act::SwigluClamp { limit: moe.act.swiglu_limit() }), + &gpu, + &specs, + &set, + &w, + family.shape(N, FF, 2), + program, + |moe| Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit() + }), ), Err(GpuError::Tensor { .. }), ), diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 6aa1c9afe..e9757bb7d 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -69,6 +69,7 @@ //! down GEMM. use crate::card_slots::{CardAcc, CardSlotKernels, CardSlots}; +use crate::expert_input::{Protection, Slots}; use crate::fault::FaultSink; use crate::gemm::{ Gemm32Args, Gemm32Kernels, Gemm32Weight, GemmAct, GemmAct32, GemmArgs, GemmInput, GemmKernels, @@ -88,6 +89,7 @@ use crate::weights::{DevWeight, Weights}; use crate::{DeviceTensor, Gpu, GpuError, Q8Act}; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use gguf::GgmlType; +pub use model::ops::ExpertInput; use model::placement::accepts::{self, LegArms}; use model::placement::kernels::{self, DownAct, DownEntry, DownForms, GateUpEntry, GemmEntry}; use std::sync::Arc; @@ -154,6 +156,7 @@ pub struct LegBufs { h: DeviceBuffer, act_h: ActForms, down: DeviceBuffer, + protection: Option, } impl LegBufs { @@ -202,9 +205,22 @@ impl LegBufs { h: DeviceBuffer::zeroed(stream, slots * shape.ff)?, act_h, down: DeviceBuffer::zeroed(stream, slots * shape.n)?, + protection: None, }) } + /// Select the loaded down-input policy before capture or execution. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + match policy { + ExpertInput::Direct => self.protection = None, + ExpertInput::AmaxF16 if self.protection.is_none() => { + self.protection = Some(Protection::new(gpu, self.shape.slots())?); + } + ExpertInput::AmaxF16 => {} + } + Ok(()) + } + /// The shape the buffers were made for. #[must_use] pub fn shape(&self) -> LegShape { @@ -215,7 +231,11 @@ impl LegBufs { /// their shape and forms. #[must_use] pub fn bytes(&self) -> usize { - self.act_x.device_bytes() + self.h.num_bytes() + self.act_h.bytes() + self.down.num_bytes() + self.act_x.device_bytes() + + self.h.num_bytes() + + self.act_h.bytes() + + self.down.num_bytes() + + self.protection.as_ref().map_or(0, Protection::bytes) } /// The down rows of the last run: `n` values a slot, slot-major, the @@ -386,6 +406,29 @@ impl RouteBlock<'_> { n_slots: usize, (io, rule): (RouteIo<'_>, Act), fault: FaultSink, + ) -> Result<(), GpuError> { + self.enqueue_input( + gpu, + (gemm, g32), + (route, top_k), + n_slots, + (io, rule), + None, + fault, + ) + } + + /// The routed block under its loaded expert-input policy. + #[allow(clippy::too_many_arguments, reason = "route execution contract")] + pub(crate) fn enqueue_input( + &self, + gpu: &Gpu, + (gemm, g32): (Option<&GemmKernels>, &Gemm32Kernels), + (route, top_k): (&GemmRoute, usize), + n_slots: usize, + (io, rule): (RouteIo<'_>, Act), + mut protection: Option<&mut Protection>, + fault: FaultSink, ) -> Result<(), GpuError> { const WHAT_B: &str = "card_leg::RouteBlock"; let stream = gpu.stream(); @@ -461,12 +504,34 @@ impl RouteBlock<'_> { )); } } + let (ids, counts) = route.expert_slots(); + let selected = Slots { + ids, + counts, + routed: true, + n_card: 0, + len: n_slots, + }; + if let Some(p) = protection.as_deref_mut() { + p.kernels.normalize( + gpu, + (g, u, &mut p.scales), + &selected, + self.ff, + Some(rule), + fault, + )?; + } let per_slot = GemmInput::PerSlot; match self.down { RouteW::Grouped(ty, w) => { let act = h128.ok_or_else(|| lacks("q8_1 blocks of 128 (the down)"))?; let gemm = grouped_k()?; - gemm.enqueue_swiglu_act_quant(stream, g, u, rule, n_slots, act, fault)?; + if protection.is_some() { + gpu.enqueue_quantize_gemm(g, n_slots, act, fault)?; + } else { + gemm.enqueue_swiglu_act_quant(stream, g, u, rule, n_slots, act, fault)?; + } gemm.enqueue_gemm( stream, GemmArgs { @@ -478,15 +543,21 @@ impl RouteBlock<'_> { input: per_slot, y, }, - ) + )?; } RouteW::Blocks32(w) => { let act = h32.ok_or_else(|| lacks("q8 blocks of 32 (the down)"))?; - match rule { - Act::SiluMul => { - g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)? + if protection.is_some() { + g32.enqueue_quantize_gemm32(stream, g, n_slots, act, fault)?; + } else { + match rule { + Act::SiluMul => { + g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)? + } + _ => { + g32.enqueue_swiglu_act_quant32(stream, g, u, rule, n_slots, act, fault)? + } } - _ => g32.enqueue_swiglu_act_quant32(stream, g, u, rule, n_slots, act, fault)?, } g32.enqueue_gemm32( stream, @@ -498,9 +569,14 @@ impl RouteBlock<'_> { input: per_slot, y, }, - ) + )?; } } + if let Some(p) = protection { + p.kernels + .restore(gpu, y, &p.scales, &selected, self.hidden)?; + } + Ok(()) } } @@ -636,7 +712,6 @@ impl<'a> LegStacks<'a> { ) } - /// [`LegStacks::of`] for a body that loads every tensor in the format /// [`gguf::card::CardFormat::of`] names (`Weights::load`), whose downs may /// also be Q5_0 in the `pack_q5_0` layout ([`accepts::resident_down_runs`]). @@ -879,6 +954,7 @@ struct Scratch<'a> { act_x: &'a mut Q8Act, h: &'a mut DeviceBuffer, act_h: &'a mut ActForms, + protection: &'a mut Option, } impl CardLeg { @@ -924,6 +1000,7 @@ impl CardLeg { h, act_h, down, + protection, } = bufs; let shape = *shape; if sum.is_some() { @@ -934,6 +1011,7 @@ impl CardLeg { act_x, h, act_h, + protection, }; self.launch(gpu, scratch, run, down)?; let Some(sum) = sum else { @@ -966,6 +1044,7 @@ impl CardLeg { act_x, h, act_h, + protection, .. } = bufs; let scratch = Scratch { @@ -973,6 +1052,7 @@ impl CardLeg { act_x, h, act_h, + protection, }; self.launch(gpu, scratch, run, down) } @@ -1029,6 +1109,7 @@ impl CardLeg { act_x, h, act_h, + protection, } = scratch; if run.m == 0 || run.m > cap { return Err(GpuError::Shape { @@ -1114,6 +1195,23 @@ impl CardLeg { } _ => return Err(st.gate.unrun()), } + let selected = Slots { + ids: run.sel, + counts: run.sel, + routed: false, + n_card: run.n_card, + len: slots, + }; + if let Some(p) = protection { + p.kernels.normalize( + gpu, + (h, run.normed, &mut p.scales), + &selected, + ff, + None, + fault, + )?; + } let q = QuantSel { x: h, cols: 0..slots, @@ -1233,6 +1331,9 @@ impl CardLeg { } _ => return Err(st.down.unrun()), } + if let Some(p) = protection { + p.kernels.restore(gpu, down, &p.scales, &selected, n)?; + } Ok(()) } } diff --git a/crates/gpu/src/expert_input.rs b/crates/gpu/src/expert_input.rs new file mode 100644 index 000000000..47f3a10f1 --- /dev/null +++ b/crates/gpu/src/expert_input.rs @@ -0,0 +1,257 @@ +//! Per-slot protection of quantized expert down inputs and restoration before routing. + +use crate::fault::{FaultSink, FaultSite}; +use crate::kquant::{Act, act::apply_checked}; +use crate::{Gpu, GpuError, launch_u32}; +use cuda_core::{DeviceBuffer, LaunchConfig1D}; +use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread, warp}; +use cuda_host::cuda_module; + +/// Slot selection: resident `_sel` places, or the route's unique slot list and refused count. +pub(crate) struct Slots<'a> { + pub ids: &'a DeviceBuffer, + pub counts: &'a DeviceBuffer, + pub routed: bool, + pub n_card: usize, + pub len: usize, +} + +#[cuda_module] +mod expert_input_kernels { + use super::*; + + #[inline(always)] + unsafe fn slot( + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + len: u32, + i: usize, + ) -> Option { + if i >= len as usize { + return None; + } + if routed != 0 { + // SAFETY: the route owns two counters; its slot list has len entries. + if i >= len as usize - unsafe { *counts.get_unchecked(1) } as usize { + return None; + } + // SAFETY: i < len <= ids.len(). Route slots are unique and below len. + Some(unsafe { *ids.get_unchecked(i) } as usize) + } else { + // SAFETY: i < len <= ids.len(). + (unsafe { *ids.get_unchecked(i) } < n_card).then_some(i) + } + } + + #[allow(clippy::too_many_arguments, reason = "flat device ABI")] + #[kernel] + #[launch_bounds(32)] + #[launch_contract(domain = 1, block = (32, 1, 1), requires = ( + ids.len() >= slots, counts.len() >= 2 * routed, h.len() >= slots * width, + scales.len() >= slots, u.len() >= activate * slots * width + ))] + pub fn normalize( + mut h: DisjointSlice, + u: &[f32], + mut scales: DisjointSlice, + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + slots: u32, + width: u32, + activate: u32, + act: u32, + limit: f32, + fault: FaultSink, + ) { + let i = thread::index_1d().get() / 32; + // SAFETY: the launch contract bounds ids and counts; route invariants bound slot ids. + let Some(s) = (unsafe { slot(ids, counts, routed, n_card, slots, i) }) else { + return; + }; + let lane = warp::lane_id() as usize; + let base = s * width as usize; + let mut max = 0.0f32; + let mut bad = false; + let mut d = lane; + while d < width as usize { + // SAFETY: s < slots, d < width; each lane owns its strided cells. + let mut v = unsafe { *h.get_unchecked_mut(base + d) }; + if activate != 0 { + // SAFETY: u holds slots whole rows when activate is selected. + v = apply_checked(act, limit, v, unsafe { *u.get_unchecked(base + d) }, fault); + // SAFETY: this lane alone owns this cell. + unsafe { *h.get_unchecked_mut(base + d) = v }; + } + bad |= !v.is_finite(); + max = max.max(v.abs()); + d += 32; + } + let max = warp::reduce_max_f32(max); + let bad = warp::ballot(bad) != 0; + // Keep the pinned graph's log/multiply/ceil/clamp/multiply/exp order. + let e = if bad { + f32::NAN + } else { + (((max * (1.0 / 1024.0)).ln() * std::f32::consts::LOG2_E) + .ceil() + .clamp(0.0, 127.0) + * std::f32::consts::LN_2) + .exp() + }; + if lane == 0 { + // SAFETY: this warp uniquely owns slot s. + unsafe { *scales.get_unchecked_mut(s) = e }; + if bad { + fault.raise(FaultSite::QuantColumn); + } + } + d = lane; + while d < width as usize { + // SAFETY: the same lane owns this cell, and s < slots. + unsafe { + let v = *h.get_unchecked_mut(base + d); + *h.get_unchecked_mut(base + d) = v / e; + } + d += 32; + } + } + + #[allow(clippy::too_many_arguments, reason = "flat device ABI")] + #[kernel] + #[launch_bounds(32)] + #[launch_contract(domain = 1, block = (32, 1, 1), requires = ( + ids.len() >= slots, counts.len() >= 2 * routed, y.len() >= slots * width, scales.len() >= slots + ))] + pub fn restore( + mut y: DisjointSlice, + scales: &[f32], + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + slots: u32, + width: u32, + ) { + let i = thread::index_1d().get() / 32; + // SAFETY: launch bounds and route invariants are as in normalize. + let Some(s) = (unsafe { slot(ids, counts, routed, n_card, slots, i) }) else { + return; + }; + // SAFETY: s < slots <= scales.len(). + let e = unsafe { *scales.get_unchecked(s) }; + let mut d = warp::lane_id() as usize; + while d < width as usize { + // SAFETY: each lane owns its cells of the unique selected slot. + unsafe { + let v = *y.get_unchecked_mut(s * width as usize + d); + *y.get_unchecked_mut(s * width as usize + d) = v * e; + } + d += 32; + } + } +} + +pub(crate) struct ExpertInputKernels { + module: expert_input_kernels::LoadedModule, +} + +impl ExpertInputKernels { + pub fn load(gpu: &Gpu) -> Result { + // SAFETY: this crate owns the embedded module and enforces its launch contracts. + let module = unsafe { crate::shared_module!(expert_input_kernels, gpu.context())? }; + Ok(Self { module }) + } + + pub fn normalize( + &self, + gpu: &Gpu, + (h, u, scales): ( + &mut DeviceBuffer, + &DeviceBuffer, + &mut DeviceBuffer, + ), + slots: &Slots<'_>, + width: usize, + rule: Option, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "ExpertInput::normalize"; + let (act, limit) = rule.unwrap_or(Act::SiluMul).code(what)?; + let len = launch_u32(what, "slots", slots.len)?; + let width = launch_u32(what, "width", width)?; + let n_card = launch_u32(what, "n_card", slots.n_card)?; + let prep = self + .module + .prepare_normalize(LaunchConfig1D::new(len, 32, 0))?; + self.module.normalize( + gpu.stream(), + &prep, + h, + u, + scales, + slots.ids, + slots.counts, + u32::from(slots.routed), + n_card, + len, + width, + u32::from(rule.is_some()), + act, + limit, + fault, + )?; + Ok(()) + } + + pub fn restore( + &self, + gpu: &Gpu, + y: &mut DeviceBuffer, + scales: &DeviceBuffer, + slots: &Slots<'_>, + width: usize, + ) -> Result<(), GpuError> { + let what = "ExpertInput::restore"; + let len = launch_u32(what, "slots", slots.len)?; + let width = launch_u32(what, "width", width)?; + let n_card = launch_u32(what, "n_card", slots.n_card)?; + let prep = self + .module + .prepare_restore(LaunchConfig1D::new(len, 32, 0))?; + self.module.restore( + gpu.stream(), + &prep, + y, + scales, + slots.ids, + slots.counts, + u32::from(slots.routed), + n_card, + len, + width, + )?; + Ok(()) + } +} + +pub(crate) struct Protection { + pub kernels: ExpertInputKernels, + pub scales: DeviceBuffer, +} + +impl Protection { + pub fn new(gpu: &Gpu, slots: usize) -> Result { + Ok(Self { + kernels: ExpertInputKernels::load(gpu)?, + scales: DeviceBuffer::zeroed(gpu.stream(), slots)?, + }) + } + + pub fn bytes(&self) -> usize { + self.scales.num_bytes() + } +} diff --git a/crates/gpu/src/flash_gqa_h128_swa.rs b/crates/gpu/src/flash_gqa_h128_swa.rs index 68379d062..7d1d7c0a5 100644 --- a/crates/gpu/src/flash_gqa_h128_swa.rs +++ b/crates/gpu/src/flash_gqa_h128_swa.rs @@ -67,7 +67,7 @@ fn live_keys(count: u32, ctx: usize) -> usize { clippy::too_many_arguments, reason = "the tile step's state is the caller's registers, passed by reference" )] -unsafe fn fold_tile( +pub(crate) unsafe fn fold_tile( sc: f32, live: bool, mx: &mut f32, diff --git a/crates/gpu/src/gemm/route.rs b/crates/gpu/src/gemm/route.rs index 2abfceb70..49607235d 100644 --- a/crates/gpu/src/gemm/route.rs +++ b/crates/gpu/src/gemm/route.rs @@ -159,6 +159,11 @@ impl GemmRoute { self.n_experts } + /// The unique active-slot list and counters for per-slot expert operations. + pub(crate) fn expert_slots(&self) -> (&DeviceBuffer, &DeviceBuffer) { + (&self.cols, &self.n_tiles) + } + /// Device bytes of the table. #[must_use] pub fn bytes(&self) -> usize { diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 51df65d8d..502a05bca 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -281,8 +281,8 @@ mod handoff_kernels { unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; } - /// [`ds41_ffn_handoff`] and the row's f32 tier image, with the same - /// per-thread writes and following go as [`ds41_ffn_handoff_8_tier`]. + /// Six slots' f32 tier image, with the same per-thread writes and + /// following go as the four- and eight-slot tier entries. #[allow( clippy::too_many_arguments, reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" @@ -1630,8 +1630,10 @@ impl HandoffKernels { /// slot entries. The generated launch checks the input and output spans. fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result { let slots = HandoffSlots::of(lay.n_used); - if !matches!(slots, Ok(HandoffSlots::Four | HandoffSlots::Six | HandoffSlots::Eight)) - || tl.n_used != lay.n_used + if !matches!( + slots, + Ok(HandoffSlots::Four | HandoffSlots::Six | HandoffSlots::Eight) + ) || tl.n_used != lay.n_used || tl.x_len != lay.hidden || tl.q3_words != 0 || tl.d8_len != 0 diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 04572a78a..0ed10e8c0 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -40,7 +40,7 @@ use super::run::{HostRun, HostWidths}; use super::tier::{ TierAct, TierBlock, TierCard, TierExperts, TierInput, TierIo, TierOpen, TierSet, TierShape, }; -use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks}; +use crate::card_leg::{CardLeg, ExpertInput, LegBufs, LegRun, LegShape, LegStacks}; use crate::hybrid::{Hybrid, SlotMap}; use crate::kquant::Act; use crate::model::CardBytes; @@ -264,18 +264,10 @@ impl LegTier { arms, }) } -} -/// `names` read by `arms`, or by the card leg's own arms when the family -/// named none ([`LegStacks::of`]). -fn stacks_of<'a>( - arms: Option<(LegArms, (&'static str, &'static str))>, - w: &'a Weights, - names: [&'a str; 3], -) -> Result, GpuError> { - match arms { - None => LegStacks::of(w, names), - Some((arms, roles)) => LegStacks::of_arms(w, names, arms, roles), + /// Select the loaded policy shared by stage-card and tier-card experts. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + self.bufs.set_expert_input(gpu, policy) } } @@ -330,7 +322,7 @@ impl TierExperts for LegTier { .. } = *shape; let tl = layer_of(layers, cfg, layer, W)?; - let stacks = LegStacks::of_arms(weights, tl.names(), *arms, STACK_ROLES)?; + let stacks = LegStacks::of_arms(weights, tl.names(), arms, STACK_ROLES)?; for c0 in (0..io.cols).step_by(cap) { let cn = cap.min(io.cols - c0); let slots = cn * slots_per_col; @@ -365,7 +357,6 @@ impl TierExperts for LegTier { io: TierIo<'_>, ) -> Result<(), GpuError> { const W: &str = "LegTier::enqueue_layer"; - let arms = self.arms; let tl = layer_of(&self.layers, &self.cfg, layer, W)?; let TierInput::F32(x) = io.act else { return Err(GpuError::Shape { @@ -523,6 +514,11 @@ pub trait LegFamily { /// The file as the family's placement reads it. type Inputs; + /// The loaded down-input policy, shared with the stage card and host. + fn expert_input(_inputs: &Self::Inputs) -> ExpertInput { + ExpertInput::Direct + } + /// `file` read as the family's placement reads it; its refusals are /// the family's. fn describe(file: &Split) -> Result; @@ -569,7 +565,6 @@ pub trait LegFamily { limit: moe.act.swiglu_limit(), }) } - } /// The family `F`'s tier node service: the placement tensors of a file, the @@ -614,7 +609,7 @@ fn node_experts( let inputs = node_inputs::(file, open)?; let widths = F::widths(&inputs); let shape = widths.shape(F::CHUNK); - Ok(Box::new(LegTier::new( + let mut tier = LegTier::new( gpu, F::layers(&inputs), &open.set, @@ -622,7 +617,9 @@ fn node_experts( shape, widths.program, F::tier_act, - )?)) + )?; + tier.set_expert_input(gpu, F::expert_input(&inputs))?; + Ok(Box::new(tier)) } fn node_host(file: Arc, open: &NodeOpen) -> Result { @@ -640,8 +637,12 @@ fn node_host(file: Arc, open: &NodeOpen) -> Result(views) }, ) } diff --git a/crates/gpu/src/host/run.rs b/crates/gpu/src/host/run.rs index 405d5aade..01d434628 100644 --- a/crates/gpu/src/host/run.rs +++ b/crates/gpu/src/host/run.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use gguf::Split; use model::moe::{HostLayer, HostScratch, UNION_MAX_COLS, UnionScratch}; -use model::ops::PostNorm; +use model::ops::{ExpertInput, PostNorm}; use model::r8file::{R8Pair, R8Source}; use model::{Tensor2, Tensor2View}; @@ -174,6 +174,9 @@ impl HostRun { scratch.len() - 1 } }; + if view.expert_input() == ExpertInput::AmaxF16 { + scratch[scratch_of[l]].set_expert_input(ExpertInput::AmaxF16); + } layers[l] = Some(view); } Ok(HostRun { @@ -257,13 +260,15 @@ impl HostRun { .scratch .iter() .map(|s| { - if cols <= UNION_MAX_COLS { + let mut union = if cols <= UNION_MAX_COLS { UnionScratch::new_routed(embd, s.ff(), cols, n_used) } else { let groups = cols.div_ceil(UNION_MAX_COLS); let slots = groups * UNION_MAX_COLS * n_used; UnionScratch::new_tail(embd, s.ff(), groups, slots, n_used) - } + }?; + union.set_expert_input(s.expert_input()); + Ok::<_, model::ModelError>(union) }) .collect::>()?; Ok(()) diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 39b4af101..442c3fbce 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -29,6 +29,8 @@ use cuda_host::cuda_module; use std::sync::{Arc, Mutex, PoisonError}; pub mod arch; +pub mod attention_h128; +mod attention_h128_ring; pub mod cand; pub mod capsync; pub mod card_experts; @@ -41,6 +43,7 @@ pub mod draft; pub mod draft_taps; pub mod dspark; pub mod elem; +mod expert_input; pub mod fault; pub mod flash; pub mod flash_gqa; diff --git a/crates/gpu/src/ring_gemm.rs b/crates/gpu/src/ring_gemm.rs index 008cedc73..f631a68dc 100644 --- a/crates/gpu/src/ring_gemm.rs +++ b/crates/gpu/src/ring_gemm.rs @@ -8,7 +8,9 @@ use cuda_core::DeviceBuffer; use model::arch::place_hybrid::{RingForms, ring_pass_bytes_in}; +use crate::card_leg::ExpertInput; use crate::card_leg::{RouteBlock, RouteIo}; +use crate::expert_input::Protection; use crate::gemm::{Gemm32Kernels, GemmAct, GemmAct32, GemmKernels, GemmRoute}; use crate::host::xstream::LandBatch; use crate::kquant::Act; @@ -50,6 +52,8 @@ pub(crate) struct RingGemm { h: Rows, route: GemmRoute, cap: usize, + slots: usize, + protection: Option, } impl RingGemm { @@ -72,6 +76,8 @@ impl RingGemm { h: Rows::new(gpu, slots, ff, forms.down32)?, route: GemmRoute::new(stream, slots, half_slots)?, cap, + slots, + protection: None, }; let want = ring_pass_bytes_in(cap, n_used, n, ff, half_slots, forms); if ring.bytes() != want { @@ -87,6 +93,21 @@ impl RingGemm { Ok(ring) } + pub(crate) fn set_expert_input( + &mut self, + gpu: &Gpu, + policy: ExpertInput, + ) -> Result<(), GpuError> { + match policy { + ExpertInput::Direct => self.protection = None, + ExpertInput::AmaxF16 if self.protection.is_none() => { + self.protection = Some(Protection::new(gpu, self.slots)?); + } + ExpertInput::AmaxF16 => {} + } + Ok(()) + } + /// Device bytes of the buffers. pub(crate) fn bytes(&self) -> usize { self.x.bytes() @@ -94,6 +115,7 @@ impl RingGemm { + self.u.num_bytes() + self.h.bytes() + self.route.bytes() + + self.protection.as_ref().map_or(0, Protection::bytes) } /// The slots of the half the buffers were made for. @@ -148,7 +170,13 @@ impl RingGemm { ) -> Result<(), GpuError> { let stream = gpu.stream(); let RingGemm { - x, g, u, h, route, .. + x, + g, + u, + h, + route, + protection, + .. } = self; batch.with_map(gpu.context(), n_expert, |row| { g32.enqueue_route_remap(stream, ids, row, n_slots, &mut *route, fault) @@ -164,7 +192,7 @@ impl RingGemm { Rows::Q128(a) => (Some(&mut **a), None), Rows::Blocks32(a) => (None, Some(&mut **a)), }; - blk.enqueue_rule( + blk.enqueue_input( gpu, (gemm, g32), (&*route, top_k), @@ -181,6 +209,7 @@ impl RingGemm { }, rule, ), + protection.as_mut(), fault, ) } diff --git a/crates/gpu/src/route_body.rs b/crates/gpu/src/route_body.rs index 6058f9154..253d668d7 100644 --- a/crates/gpu/src/route_body.rs +++ b/crates/gpu/src/route_body.rs @@ -68,9 +68,11 @@ pub unsafe fn publish_n( while e < E { // SAFETY: at + e < at + E, inside probs by this fn's contract. let p = unsafe { core::ptr::read_volatile(probs.add(at + e)) }; - // SAFETY: any supplied bias holds E entries by this fn's contract. let v = match bias { - Some(bias) => p + unsafe { *bias.get_unchecked(e) }, + Some(bias) => { + // SAFETY: e < E and any supplied bias holds E entries. + p + unsafe { *bias.get_unchecked(e) } + } None => p, }; if !v.is_finite() { @@ -107,7 +109,12 @@ pub unsafe fn publish_n( reason = "the selecting block's shared arrays, its lane and scale, the fault and the outputs" )] #[inline(always)] -pub unsafe fn select_n( +pub unsafe fn select_n< + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( score: *const f32, sel_v: *const f32, slot_p: *mut f32, @@ -238,7 +245,13 @@ pub unsafe fn select_n( +pub unsafe fn router_body< + S: Score, + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( w: &[f32], x: &[f32], bias: Option<&[f32]>, @@ -304,7 +317,9 @@ pub unsafe fn router_body(score, sel_v, slot_p, lane, scale, fault, ids, weights, 0) + select_n::( + score, sel_v, slot_p, lane, scale, fault, ids, weights, 0, + ) }; if lane == 0 { // SAFETY: `ticket` is done[0], inside `done` by this fn's @@ -331,7 +346,12 @@ pub unsafe fn router_body( +pub unsafe fn pick_body< + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( probs: &[f32], bias: Option<&[f32]>, n_expert: u32, @@ -982,11 +1002,23 @@ pub mod sigmoid_floor_256_6 { /// Both metadata bindings name the same optional-bias device entries. pub const ROW: RouterInst = shape::router_row_rule(RouterBody::SigmoidFloor, rules::SIGMOID_FLOOR, 256, 6); - pub const BIASED_ROW: RouterInst = - shape::router_row_rule(RouterBody::SigmoidFloor, rules::BIASED_SIGMOID_FLOOR, 256, 6); - const _: () = assert!(shape::router_row_is(ROW, rules::SIGMOID_FLOOR, false, (6, 6))); + pub const BIASED_ROW: RouterInst = shape::router_row_rule( + RouterBody::SigmoidFloor, + rules::BIASED_SIGMOID_FLOOR, + 256, + 6, + ); const _: () = assert!(shape::router_row_is( - BIASED_ROW, rules::BIASED_SIGMOID_FLOOR, false, (6, 6) + ROW, + rules::SIGMOID_FLOOR, + false, + (6, 6) + )); + const _: () = assert!(shape::router_row_is( + BIASED_ROW, + rules::BIASED_SIGMOID_FLOOR, + false, + (6, 6) )); const _: () = assert!(N_EXPERT.is_multiple_of(ROWS_PER_BLOCK)); const _: () = assert!(N_EXPERT.div_ceil(32) <= 32); @@ -1045,9 +1077,18 @@ pub mod sigmoid_floor_256_6 { // Shared arrays belong to this block and have the body's sizes. unsafe { router_body::( - w, x, if has_bias != 0 { Some(bias) } else { None }, - N_EXPERT as u32, k, scale, - &mut logits, &mut probs, &mut ids, &mut weights, &mut done, fault, + w, + x, + if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, + k, + scale, + &mut logits, + &mut probs, + &mut ids, + &mut weights, + &mut done, + fault, SharedArray::as_raw_mut_ptr(&raw mut LAST), SharedArray::as_raw_mut_ptr(&raw mut SCORE), SharedArray::as_raw_mut_ptr(&raw mut SEL_V), @@ -1091,8 +1132,14 @@ pub mod sigmoid_floor_256_6 { // block; shared arrays belong to this block at the body's sizes. unsafe { pick_body::( - probs, if has_bias != 0 { Some(bias) } else { None }, - N_EXPERT as u32, t, scale, &mut ids, &mut weights, fault, + probs, + if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, + t, + scale, + &mut ids, + &mut weights, + fault, SharedArray::as_raw_mut_ptr(&raw mut SCORE), SharedArray::as_raw_mut_ptr(&raw mut SEL_V), SharedArray::as_raw_mut_ptr(&raw mut SLOT_P), @@ -1140,19 +1187,26 @@ pub mod sigmoid_floor_256_6 { if w.rows() != N_EXPERT || k == 0 || !k.is_multiple_of(32) { return Err(GpuError::shape( what, - format!("router weight is {} x {k}, want 256 rows of a positive multiple of 32", w.rows()), + format!( + "router weight is {} x {k}, want 256 rows of a positive multiple of 32", + w.rows() + ), )); } Ok(k) } fn check_bias(what: &'static str, bias: Option<&DeviceBuffer>) -> Result<(), GpuError> { - if let Some(bias) = bias { - if bias.len() < N_EXPERT { - return Err(GpuError::shape( - what, format!("selection bias has {} entries, want at least 256", bias.len()), - )); - } + if let Some(bias) = bias + && bias.len() < N_EXPERT + { + return Err(GpuError::shape( + what, + format!( + "selection bias has {} entries, want at least 256", + bias.len() + ), + )); } Ok(()) } @@ -1188,18 +1242,43 @@ pub mod sigmoid_floor_256_6 { let k = weight_k(what, w)?; check_bias(what, bias)?; if x.len() < k { - return Err(GpuError::shape(what, format!("x.len() {} < k {k}", x.len()))); + return Err(GpuError::shape( + what, + format!("x.len() {} < k {k}", x.len()), + )); } let k = launch_u32(what, "k", k)?; - let prep = self.module.prepare_sigmoid_floor_256_6_router( - LaunchConfig1D::new((N_EXPERT / ROWS_PER_BLOCK) as u32, ROUTER_THREADS_U32, 0), - )?; - let RouterOut { logits, probs, ids, weights, done } = out; + let prep = self + .module + .prepare_sigmoid_floor_256_6_router(LaunchConfig1D::new( + (N_EXPERT / ROWS_PER_BLOCK) as u32, + ROUTER_THREADS_U32, + 0, + ))?; + let RouterOut { + logits, + probs, + ids, + weights, + done, + } = out; // With no bias the weight slice is only an ABI borrow: the // kernel takes None and never loads it as a selection bias. self.module.sigmoid_floor_256_6_router( - stream, &prep, w.buf(), x, bias.unwrap_or(w.buf()), u32::from(bias.is_some()), - k, scale, logits, probs, ids, weights, done, fault, + stream, + &prep, + w.buf(), + x, + bias.unwrap_or(w.buf()), + u32::from(bias.is_some()), + k, + scale, + logits, + probs, + ids, + weights, + done, + fault, )?; Ok(()) } @@ -1231,25 +1310,53 @@ pub mod sigmoid_floor_256_6 { let k = weight_k(what, w)?; check_bias(what, bias)?; let tt = launch_u32(what, "t", t)?; - let input = k.checked_mul(t).ok_or_else(|| GpuError::shape(what, "input size overflow"))?; - let scores = N_EXPERT.checked_mul(t).ok_or_else(|| GpuError::shape(what, "score size overflow"))?; - let slots = N_USED.checked_mul(t).ok_or_else(|| GpuError::shape(what, "slot size overflow"))?; - if t == 0 || x.len() < input || probs.len() < scores || ids.len() < slots || weights.len() < slots { + let input = k + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "input size overflow"))?; + let scores = N_EXPERT + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "score size overflow"))?; + let slots = N_USED + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "slot size overflow"))?; + if t == 0 + || x.len() < input + || probs.len() < scores + || ids.len() < slots + || weights.len() < slots + { return Err(GpuError::shape( what, - format!("{t} tokens of {k} (at least one): x {}, probs {}, ids {}, weights {}", - x.len(), probs.len(), ids.len(), weights.len()), + format!( + "{t} tokens of {k} (at least one): x {}, probs {}, ids {}, weights {}", + x.len(), + probs.len(), + ids.len(), + weights.len() + ), )); } // The tile GEMM uses the one-token row walk and warp sum tree. wide.enqueue_f32_tile(stream, w, x, t, probs)?; gpu.elem().enqueue_sigmoid_rows(stream, probs, scores)?; - let prep = self.module.prepare_sigmoid_floor_256_6_router_pick( - LaunchConfig1D::new(tt, ROUTER_THREADS_U32, 0), - )?; + let prep = self + .module + .prepare_sigmoid_floor_256_6_router_pick(LaunchConfig1D::new( + tt, + ROUTER_THREADS_U32, + 0, + ))?; self.module.sigmoid_floor_256_6_router_pick( - stream, &prep, probs, bias.unwrap_or(w.buf()), u32::from(bias.is_some()), - tt, scale, ids, weights, fault, + stream, + &prep, + probs, + bias.unwrap_or(w.buf()), + u32::from(bias.is_some()), + tt, + scale, + ids, + weights, + fault, )?; Ok(()) } @@ -1259,6 +1366,8 @@ pub mod sigmoid_floor_256_6 { const N_EXPERT: usize = N_EXPERT; const N_USED: usize = N_USED; const OPTIONAL_BIAS: bool = true; + const NORM: model::arch::models::RouterNorm = + model::arch::models::RouterNorm::SelectedSumFloor; type Out = RouterOut; fn load(ctx: &Arc) -> Result { @@ -1274,8 +1383,11 @@ pub mod sigmoid_floor_256_6 { &out.weights } fn out_bytes(out: &RouterOut) -> usize { - out.logits.num_bytes() + out.probs.num_bytes() + out.ids.num_bytes() - + out.weights.num_bytes() + out.done.num_bytes() + out.logits.num_bytes() + + out.probs.num_bytes() + + out.ids.num_bytes() + + out.weights.num_bytes() + + out.done.num_bytes() } fn route( &self, @@ -1295,8 +1407,18 @@ pub mod sigmoid_floor_256_6 { rows: RoutedRows<'_>, ) -> Result<(), GpuError> { self.enqueue_router_rows( - stream, gpu, wide, r.w, x, r.bias, r.scale, t, - rows.probs, rows.ids, rows.weights, r.fault, + stream, + gpu, + wide, + r.w, + x, + r.bias, + r.scale, + t, + rows.probs, + rows.ids, + rows.weights, + r.fault, ) } } diff --git a/crates/gpu/src/routed_block.rs b/crates/gpu/src/routed_block.rs index 7ad972410..146707c34 100644 --- a/crates/gpu/src/routed_block.rs +++ b/crates/gpu/src/routed_block.rs @@ -74,6 +74,8 @@ pub trait RouterEntry: Sized { const N_USED: usize; /// Whether an absent selection bias is a native binding, not a zero buffer. const OPTIONAL_BIAS: bool = false; + /// The selected-score normalization implemented by this compiled binding. + const NORM: model::arch::models::RouterNorm = model::arch::models::RouterNorm::SelectedSum; /// Where one token's launch leaves its results. type Out; diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index 696423f1d..446d76fa9 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -46,6 +46,66 @@ pub struct Available { /// Everything the programs run that a [`Need`] names. A need no row covers /// is an item of [`check`]; a row removed makes its needs items. pub const AVAILABLE: &[Available] = &[ + Available { + programs: &[Program::FlintBody], + at: "gpu/src/attention_h128.rs Binding::enqueue and attention_h128_ring.rs", + runs: |n| { + matches!( + n, + Need::Gqa { + head: 128, + value: 128, + group: 8, + window: None | Some(513), + sinks: true, + scaled: false, + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu/src/attention_h128_ring.rs (head128 RMS with optional full NeoX)", + runs: |n| { + matches!( + n, + Need::QkRope { + qk_norm: true, + head: 128, + mode: RopeMode::None, + dims: 0, + } | Need::QkRope { + qk_norm: true, + head: 128, + mode: RopeMode::Neox, + dims: 128, + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu/src/route_body.rs sigmoid_floor_256_6", + runs: |n| routes(n, &[RouterBody::SigmoidFloor]), + }, + Available { + programs: &[Program::FlintBody], + at: "gpu-deepseek41/src/shared_routed.rs (ungated shared expert)", + runs: |n| { + matches!( + n, + Need::Shared { + sigmoid_gate: false, + .. + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu-deepseek41/src/shared_routed.rs dense-leading gated blocks", + runs: |n| matches!(n, Need::DenseFfn { ff } if ff.is_multiple_of(256)), + }, Available { programs: &[Program::Deepseek41Chain], at: "gpu-deepseek41/src/attn.rs LATENT; chain/attn.rs rope dims", @@ -1500,6 +1560,46 @@ struct RowPin { /// and per-layer input projection (the host's rows), each `what` naming the /// types its row takes or the table row it follows. const ROW_PINS: &[RowPin] = &[ + RowPin { + program: Program::FlintBody, + role: Role::Attention, + matrices: true, + names: None, + what: "head128 attention projections (common dense sites)", + row: h128_projection, + }, + RowPin { + program: Program::FlintBody, + role: Role::DenseFfn, + matrices: true, + names: None, + what: "dense-leading gated blocks", + row: gated_block_site, + }, + RowPin { + program: Program::FlintBody, + role: Role::SharedExpert, + matrices: true, + names: None, + what: "shared gated blocks", + row: gated_block_site, + }, + RowPin { + program: Program::FlintBody, + role: Role::TokenEmbedding, + matrices: true, + names: None, + what: "common token embedding", + row: |ty| embed_of_widened(ty).is_some(), + }, + RowPin { + program: Program::FlintBody, + role: Role::Head, + matrices: true, + names: None, + what: "common output head", + row: |ty| head_of_widened(ty).is_some(), + }, // PIN(2026-10-22): Qwen38 dense pins admit MXFP8/FP8_B128 and Q8_0 indexers via BY_NAME; the old MXFP8 refusal fails the new coverage test (PR FAIL-first). RowPin { program: Program::Qwen38Body, @@ -1672,6 +1772,35 @@ const ROW_PINS: &[RowPin] = &[ }, ]; +fn h128_projection(ty: GgmlType) -> bool { + matches!( + ty, + GgmlType::Q3_K + | GgmlType::Q4_K + | GgmlType::Q5_K + | GgmlType::Q6_K + | GgmlType::IQ4_XS + | GgmlType::Q5_0 + | GgmlType::Q8_0 + | GgmlType::F32 + | GgmlType::MXFP8 + | GgmlType::FP8_B128 + | GgmlType::BF16 + ) +} + +fn gated_block_site(ty: GgmlType) -> bool { + matches!( + ty, + GgmlType::Q8_0 + | GgmlType::Q4_K + | GgmlType::Q5_K + | GgmlType::Q6_K + | GgmlType::IQ4_XS + | GgmlType::MXFP8 + ) +} + /// `Body35`'s projection types (`PROJ`): every attention, delta-rule and /// dense FFN matrix. const Q35_PROJ: &[GgmlType] = &[ @@ -1878,7 +2007,7 @@ pub fn check_with( // stack's row width ([`routed_unrun`], the one owner). A layer // with a stack the card experts do not read keeps all its // experts on the host. - Some(p @ (Program::Glm5nextBody | Program::MiniMaxM3Body)) => { + Some(p @ (Program::Glm5nextBody | Program::MiniMaxM3Body | Program::FlintBody)) => { if routed_unrun(t, |ty| leg_card_routed(p, ty)) { at(t.layer, Need::RoutedFormat(t.ty)); } diff --git a/crates/model/src/arch/flint/host.rs b/crates/model/src/arch/flint/host.rs new file mode 100644 index 000000000..714fd4eb2 --- /dev/null +++ b/crates/model/src/arch/flint/host.rs @@ -0,0 +1,83 @@ +//! Flint's validated routed block as the common host and NVMe expert tiers serve it. + +use models::{Act, Ffn, ModelSpec}; + +use crate::ModelError; +use crate::arch::names; +use crate::moe::{HostLayer, HostLayerSpec}; +use crate::ops::ExpertInput; +use crate::r8file::R8Source; + +/// Build the requested layer's routed stacks with Flint's unclamped SwiGLU and activated-value +/// scaling. Dense layers, layers outside the description and non-SwiGLU routed blocks are refused +/// before a tensor is opened. Load-time only. +pub fn layer(src: R8Source<'_>, spec: &ModelSpec, l: usize) -> Result { + let refuse = |detail: String| ModelError::Metadata { + key: format!("layer {l} routed block"), + detail, + }; + let block = spec + .layers + .get(l) + .ok_or_else(|| refuse(format!("outside the file's {} layers", spec.layers.len())))?; + let Ffn::Moe(moe) = &block.ffn else { + return Err(refuse("is dense, not a routed block".to_string())); + }; + if moe.act != (Act::SwiGlu { limit: None }) { + return Err(refuse(format!( + "activation {:?}; Flint's host experts require unclamped SwiGLU", + moe.act + ))); + } + let [gate, up, down] = names::routed_stacks(l); + let mut layer = HostLayer::build( + src, + &HostLayerSpec { + gate: &gate, + up: &up, + down: &down, + n_expert: moe.experts as usize, + embd: spec.hidden as usize, + ff: moe.expert_ff as usize, + swiglu_limit: 0.0, + }, + )?; + layer.set_expert_input(ExpertInput::AmaxF16); + Ok(layer) +} + +#[cfg(test)] +mod tests { + use super::super::tests::{keys, tensors}; + use super::*; + use crate::arch::synthetic::{V, header_shaped}; + + #[test] + fn refuses_dense_outside_and_non_swiglu_before_opening_experts() { + let path = header_shaped( + "flint-host-layer", + "flint", + &keys(), + &[("tokenizer.ggml.pre", V::Str("flint"))], + &tensors(), + ); + let split = gguf::Split::open(&path).unwrap(); + let mut spec = super::super::spec::read(&split).unwrap().spec; + let src = R8Source::rows(&split); + for (l, fragment) in [(0, "is dense"), (4, "outside the file's 4 layers")] { + let error = layer(src, &spec, l).err().expect("typed refusal"); + assert!(matches!(error, ModelError::Metadata { ref key, ref detail } + if key == &format!("layer {l} routed block") && detail.contains(fragment))); + } + let Ffn::Moe(moe) = &mut spec.layers[1].ffn else { + panic!("routed"); + }; + moe.act = Act::ReluSqr; + let error = layer(src, &spec, 1) + .err() + .expect("non-gated activation refused"); + assert!(matches!(error, ModelError::Metadata { ref detail, .. } + if detail.contains("require unclamped SwiGLU"))); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/model/src/arch/flint/mod.rs b/crates/model/src/arch/flint/mod.rs index 5297cb5f4..afee36297 100644 --- a/crates/model/src/arch/flint/mod.rs +++ b/crates/model/src/arch/flint/mod.rs @@ -1,12 +1,11 @@ -//! Flint's header description and exact tensor roles. Full and sliding attention -//! independently select NoPE or NeoX rotation; every layer has head-local q/k -//! norms and attention sinks. Routed blocks carry one ungated shared MLP whose -//! width is explicit, not a count of physical modules. -//! -//! This is a reader, not an executable program. Coverage retains `Need::Program` -//! and the missing numerical/site bindings; no fixture or serve seat is registered. +//! Flint's validated description, placement inputs and routed host adapter. Full and sliding +//! attention independently select NoPE or NeoX rotation; every layer has head-local q/k norms and +//! attention sinks. Routed blocks carry one ungated shared MLP whose width is explicit, not a +//! count of physical modules. Executable admission belongs to the common program/site registry. +pub mod host; mod hparams; +pub mod place; mod roles; pub mod spec; diff --git a/crates/model/src/arch/flint/place.rs b/crates/model/src/arch/flint/place.rs new file mode 100644 index 000000000..53a5006a7 --- /dev/null +++ b/crates/model/src/arch/flint/place.rs @@ -0,0 +1,478 @@ +//! Flint's validated description attached to the common hybrid placement and buffer owners. + +use gguf::Split; +use models::{Ffn, Mixer, ModelSpec}; + +use super::spec; +use crate::arch::{names, place_hybrid}; +use crate::placement::accepts::{Program, leg_down_forms}; +use crate::placement::kernels::attention::{Forms, Head128Scratch}; +use crate::placement::kernels::ffn::{PromptWalk, RoutedBatch, WideBlock, WideForms}; +use crate::placement::workstation::{TierBatchBytes, TierLegScratch}; +use crate::placement::{ModelTensors, PlacementError, kernels, workstation}; + +pub use place_hybrid::{KvLayout, PlaceError}; + +/// No second hyperparameter reader: the validated description owns Flint's facts. +pub type PlanInputs = place_hybrid::PlanInputs<()>; + +/// The common host union's prompt capacity, also the window ring's batch allowance. +pub const PROMPT_ROWS: usize = crate::moe::UNION_MAX_COLS; +/// Tokens served by one run of the common expert tier leg. +pub const TIER_CHUNK: u64 = 8; + +fn overflow() -> PlacementError { + PlacementError::Metadata { + key: "flint prompt reserve".into(), + detail: "buffer bytes overflow u64".into(), + } +} + +fn reads(model: &ModelTensors, name: &str) -> Result { + let tensor = model + .tensors + .iter() + .find(|t| t.name == name) + .ok_or_else(|| PlacementError::Tensor { + name: name.into(), + detail: "is not in the validated description".into(), + })?; + kernels::dense(tensor.ty) + .map(|site| site.reads()) + .ok_or_else(|| PlacementError::Tensor { + name: name.into(), + detail: format!("type {:?} has no dense projection site", tensor.ty), + }) +} + +/// The actual Q/K/V and output forms, widest rows and maximum partial segments of the shared +/// head128 attention scratch. NoPE changes no buffer width; windowing changes stores, not the +/// maximum flash launch scratch a full layer needs. +pub fn attention_scratch( + spec: &ModelSpec, + model: &ModelTensors, +) -> Result { + let mut scratch = Head128Scratch { + hidden: u64::from(spec.hidden), + query: 0, + key_value: 0, + heads: 0, + segments: 0, + input: Forms::default(), + output: Forms::default(), + }; + for (l, layer) in spec.layers.iter().enumerate() { + let Mixer::Gqa(g) = &layer.mixer else { + return Err(PlacementError::Metadata { + key: format!("layer {l} attention"), + detail: "Flint requires GQA".into(), + }); + }; + scratch.query = scratch + .query + .max(u64::from(g.heads) * u64::from(g.head_dim)); + scratch.key_value = scratch + .key_value + .max(u64::from(g.kv_heads) * u64::from(g.head_dim)); + scratch.heads = scratch.heads.max(u64::from(g.heads)); + scratch.segments = scratch + .segments + .max(g.window.map_or(80, |w| u64::from(w).div_ceil(64))); + for name in [names::attn_q(l), names::attn_k(l), names::attn_v(l)] { + scratch.input = scratch.input.with(reads(model, &name)?); + } + scratch.output = scratch.output.with(reads(model, &names::attn_output(l))?); + } + Ok(scratch) +} + +fn block_forms( + model: &ModelTensors, + names: [String; 3], + block: &mut WideBlock, +) -> Result<(), PlacementError> { + block.input = block + .input + .with(reads(model, &names[0])?) + .with(reads(model, &names[1])?); + block.down = block.down.with(reads(model, &names[2])?); + Ok(()) +} + +/// The common routed/shared batch with selected dense/shared forms. `card` prices the allocation +/// made when either resident card serves experts; a plan reserves that candidate before assigning +/// expert ids. `tiered` adds only the prompt join's three slot words. +pub fn routed_batch( + spec: &ModelSpec, + model: &ModelTensors, + rows: u64, + card: bool, + tiered: bool, +) -> Result { + let empty = WideBlock { + hidden: u64::from(spec.hidden), + ff: 0, + input: WideForms::default(), + down: WideForms::default(), + }; + let mut shared = empty; + let mut dense = empty; + let mut experts = 0; + let mut used = 0; + let mut expert_ff = 0; + for (l, layer) in spec.layers.iter().enumerate() { + match &layer.ffn { + Ffn::Dense { ff, .. } => { + dense.ff = dense.ff.max(u64::from(*ff)); + block_forms( + model, + [names::ffn_gate(l), names::ffn_up(l), names::ffn_down(l)], + &mut dense, + )?; + } + Ffn::Moe(moe) => { + experts = experts.max(u64::from(moe.experts)); + used = used.max(u64::from(moe.top_k)); + expert_ff = expert_ff.max(u64::from(moe.expert_ff)); + let sh = moe + .shared + .as_ref() + .ok_or_else(|| PlacementError::Metadata { + key: format!("layer {l} shared block"), + detail: "Flint requires one ungated shared MLP".into(), + })?; + shared.ff = shared.ff.max(u64::from(sh.ff)); + block_forms( + model, + [ + names::ffn_gate_shexp(l), + names::ffn_up_shexp(l), + names::ffn_down_shexp(l), + ], + &mut shared, + )?; + } + } + } + Ok(RoutedBatch { + hidden: u64::from(spec.hidden), + experts, + used, + shared, + dense: (dense.ff > 0).then_some(dense), + card: card.then_some(TierLegScratch { + n_embd: u64::from(spec.hidden), + ff: expert_ff, + n_used: used, + chunk: rows, + downs: leg_down_forms(Program::FlintBody, model), + input_scales: true, + }), + tiered, + }) +} + +/// Prompt reserve from the same descriptors the attention, FFN and walk allocate by. It includes +/// the candidate card batch, not a quantization padding estimate or a per-layer copy of scratch. +pub fn prompt_reserve_bytes( + spec: &ModelSpec, + model: &ModelTensors, + rows: u64, +) -> Result { + let attention = attention_scratch(spec, model)? + .bytes(rows) + .ok_or_else(overflow)?; + let walk = PromptWalk { + hidden: u64::from(spec.hidden), + sets: 1, + } + .bytes(rows) + .ok_or_else(overflow)?; + let ffn = routed_batch(spec, model, rows, true, false)? + .bytes(rows) + .ok_or_else(overflow)?; + attention + .checked_add(walk) + .and_then(|bytes| bytes.checked_add(ffn)) + .ok_or_else(overflow) +} + +/// Price the selected prompt group before borrowing the inputs for a plan. Reconfiguration is +/// idempotent: only unit rows multiply; attention, FFN and the host-sum upload stay shared. +pub fn configure_group(inputs: &mut PlanInputs, group: usize) -> Result<(), PlacementError> { + runtime::prompt::check_group(group, bloomery_levers::PREFILL_GROUP_MAX as usize).map_err( + |error| PlacementError::Metadata { + key: runtime::prompt::GROUP_LEVER.into(), + detail: error.to_string(), + }, + )?; + let rows = PROMPT_ROWS as u64; + let hidden = u64::from(inputs.spec.hidden); + let one = PromptWalk { hidden, sets: 1 } + .bytes(rows) + .ok_or_else(overflow)?; + let selected = PromptWalk { + hidden, + sets: runtime::prompt::group_sets(group) as u64, + } + .bytes(rows) + .ok_or_else(overflow)?; + let base = prompt_reserve_bytes(&inputs.spec, &inputs.model, rows)?; + inputs.prompt_reserve = base + .checked_sub(one) + .and_then(|bytes| bytes.checked_add(selected)) + .ok_or_else(overflow)?; + Ok(()) +} + +/// The real expert tier's staging, protected leg scratch and host exchanges for one prompt unit. +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let batch = routed_batch(&inputs.spec, &inputs.model, TIER_CHUNK, true, false)?; + let leg = batch.card.ok_or_else(overflow)?; + workstation::tier_leg_batch_bytes( + leg.n_embd, + leg.ff, + leg.n_used, + TIER_CHUNK, + PROMPT_ROWS as u64, + leg.downs, + true, + ) +} + +impl PlanInputs { + /// Describe once, then require the genuine program and its numerical/site bindings. + pub fn read(split: &Split) -> Result { + Self::describe(split)?.runnable() + } + + /// The validated file, common windowed stores, actual prompt reserve and tier loader contract. + /// The common host-run loader serves NVMe through the same residency arena that placement split. + pub fn describe(split: &Split) -> Result { + let read = spec::read(split)?; + let reserve = prompt_reserve_bytes(&read.spec, &read.tensors, PROMPT_ROWS as u64)?; + let batch = routed_batch(&read.spec, &read.tensors, PROMPT_ROWS as u64, false, true)?; + let tier = + kernels::ffn::tier_places_bytes(PROMPT_ROWS as u64, batch.used).ok_or_else(overflow)?; + Ok(Self::new("Flint", (), read.tensors, read.spec)? + .with_prompt_reserve(reserve) + .with_tier_reserve(tier) + .with_nvme_tier()) + } +} + +#[cfg(test)] +mod tests { + use super::super::tests::{check, keys, tensors}; + use super::*; + use crate::arch::synthetic::{V, header_shaped}; + + #[test] + fn description_uses_real_prompt_and_tier_terms_without_admitting_unsupported_shapes() { + let path = header_shaped( + "flint-place-description", + "flint", + &keys(), + &[("tokenizer.ggml.pre", V::Str("flint"))], + &tensors(), + ); + let split = Split::open(&path).unwrap(); + let mut inputs = PlanInputs::describe(&split).unwrap(); + assert_eq!( + inputs.prompt_reserve, + attention_scratch(&inputs.spec, &inputs.model) + .unwrap() + .bytes(PROMPT_ROWS as u64) + .unwrap() + + PromptWalk { + hidden: 16, + sets: 1 + } + .bytes(PROMPT_ROWS as u64) + .unwrap() + + routed_batch(&inputs.spec, &inputs.model, PROMPT_ROWS as u64, true, false) + .unwrap() + .bytes(PROMPT_ROWS as u64) + .unwrap() + ); + assert_eq!(inputs.tier_reserve, Some(12 * PROMPT_ROWS as u64 * 2)); + let tier = tier_batch(&inputs).unwrap(); + let leg = tier.leg.unwrap(); + assert!(leg.input_scales); + assert_eq!(leg.chunk, TIER_CHUNK); + assert_eq!(tier.scratch, leg.bytes(leg.downs).unwrap()); + assert!(matches!( + PlanInputs::read(&split), + Err(PlacementError::Unimplemented(_)) + )); + let one = inputs.prompt_reserve; + configure_group(&mut inputs, 2).unwrap(); + let grouped = inputs.prompt_reserve; + assert_eq!(grouped - one, 2 * PROMPT_ROWS as u64 * (8 * 16 + 8)); + configure_group(&mut inputs, 2).unwrap(); + assert_eq!(inputs.prompt_reserve, grouped); + configure_group(&mut inputs, 1).unwrap(); + assert_eq!(inputs.prompt_reserve, one); + assert!(configure_group(&mut inputs, 0).is_err()); + assert_eq!(inputs.prompt_reserve, one); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn released_window_geometry_counts_each_resident_sequence() { + let read = check("place-window-slots", &keys(), &tensors()).unwrap(); + let mut full = read.spec.layers[0].mixer.clone(); + let Mixer::Gqa(g) = &mut full else { + panic!("GQA"); + }; + g.heads = 32; + g.kv_heads = 4; + g.head_dim = 128; + g.value_dim = 128; + g.window = None; + let mut window = full.clone(); + let Mixer::Gqa(g) = &mut window else { + panic!("GQA"); + }; + g.window = Some(513); + let mixers: Vec<_> = (0..49) + .map(|l| if l < 14 { full.clone() } else { window.clone() }) + .collect(); + let kv = KvLayout::of(&mixers).unwrap(); + for ctx in [1, 512, 513, 1023, 1024, 1025, 4096] { + let expected = 2048 * (14 * ctx + 35 * ctx.min(1024)); + assert_eq!(kv.bytes(0..49, ctx), expected); + assert_eq!(kv.rows_at(14, ctx), ctx.min(1024)); + assert_eq!(kv.rows_at(0, ctx), ctx); + for slots in [1, 3] { + assert_eq!(kv.seq_terms().bytes(ctx, slots), slots * expected); + } + } + } +} + +#[cfg(test)] +mod planning_tests { + use super::super::tests::{check, keys, tensors}; + use super::*; + use crate::arch::synthetic::V; + use crate::placement::{PlanLevers, Role}; + + #[test] + fn selected_projection_forms_are_a_union_not_blanket_padding() { + let mut read = check("place-selected-forms", &keys(), &tensors()).unwrap(); + let plain = attention_scratch(&read.spec, &read.tensors).unwrap(); + assert_eq!(plain.input, Forms::default()); + assert_eq!(plain.output, Forms::default()); + for (name, ty) in [ + (names::attn_q(0), gguf::GgmlType::Q4_K), + (names::attn_k(1), gguf::GgmlType::Q8_0), + (names::attn_output(2), gguf::GgmlType::Q6_K), + ] { + read.tensors + .tensors + .iter_mut() + .find(|t| t.name == name) + .unwrap() + .ty = ty; + } + let mixed = attention_scratch(&read.spec, &read.tensors).unwrap(); + assert_eq!( + mixed.input, + Forms { + q128: true, + q32: true + } + ); + assert_eq!( + mixed.output, + Forms { + q128: true, + q32: false + } + ); + let rows = PROMPT_ROWS as u64; + assert_eq!( + mixed.bytes(rows).unwrap() - plain.bytes(rows).unwrap(), + mixed.input.bytes(rows, mixed.hidden).unwrap() + + mixed.output.bytes(rows, mixed.query).unwrap() + ); + } + + #[test] + fn nvme_loader_opt_in_preserves_unserved_loader_refusal_and_slot_kv() { + let mut keys = keys(); + for (key, value) in &mut keys { + *value = match *key { + "block_count" => V::U32(8), + "expert_count" => V::U32(256), + "expert_used_count" => V::U32(6), + "attention.sliding_window_pattern" => { + V::Bools((0..8).map(|l| l % 2 == 1).collect()) + } + "attention.rope_pattern" => V::Bools((0..8).map(|l| l % 2 == 0).collect()), + _ => continue, + }; + } + let mut ts = tensors(); + let template: Vec<_> = ts + .iter() + .filter(|(n, _)| n.starts_with("blk.1.")) + .cloned() + .collect(); + for l in 4..8 { + ts.extend( + template + .iter() + .map(|(n, dims)| (n.replacen("blk.1.", &format!("blk.{l}."), 1), dims.clone())), + ); + } + for (name, dims) in &mut ts { + if name.contains("_exps.weight") { + dims[2] = 256; + } else if name.ends_with("ffn_gate_inp.weight") { + dims[1] = 256; + } + } + let read = check("place-paging", &keys, &ts).unwrap(); + let mut inputs = PlanInputs::new("Flint", (), read.tensors, read.spec).unwrap(); + for tensor in &mut inputs.model.tensors { + if tensor.role == Role::RoutedExperts { + tensor.dims[1] *= 4096; + tensor.file_bytes *= 4096; + } + } + inputs.prompt_reserve = + prompt_reserve_bytes(&inputs.spec, &inputs.model, PROMPT_ROWS as u64).unwrap(); + inputs.room = (u64::MAX, workstation::HostRead::Given); + let machine = workstation::plan_gate(8); + let levers = PlanLevers::default(); + let (need, held) = { + let ample = inputs.plan(&machine, 2048, &levers).unwrap(); + ( + workstation::HostNeed::of(&le, 0).bytes(), + ample.host.expert_bytes, + ) + }; + inputs.room.0 = need - held / 8; + assert!(matches!(inputs.plan(&machine, 2048, &levers), + Err(PlaceError::NoNvmeTier { paged, .. }) if paged > 0)); + let inputs = inputs.with_nvme_tier(); + for slots in [1, 3] { + let plan = inputs + .plan_with_slots( + &machine, + 2048, + &levers, + std::num::NonZeroUsize::new(slots).unwrap(), + ) + .unwrap(); + assert!(plan.host.nvme_expert_bytes > 0); + assert_eq!(plan.host.expert_bytes + plan.host.nvme_expert_bytes, held); + assert_eq!( + plan.cards.iter().map(|c| c.kv_bytes).sum::(), + slots as u64 * inputs.kv.bytes(0..8, 2048) + ); + } + } +} diff --git a/crates/model/src/arch/flint/tests.rs b/crates/model/src/arch/flint/tests.rs index 698806d85..0ec55ee4c 100644 --- a/crates/model/src/arch/flint/tests.rs +++ b/crates/model/src/arch/flint/tests.rs @@ -3,7 +3,7 @@ use crate::arch::synthetic::{V, header_shaped}; use crate::placement::PlacementError; use models::{Ffn, Mixer, Need, RopeMode}; -fn keys() -> Vec<(&'static str, V)> { +pub(super) fn keys() -> Vec<(&'static str, V)> { vec![ ("block_count", V::U32(4)), ("embedding_length", V::U32(16)), @@ -36,7 +36,7 @@ fn keys() -> Vec<(&'static str, V)> { ] } -fn tensors() -> Vec<(String, Vec)> { +pub(super) fn tensors() -> Vec<(String, Vec)> { let mut out = vec![ ("token_embd.weight".into(), vec![16, 2]), ("output.weight".into(), vec![16, 2]), @@ -76,7 +76,7 @@ fn tensors() -> Vec<(String, Vec)> { out } -fn check( +pub(super) fn check( tag: &str, keys: &[(&str, V)], tensors: &[(String, Vec)], @@ -162,12 +162,6 @@ fn masks_are_independent_and_nope_still_requires_head_norm() { .count(), 4 ); - assert!(crate::arch::coverage::program_of(r.spec.arch).is_none()); - assert!( - crate::arch::coverage::check(&r.spec, &r.tensors) - .iter() - .any(|item| item.feature == Need::Program("flint").to_string()) - ); } #[test] diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index d497ca063..670d34f98 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -146,7 +146,6 @@ fn down_forms(downs: impl Iterator) -> DownForms { .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)) } - /// Tokens a chunk of the tier's prompt-batch service runs at most: the /// stage card's chunk of the card experts' launches, which the body binds /// to its own constant. @@ -166,6 +165,7 @@ pub fn tier_batch(inputs: &PlanInputs) -> Result TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, crate::placement::accepts::leg_down_forms(Program::Glm5nextBody, &inputs.model), + false, ) } @@ -421,7 +421,6 @@ pub enum PlaceError { /// A plan of no resident sequence: a load serves at least one. #[error("a plan of {slots} resident sequences: a load serves at least one")] Slots { slots: usize }, - } impl PlanRefusal for PlaceError { @@ -692,7 +691,6 @@ impl PlanInputs { } Ok(plan) } - } /// Card bytes the NextN walk's arena may hold beside the plan's tensors @@ -1886,7 +1884,7 @@ mod tests { use GgmlType::{IQ3_S, IQ4_XS, NVFP4, Q4_K, Q5_K, Q6_K, Q8_0}; let q8 = DownForms::of(DownAct::Q8Act); let b32 = DownForms::of(DownAct::Q8Blocks32); - let base = tier_batch(&inputs()); + let base = tier_batch(&inputs()).expect("the described tier geometry"); let hp = inputs().hp; let leg = |forms| { leg_bytes( @@ -1899,7 +1897,6 @@ mod tests { }; assert_ne!(leg(q8), leg(b32)); for (layers, forms) in [ - (vec![], q8), (vec![[Q4_K, Q4_K, Q5_K]], q8), (vec![[Q5_K, Q5_K, Q6_K]], q8), (vec![[IQ3_S, IQ3_S, IQ4_XS]], q8), @@ -1914,7 +1911,7 @@ mod tests { q8.with(DownAct::Q8Blocks32), ), ] { - let batch = tier_batch(&routed_types(&layers)); + let batch = tier_batch(&routed_types(&layers)).expect("the selected tier geometry"); assert_eq!(batch.scratch, leg(forms), "{layers:?}"); assert_eq!(batch.staging, base.staging, "{layers:?}"); assert_eq!(batch.host, base.host, "{layers:?}"); diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 1b994b4ae..4832225ce 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -121,7 +121,6 @@ pub const TIER_CHUNK: u64 = 16; /// plan's rule carries reads (the rule's down arm is the leg's `Q8Act` downs), and the host's rows /// and places ([`workstation::tier_leg_batch_bytes`]). The plan reserves them on the tier card and /// the host, and the load checks its allocations against them. -#[must_use] pub fn tier_batch(hp: &Hparams, cols: u64) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, @@ -130,6 +129,7 @@ pub fn tier_batch(hp: &Hparams, cols: u64) -> Result Result { let hp = &inputs.hp; workstation::tier_leg_batch_bytes( @@ -144,6 +143,7 @@ pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result usize { /// placed pass's rows ([`pass_rows_max`]): the experts run in the latent, so /// the tier's rows and its staging are the latent's width, through the card /// leg in chunks of [`TIER_CHUNK`]. -#[must_use] pub fn tier_batch(hp: &Hparams) -> Result { let ff = hp.expert_width(); workstation::tier_leg_batch_bytes( @@ -86,6 +85,7 @@ pub fn tier_batch(hp: &Hparams) -> Result { TIER_CHUNK, pass_rows_max(hp.n_used) as u64, crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act), + false, ) } diff --git a/crates/model/src/arch/names.rs b/crates/model/src/arch/names.rs index 3179eb093..bd531bdf8 100644 --- a/crates/model/src/arch/names.rs +++ b/crates/model/src/arch/names.rs @@ -37,6 +37,12 @@ pub fn attn_norm(l: usize) -> String { blk(l, "attn_norm.weight") } +/// `blk.{l}.attn_sinks.weight` — one learned sink logit per query head. +#[must_use] +pub fn attn_sinks(l: usize) -> String { + blk(l, "attn_sinks.weight") +} + /// `blk.{l}.attn_q.weight` — the query projection, every head. #[must_use] pub fn attn_q(l: usize) -> String { diff --git a/crates/model/src/arch/place_hybrid.rs b/crates/model/src/arch/place_hybrid.rs index 10ca6c063..44831ac72 100644 --- a/crates/model/src/arch/place_hybrid.rs +++ b/crates/model/src/arch/place_hybrid.rs @@ -9,9 +9,9 @@ //! forces: the keys it reads, the layers it refuses, the engine's name in the refusals that name it. //! //! The plan refuses by name a machine that hangs an expert tier card where the family's body serves -//! none ([`PlanInputs::tier_reserve`]), a context of no position and a host room that cannot hold -//! the host experts: the bodies this plans serve no NVMe expert tier, so a plan that pages one -//! ([`PlaceError::NoNvmeTier`]) would load experts nothing serves. +//! none ([`PlanInputs::tier_reserve`]), a context of no position and, unless the loader explicitly +//! serves the common NVMe tier ([`PlanInputs::with_nvme_tier`]), a host room that pages any expert +//! ([`PlaceError::NoNvmeTier`]). //! //! A body that makes buffers beside the plan's terms on its card (the prompt batch's rows) states //! their bytes in [`PlanInputs::prompt_reserve`]: the expert rule spreads within the card's budget @@ -72,7 +72,10 @@ pub const fn wide_bytes(cols: usize, k: usize) -> usize { /// (a [`GEMM_TILE_COLS`] column tile each, and the expert's), the two counts and the all-zero ids. #[must_use] pub const fn route_bytes(cols: usize) -> usize { - 4 * (cols + 2 * (cols / GEMM_TILE_COLS + 1) + 2 + cols) + match kernels::ffn::dense_route_bytes(cols as u64) { + Some(bytes) => bytes as usize, + None => panic!("route_bytes: route table bytes overflow u64"), + } } /// The forms a ring pass's route GEMMs read their rows in, by the stacks' file types: the gate's and @@ -196,7 +199,10 @@ pub fn stream_ring_bytes( /// the copy of the slot's route weight, three four-byte words. #[must_use] pub const fn tier_places_bytes(rows: usize, n_used: usize) -> u64 { - 3 * 4 * (rows * n_used) as u64 + match kernels::ffn::tier_places_bytes(rows as u64, n_used as u64) { + Some(bytes) => bytes, + None => panic!("tier_places_bytes: slot bytes overflow u64"), + } } /// What a plan of a hybrid file is made from, read from its headers. `H` is the family's own @@ -213,9 +219,9 @@ pub struct PlanInputs { pub kv: KvLayout, /// The host's available bytes when the inputs were read, with the reading that decided /// ([`workstation::host_available_read`]): the routed experts the room cannot hold follow it to - /// the NVMe tier ([`placement::expert_nvme_tier`]), which these bodies do not serve, so such a - /// plan is refused ([`PlaceError::NoNvmeTier`]). A gate that plans an arm by a room of its own - /// sets it, read [`HostRead::Given`]. + /// the NVMe tier ([`placement::expert_nvme_tier`]). A body without that loader contract refuses + /// a plan that pages experts ([`PlaceError::NoNvmeTier`]). A gate that plans an arm by a room of + /// its own sets it, read [`HostRead::Given`]. /// /// The effective room is [`Plan::plan_at`]'s: the reading's plan room, /// capped by the machine's host room and unified pool. @@ -233,6 +239,9 @@ pub struct PlanInputs { /// keeps `bytes` more than [`PlanInputs::prompt_reserve`] on a machine that names a tier card, /// the buffers only a tiered load makes. pub tier_reserve: Option, + /// Whether the loader serves the common host-run NVMe arena. Opt-in only: a plan cannot + /// promise paging merely because the placement can split the expert stacks. + nvme_tier: bool, /// The engine the refusals that name one name. engine: &'static str, } @@ -357,6 +366,7 @@ impl PlanInputs { arena, prompt_reserve: 0, tier_reserve: None, + nvme_tier: false, engine, }) } @@ -391,6 +401,16 @@ impl PlanInputs { } } + /// `self` for a loader that serves the common host-run NVMe arena, with the same residency + /// identity and split as the plan. Existing loaders remain non-paging unless they opt in. + #[must_use] + pub fn with_nvme_tier(self) -> PlanInputs { + PlanInputs { + nvme_tier: true, + ..self + } + } + /// `self` with each delta-rule layer's state `lanes` lanes ([`KvLayout::with_lanes`]): the /// stores of a load that verifies `lanes` rows a pass. #[must_use] @@ -457,8 +477,8 @@ impl PlanInputs { /// /// The host experts are held to the room the inputs name ([`PlanInputs::room`]): a room under /// the host need runs the common NVMe tier split ([`placement::expert_nvme_tier`], which refuses - /// a room under the tier's floor by name), and a plan that comes out paging any routed expert is - /// refused ([`PlaceError::NoNvmeTier`]), for the body serves no such tier. + /// a room under the tier's floor by name). A plan paging any routed expert is refused + /// ([`PlaceError::NoNvmeTier`]) unless the loader serves it ([`PlanInputs::with_nvme_tier`]). pub fn plan_with_slots<'a>( &'a self, machine: &'a Machine, @@ -485,7 +505,7 @@ impl PlanInputs { let need = HostNeed::of(&plan, 0).bytes(); let room = plan.plan_at(&self.room); placement::expert_nvme_tier(&mut plan, room, self.arena)?; - if plan.host.nvme_expert_bytes > 0 { + if plan.host.nvme_expert_bytes > 0 && !self.nvme_tier { return Err(PlaceError::NoNvmeTier { paged: plan.host.nvme_expert_bytes, need, diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index 98c788951..416108150 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1477,7 +1477,6 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// card's staging, one chunk of the leg's scratch and the host's rows, for /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. -#[must_use] pub fn leg_tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, @@ -1487,6 +1486,7 @@ pub fn leg_tier_batch(hp: &Hparams) -> Result { crate::moe::UNION_MAX_COLS as u64, crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act) .with(crate::placement::kernels::DownAct::Q8Blocks32), + false, ) } diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 4926c5462..a48fc5e6d 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -45,7 +45,6 @@ pub const TIER_CHUNK: u64 = 8; /// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. -#[must_use] pub fn tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, @@ -55,6 +54,7 @@ pub fn tier_batch(hp: &Hparams) -> Result { crate::moe::UNION_MAX_COLS as u64, crate::placement::kernels::DownForms::of(crate::placement::kernels::DownAct::Q8Act) .with(crate::placement::kernels::DownAct::Q8Blocks32), + false, ) } diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 90e91e2dc..6ce4aaadd 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -54,6 +54,7 @@ pub fn tier_batch(inputs: &PlanInputs) -> Result TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64, leg_down_forms(Program::Solar2Body, &inputs.model), + false, ) } @@ -137,6 +138,7 @@ pub fn prompt_reserve_bytes( n_used: experts.n_used as u64, chunk: r as u64, downs, + input_scales: false, } .bytes(downs)?; ((4 * f32s + acts + wide + 2 * route_bytes(r)) as u64) diff --git a/crates/model/src/moe.rs b/crates/model/src/moe.rs index 44725a667..312edf96b 100644 --- a/crates/model/src/moe.rs +++ b/crates/model/src/moe.rs @@ -22,7 +22,7 @@ //! the weight type's and nothing else. `crate::ops::matmul_q` is that single owner. use std::cell::RefCell; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering::Relaxed}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, Ordering::Relaxed}; use std::sync::{Mutex, PoisonError}; use std::time::Instant; @@ -30,9 +30,9 @@ use gguf::{GgmlType, Gguf, Split, TensorInfo}; use crate::ModelError; use crate::ops::{ - self, DEFER_MAX_COLS, ExpertForm, ExpertStack, GroupClaims, GroupInput, Lanes, PostNorm, - R8Stack, RowLayout, ShardTensor, StackTier, Tensor2, Tensor2View, UnionCall, UnionPlanView, - UnionSlabs, UnionStack, WarmSpan, Weight, defers, matmul_q, matmul_q_group, + self, DEFER_MAX_COLS, ExpertForm, ExpertInput, ExpertStack, GroupClaims, GroupInput, Lanes, + PostNorm, R8Stack, RowLayout, ShardTensor, StackTier, Tensor2, Tensor2View, UnionCall, + UnionPlanView, UnionSlabs, UnionStack, WarmSpan, Weight, defers, matmul_q, matmul_q_group, matmul_q_group_into_on, matmul_q_group_swiglu, matmul_q_group_swiglu_into_on, }; use crate::profile; @@ -875,6 +875,8 @@ pub struct HostScratch { pars: Vec, /// The down outputs, `{embd, 1}` each. downs: Vec, + /// Per-expert activation scales, restored before weighting the down outputs. + scales: Vec, /// The call's gate and up matrices, interleaved; empty between calls. gu_w: Vec>, /// The call's down matrices; empty between calls. @@ -910,6 +912,7 @@ impl HostScratch { gate_up: blocks(2 * n_used, ff), pars: blocks(n_used, ff), downs: blocks(n_used, embd), + scales: Vec::new(), gu_w: Vec::with_capacity(2 * n_used), down_w: Vec::with_capacity(n_used), xs: Vec::with_capacity(2 * n_used), @@ -918,6 +921,23 @@ impl HostScratch { }) } + /// Allocate protection scales at load; direct quantization needs none. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + match policy { + ExpertInput::Direct => self.scales = Vec::new(), + ExpertInput::AmaxF16 => self.scales.resize(self.n_used, 1.0), + } + } + + /// The policy this scratch has load-allocated scale storage for. + pub fn expert_input(&self) -> ExpertInput { + if self.scales.is_empty() { + ExpertInput::Direct + } else { + ExpertInput::AmaxF16 + } + } + fn fits(&self, embd: usize, ff: usize) -> bool { (self.embd, self.ff) == (embd, ff) } @@ -961,6 +981,7 @@ impl HostScratch { #[derive(Clone, Copy)] struct LegRun<'n> { form: ExpertForm, + input: ExpertInput, lanes: Lanes, post: Option<&'n PostNorm>, } @@ -971,6 +992,7 @@ impl LegRun<'_> { fn flat() -> LegRun<'static> { LegRun { form: ExpertForm::Gated { limit: 0.0 }, + input: ExpertInput::Direct, lanes: Lanes::Flat, post: None, } @@ -1042,11 +1064,18 @@ fn serve_resolved( s: &mut HostScratch, ) -> Result<(), ModelError> { let n = experts.len(); + if run.input == ExpertInput::AmaxF16 && s.scales.len() < n { + return Err(ModelError::Metadata { + key: "host expert input".to_string(), + detail: "AmaxF16 requires HostScratch::set_expert_input at load".to_string(), + }); + } let lvl = profile::level(); let HostScratch { gate_up, pars, downs, + scales, xs, srcs, .. @@ -1064,33 +1093,55 @@ fn serve_resolved( *xs = relist(xl); let gt = gt?; let mut sl: Vec> = relist(std::mem::take(srcs)); - match run.form { - ExpertForm::Gated { limit } => { - for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { - sl.push(GroupInput::SwigluClamp(gate, up, limit)); - } + let dt = if run.input == ExpertInput::AmaxF16 { + for i in 0..n { + let (gate, up) = match run.form { + ExpertForm::Gated { .. } | ExpertForm::GatedOai { .. } => { + (gate_up[2 * i].col(0), gate_up[2 * i + 1].col(0)) + } + ExpertForm::Relu2 => (&[][..], gate_up[i].col(0)), + }; + run.form.combine_into(gate, up, pars[i].col_mut(0)); + scales[i] = run.input.protect(pars[i].col_mut(0)); } - ExpertForm::GatedOai { alpha, limit } => { - for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { - sl.push(GroupInput::SwigluOai(gate, up, alpha, limit)); + sl.extend(pars[..n].iter().map(GroupInput::Ready)); + matmul_q_group_swiglu_into_on(run.lanes, "host_down", down, &sl, &mut downs[..n], &mut []) + } else { + match run.form { + ExpertForm::Gated { limit } => { + for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { + sl.push(GroupInput::SwigluClamp(gate, up, limit)); + } } - } - ExpertForm::Relu2 => { - for up in &gate_up[..n] { - sl.push(GroupInput::Relu2(up)); + ExpertForm::GatedOai { alpha, limit } => { + for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { + sl.push(GroupInput::SwigluOai(gate, up, alpha, limit)); + } + } + ExpertForm::Relu2 => { + for up in &gate_up[..n] { + sl.push(GroupInput::Relu2(up)); + } } } - } - let dt = matmul_q_group_swiglu_into_on( - run.lanes, - "host_down", - down, - &sl, - &mut downs[..n], - &mut pars[..n], - ); + matmul_q_group_swiglu_into_on( + run.lanes, + "host_down", + down, + &sl, + &mut downs[..n], + &mut pars[..n], + ) + }; *srcs = relist(sl); let dt = dt?; + if run.input == ExpertInput::AmaxF16 { + for (d, &scale) in downs[..n].iter_mut().zip(&scales[..n]) { + for v in d.col_mut(0) { + *v *= scale; + } + } + } if let ExpertForm::Relu2 = run.form { // Back to back from the top: expert `i`'s up goes to slot `2 * i + 1`, // where `HostScratch::up` reads it, and no block moves twice. @@ -1430,6 +1481,8 @@ pub struct UnionScratch { qc: Vec, /// Slot `q`'s down output at `store[q·embd..(q + 1)·embd]`. store: Vec, + /// Per-slot activation scales; claims publish them before the down reads. + scales: Vec, plan: UnionPlan, /// A narrow call's claim state per distinct expert: /// `min(max_cols, DEFER_MAX_COLS) · per_list`, every expert such a call @@ -1550,6 +1603,7 @@ impl UnionScratch { gu: vec![0.0; 2 * slots * ff], qc: vec![0; slots * c_col], store: vec![0.0; slots * embd], + scales: Vec::new(), plan: UnionPlan::with_room(cols, per_list), claims: (0..cols.min(DEFER_MAX_COLS) * per_list) .map(|_| AtomicU8::new(0)) @@ -1559,6 +1613,16 @@ impl UnionScratch { } } + /// Allocate protection scales at load; direct quantization needs none. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + self.scales = match policy { + ExpertInput::Direct => Vec::new(), + ExpertInput::AmaxF16 => (0..self.max_slots) + .map(|_| AtomicU32::new(1.0f32.to_bits())) + .collect(), + }; + } + /// The most columns a call through this scratch takes. pub fn max_cols(&self) -> usize { self.max_cols @@ -1866,6 +1930,7 @@ fn serve_batched<'w>( gu, qc, store, + scales, passes, batches, .. @@ -1886,7 +1951,8 @@ fn serve_batched<'w>( }; let whole = UnionCall::new(plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let mut xqs = whole.quantize_batched(xq, xq_up, passes); slot.clear(); slot.resize(plan.n(), 0); @@ -1927,7 +1993,8 @@ fn serve_batched<'w>( } let batch_call = UnionCall::new(sub.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; batch_call.run_batch(&mut xqs, gu, qc, &mut store[base * embd..], None, passes)?; base += sub.cols.len(); drop(pin); @@ -1982,6 +2049,7 @@ fn serve_step_split<'w>( gu, qc, store, + scales, passes, batches, claims, @@ -1998,7 +2066,8 @@ fn serve_step_split<'w>( let (order, ids) = (&*order, &*ids); let whole = UnionCall::new(plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let deferred = defers(x.ne1()); let mut xqs = if deferred { whole.x_rooms(xq, xq_up) @@ -2032,7 +2101,8 @@ fn serve_step_split<'w>( } let call = UnionCall::new(sub.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let claim = deferred.then(|| GroupClaims { x: g == 0, states: &mut claims[..], @@ -2069,7 +2139,8 @@ fn serve_planned( } let call = UnionCall::new(s.plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, &s.scales)?; let slabs = UnionSlabs { xq: &mut s.xq, xq_up: &mut s.xq_up, @@ -2637,6 +2708,7 @@ pub struct HostLayer { /// made for it ([`HostScratch::new`]). ff: usize, form: ExpertForm, + input: ExpertInput, tier: Option, /// The norm each expert's down row takes before its weight /// ([`HostLayer::set_post_norm`]); `None`, the plain weighted sum. @@ -2798,11 +2870,23 @@ impl HostLayer { n_expert, ff, form, + input: ExpertInput::Direct, tier: None, post: None, }) } + /// Select the activated expert input policy for every host serving path. + /// This is a load-time setting; the default preserves direct quantization. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + self.input = policy; + } + + /// The loaded activated-expert input policy. + pub fn expert_input(&self) -> ExpertInput { + self.input + } + /// Give the layer the norm each routed expert's down row takes before /// its router weight ([`PostNorm`]): from then on every call that serves /// the layer — the one-column leg, the union and the step union, from the @@ -3091,6 +3175,7 @@ impl HostLayer { }; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3158,6 +3243,7 @@ impl HostLayer { tier.lend(&mut stacks); let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3213,6 +3299,7 @@ impl HostLayer { )?; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3284,6 +3371,7 @@ impl HostLayer { let mut stacks = self.stacks_of(split)?; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; diff --git a/crates/model/src/ops.rs b/crates/model/src/ops.rs index 298985a72..7fbd55856 100644 --- a/crates/model/src/ops.rs +++ b/crates/model/src/ops.rs @@ -5116,6 +5116,52 @@ pub enum ExpertForm { Relu2, } +/// How an activated routed expert enters its down projection. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ExpertInput { + /// Quantize the activation unchanged. + #[default] + Direct, + /// Rescale each expert/token activation for F16 activation-block sums, + /// then restore the down output before applying its router weight. + AmaxF16, +} + +impl ExpertInput { + /// Divide one activated expert/token column by the oracle's scale. + /// The returned factor restores its down output. + pub(crate) fn protect(self, h: &mut [f32]) -> f32 { + match self { + ExpertInput::Direct => 1.0, + ExpertInput::AmaxF16 => { + let mut amax = 0.0f32; + for &v in h.iter() { + assert!(v.is_finite(), "expert input: non-finite activation {v}"); + amax = amax.max(v.abs()); + } + let exponent = ((amax / 1024.0).ln() * std::f32::consts::LOG2_E) + .ceil() + .clamp(0.0, 127.0); + let scale = (exponent * std::f32::consts::LN_2).exp(); + for v in h { + *v /= scale; + } + scale + } + } + } +} + +impl ExpertForm { + pub(crate) fn combine_into(self, gate: &[f32], up: &[f32], out: &mut [f32]) { + match self { + ExpertForm::Gated { limit } => swiglu_into(gate, up, Some(limit), out), + ExpertForm::GatedOai { alpha, limit } => swiglu_oai_into(gate, up, alpha, limit, out), + ExpertForm::Relu2 => relu2_into(up, out), + } + } +} + /// A layer's norm of each routed expert's down row before the row's router /// weight applies: the row scaled by `gain`, then divided by the root of its /// mean square plus `eps` (the card's `gain_rms_norm`). An architecture whose @@ -5375,6 +5421,7 @@ struct Combines<'c> { gu: &'c [f32], ff: usize, form: ExpertForm, + scales: Option<&'c [std::sync::atomic::AtomicU32]>, ty: GgmlType, qc: SharedQuantCols, } @@ -5413,6 +5460,10 @@ impl Combines<'_> { relu2_into(up, par); } } + if let Some(scales) = self.scales { + let scale = ExpertInput::AmaxF16.protect(par); + scales[q].store(scale.to_bits(), std::sync::atomic::Ordering::Relaxed); + } // SAFETY: the caller owns slot `q` of `qc`. unsafe { self.qc.quantize_into(self.ty, self.ff, par, q) }; }); @@ -5824,6 +5875,7 @@ pub(crate) struct UnionCall<'c, 'w> { /// The norm each down takes before its weight in the sums, when the /// layer has one. post: Option<&'c PostNorm>, + scales: Option<&'c [std::sync::atomic::AtomicU32]>, } impl<'c, 'w> UnionCall<'c, 'w> { @@ -5861,6 +5913,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { down, spread: Lanes::Flat, post: None, + scales: None, }) } @@ -5877,6 +5930,28 @@ impl<'c, 'w> UnionCall<'c, 'w> { UnionCall { post, ..self } } + /// Apply the loaded activation policy, using load-allocated slot scales. + pub(crate) fn expert_input( + self, + input: ExpertInput, + scales: &'c [std::sync::atomic::AtomicU32], + ) -> Result { + let scales = match input { + ExpertInput::Direct => None, + ExpertInput::AmaxF16 => { + if scales.len() < self.plan.slots() { + return Err(crate::ModelError::Metadata { + key: "host union expert input".to_string(), + detail: "AmaxF16 requires UnionScratch::set_expert_input at load" + .to_string(), + }); + } + Some(scales) + } + }; + Ok(UnionCall { scales, ..self }) + } + /// The cells the gate/up pass writes: two columns a slot gated — the /// gate's and the up's — one a slot for a relu² layer, the up's alone. fn gu_cells(&self) -> usize { @@ -6215,6 +6290,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { cb: self.down.cb, ptr: qc.as_mut_ptr(), }, + scales: self.scales, } } @@ -6299,6 +6375,14 @@ impl<'c, 'w> UnionCall<'c, 'w> { let mut pacc = profile::CallAcc::new(); let t0 = if lvl > 0 { Some(Instant::now()) } else { None }; run_row_pool(&set, lvl, &mut pacc, true, self.spread)?; + if let Some(scales) = self.scales { + for (row, scale) in store.chunks_exact_mut(self.down.n).zip(scales) { + let scale = f32::from_bits(scale.load(std::sync::atomic::Ordering::Relaxed)); + for v in row { + *v *= scale; + } + } + } let claim_ns = claims.claim_ns.into_inner(); if let Some(t0) = t0 { let wall = t0.elapsed().as_nanos() as u64; @@ -6757,6 +6841,35 @@ mod tests { }; use gguf::{GgmlType, Gguf}; + #[test] + fn expert_range_protection_preserves_values_and_bounds_large_activated_inputs() { + use super::ExpertInput; + for peak in [0.0f32, 1024.0, 1024.125, 2048.0, 2048.25, 1.0e8] { + let original = [peak, -peak * 0.5, peak * 0.25]; + let mut direct = original; + assert_eq!(ExpertInput::Direct.protect(&mut direct), 1.0); + assert_eq!(direct, original); + let mut protected = original; + let scale = ExpertInput::AmaxF16.protect(&mut protected); + assert!( + protected + .iter() + .all(|v| v.abs() <= 1024.0 * (1.0 + 4.0 * f32::EPSILON)) + ); + if peak <= 1024.0 { + assert_eq!(scale, 1.0); + assert_eq!(protected, original); + } else { + assert!(scale > 1.0); + for (actual, expected) in protected.into_iter().zip(original) { + assert!( + (actual * scale - expected).abs() <= expected.abs() * 2.0 * f32::EPSILON + ); + } + } + } + } + /// A GGUF v3 file holding one `[k, 2]` tensor `w` of type `ty` over zero bytes — /// the least the strict reader opens. fn one_tensor_file(ty: GgmlType, k: usize) -> std::path::PathBuf { diff --git a/crates/model/tests/flint_meta.rs b/crates/model/tests/flint_meta.rs index bde81addf..04887c059 100644 --- a/crates/model/tests/flint_meta.rs +++ b/crates/model/tests/flint_meta.rs @@ -106,30 +106,46 @@ fn hw_flint_header() { .filter(|(n, _)| matches!(n, Need::Gqa { sinks: true, .. })) .count(); assert_eq!((norms, rotary, sinks), (49, 35, 49)); - let coverage = model::arch::coverage::check(s, &r.tensors); - for (need, layer) in &needs { + for (need, _) in &needs { if let Need::Router(shape) = need { let binding = model::arch::models::shape::select_router(*shape).unwrap(); - assert_eq!(binding.body, model::arch::models::shape::RouterBody::SigmoidFloor); + assert_eq!( + binding.body, + model::arch::models::shape::RouterBody::SigmoidFloor + ); assert_eq!(binding.rule, shape.rule); - assert_eq!((binding.experts(), binding.top_k_min, binding.top_k_max), (256, 6, 6)); - assert!(coverage.iter().any(|u| { - u.layer == layer.map(|l| l as usize) && u.feature == need.to_string() - })); + assert_eq!( + (binding.experts(), binding.top_k_min, binding.top_k_max), + (256, 6, 6) + ); } } - assert!( - coverage - .iter() - .any(|u| u.feature == Need::Program("flint").to_string()) - ); - assert!(model::arch::coverage::program_of(s.arch).is_none()); let bytes: u64 = r.tensors.tensors.iter().map(|t| t.file_bytes).sum(); assert_eq!(bytes, 25_049_218_176); + let mut inputs = model::arch::flint::place::PlanInputs::read(&split) + .expect("the validated header has genuine program and site bindings"); + inputs.room = (u64::MAX, model::placement::workstation::HostRead::Given); + let machine = model::placement::workstation::plan_gate(s.layers.len()); + let levers = model::placement::PlanLevers::default(); + for ctx in [512, 1024, 4096] { + for slots in [1, 3] { + let plan = inputs + .plan_with_slots( + &machine, + ctx, + &levers, + std::num::NonZeroUsize::new(slots).unwrap(), + ) + .expect("the real mixed-quant header plans with resident slots"); + let cache = slots as u64 * 2048 * (14 * ctx + 35 * ctx.min(1024)); + assert_eq!(plan.cards.iter().map(|c| c.kv_bytes).sum::(), cache); + assert!(inputs.prompt_batch_bytes(&machine).unwrap() > 0); + } + } + let tier = model::arch::flint::place::tier_batch(&inputs).unwrap(); + assert!(tier.leg.unwrap().input_scales); + assert!(tier.card().unwrap() > 0); println!( "flint header_sha={HEADER_SHA} tensors=779 layers=49 dense=2 routed=47 norms={norms} rotary={rotary} sinks={sinks} payload_bytes={bytes}" ); - for item in coverage { - println!("unimplemented layer={:?} {}", item.layer, item.feature); - } } diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index 972d82125..71b1e998f 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -529,7 +529,10 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); let machine = tiered( - Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.hp).expect("tier batch bytes"), + Q36_LAYERS, + stage, + tier, + q35::leg_tier_batch(&inputs.hp).expect("tier batch bytes"), ); let plan = inputs.plan_rule( &machine, @@ -546,7 +549,10 @@ fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place fn qwen3_30b_q8_bp_tier_experts() { let inputs = qwen3_inputs(); let machine = tiered( - Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp).expect("tier batch bytes"), + Q3_LAYERS, + A6000, + RTX_3090, + q3::tier_batch(&inputs.hp).expect("tier batch bytes"), ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) diff --git a/crates/models/src/shape.rs b/crates/models/src/shape.rs index 6fd7f1592..8680797c4 100644 --- a/crates/models/src/shape.rs +++ b/crates/models/src/shape.rs @@ -934,8 +934,8 @@ pub fn gqa_row(head: u32, group: u32) -> Option { #[cfg(test)] mod tests { use super::{ - AttnShape, GQA, GroupRule, LANE_PICKS, MoeShape, ROUTERS, Refusal, RouterInst, RouterRule, - Shape, gqa_row_v, router_row, router_row_is, router_row_k, rules, select_gqa, + AttnShape, GQA, GroupRule, LANE_PICKS, MoeShape, ROUTERS, Refusal, RouterBody, RouterInst, + RouterRule, Shape, gqa_row_v, router_row, router_row_is, router_row_k, rules, select_gqa, select_router, }; use crate::{Act, Moe, Router, RouterNorm, Score, Shared}; @@ -1022,11 +1022,13 @@ mod tests { #[test] fn floor_top_six_serves_both_bias_modes_and_only_its_shape() { for bias in [false, true] { - let rule = RouterRule { bias, ..rules::SIGMOID_FLOOR }; + let rule = RouterRule { + bias, + ..rules::SIGMOID_FLOOR + }; let row = select_router(moe(rule, 256, 6)).unwrap(); assert_eq!(row.body, RouterBody::SigmoidFloor); assert_eq!(row.rule, rule); - assert_eq!(row, router_row_rule(RouterBody::SigmoidFloor, rule, 256, 6)); for top_k in [0, 5, 7, 8, 257] { assert_eq!( select_router(moe(rule, 256, top_k)).unwrap_err().why, @@ -1038,7 +1040,11 @@ mod tests { Refusal::RouterWidth { ref served } if served == &[256] )); for (score, gated) in [(Score::Softmax, false), (Score::Sigmoid, true)] { - let wrong = RouterRule { score, gated, ..rule }; + let wrong = RouterRule { + score, + gated, + ..rule + }; assert_eq!( select_router(moe(wrong, 256, 6)).unwrap_err().why, Refusal::NoRouterRule @@ -1048,10 +1054,16 @@ mod tests { let legacy = select_router(moe(rules::BIASED_SIGMOID, 256, 6)).unwrap(); assert_eq!(legacy.body, RouterBody::BiasedSigmoid); assert_eq!( - select_router(moe(RouterRule { - norm: RouterNorm::SelectedSum, - ..rules::SIGMOID_FLOOR - }, 256, 6)).unwrap_err().why, + select_router(moe( + RouterRule { + norm: RouterNorm::SelectedSum, + ..rules::SIGMOID_FLOOR + }, + 256, + 6 + )) + .unwrap_err() + .why, Refusal::NoRouterRule ); } @@ -1315,9 +1327,7 @@ mod tests { assert_eq!(MoeShape::of(&m).rule, rules::SOFTMAX_NORM); } - /// Rows are distinct instances: no two rows run one rule at one width - /// and one pick count, and one body holds two rows at a width only by - /// disjoint pick ranges. + /// No two rows select the same rule, expert width and pick count. #[test] fn no_two_router_rows_overlap() { for (i, a) in ROUTERS.iter().enumerate() { @@ -1325,12 +1335,6 @@ mod tests { let picks = a.top_k_min <= b.top_k_max && b.top_k_min <= a.top_k_max; let overlap = a.experts == b.experts && (a.runs(b.rule) || b.runs(a.rule)) && picks; assert!(!overlap, "{} and {}", a.at, b.at); - assert!( - !((a.body, a.experts) == (b.body, b.experts) && picks), - "{} and {}", - a.at, - b.at - ); } assert!(a.top_k_min >= 1 && a.top_k_min <= a.top_k_max, "{}", a.at); assert!(a.per_lane() >= 1 && a.per_lane() <= 32, "{}", a.at); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index daa05c3ec..856aee47b 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -67,6 +67,8 @@ pub enum Program { /// gate·up and down `_sel` entries ([`MUSESPARK_LEG`]) and the rest on the /// host tier. MuseSparkBody, + /// The head128 sink/SWA body with floor-normalized top-six experts. + FlintBody, } /// The program that runs `arch`'s layers; `None` when none does. @@ -85,7 +87,7 @@ pub fn program_of(arch: Arch) -> Option { Arch::NemotronH => Some(Program::NemotronHBody), Arch::NemotronHMoe => Some(Program::NemotronHMoeBody), Arch::GemmaEmbedding | Arch::GemmaEmbedding2 => Some(Program::GemmaEmbedding2Body), - Arch::Flint => None, + Arch::Flint => Some(Program::FlintBody), Arch::MuseSpark => Some(Program::MuseSparkBody), } } @@ -110,6 +112,7 @@ impl Program { Self::GemmaEmbedding2Body => "gemma-embedding2", Self::Raon1Body => "raon1", Self::MuseSparkBody => "musespark", + Self::FlintBody => "flint", } } } @@ -404,7 +407,7 @@ pub struct LegArms { #[must_use] pub fn leg_arms(p: Program) -> Option { match p { - Program::Qwen3moeBody | Program::Qwen35Body => Some(LegArms { + Program::Qwen3moeBody | Program::Qwen35Body | Program::FlintBody => Some(LegArms { gate_up: prompt_leg_gate_up, down: prompt_leg_down, }), @@ -500,7 +503,7 @@ pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> ker #[must_use] pub fn routed_rule(p: Program) -> RoutedRule { match p { - Program::Qwen3moeBody | Program::Qwen35Body => RoutedRule::Leg { + Program::Qwen3moeBody | Program::Qwen35Body | Program::FlintBody => RoutedRule::Leg { gate_up: |ty| prompt_leg_gate_up(ty).then(|| leg_card_format(ty)), down: |ty| prompt_leg_down(ty).then(|| leg_card_format(ty)), }, @@ -578,7 +581,7 @@ mod tests { use super::*; use crate::placement::{ModelTensor, ModelTensors, Role}; - const PROGRAMS: [Program; 12] = [ + const PROGRAMS: [Program; 13] = [ Program::Deepseek41Chain, Program::Qwen3moeBody, Program::Qwen35Body, @@ -591,6 +594,7 @@ mod tests { Program::NemotronHMoeBody, Program::Raon1Body, Program::MuseSparkBody, + Program::FlintBody, ]; /// Every type the file format names. diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index f2d85d534..ac0f34f34 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -16,6 +16,9 @@ pub use gguf::card::{ }; use models::Role; +pub mod attention; +pub mod ffn; + /// The common card gate·up `_sel` entry that runs a stack of type `ty`. #[must_use] pub const fn gate_up(ty: GgmlType) -> Option { diff --git a/crates/placement/src/kernels/attention.rs b/crates/placement/src/kernels/attention.rs new file mode 100644 index 000000000..c896272fd --- /dev/null +++ b/crates/placement/src/kernels/attention.rs @@ -0,0 +1,79 @@ +//! The buffers of the common head128 projection/attention scratch. + +use super::{SiteAct, try_q8_act_bytes}; + +/// Quantized wide forms beside the caller-owned f32 input rows. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Forms { + pub q128: bool, + pub q32: bool, +} + +impl Forms { + /// Include the activation form a selected projection reads. + #[must_use] + pub const fn with(self, form: SiteAct) -> Self { + Self { + q128: self.q128 || matches!(form, SiteAct::Q8x128), + q32: self.q32 || matches!(form, SiteAct::Q8x32 | SiteAct::Q8Blocks32), + } + } + + /// Bytes of these forms over `rows` columns of `width` values. + #[must_use] + pub fn bytes(self, rows: u64, width: u64) -> Option { + let q128 = if self.q128 { + try_q8_act_bytes(rows, width)? + } else { + 0 + }; + let q32 = if self.q32 { + wide32_bytes(rows, width)? + } else { + 0 + }; + q128.checked_add(q32) + } +} + +/// Bytes of wide q8 blocks of 32: twenty words per padded 64-value step. +#[must_use] +pub const fn wide32_bytes(rows: u64, width: u64) -> Option { + match rows.checked_mul(width.div_ceil(64)) { + Some(steps) => steps.checked_mul(80), + None => None, + } +} + +/// The loaded projection widths, partial segments and actual selected forms. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Head128Scratch { + pub hidden: u64, + pub query: u64, + pub key_value: u64, + pub heads: u64, + pub segments: u64, + pub input: Forms, + pub output: Forms, +} + +impl Head128Scratch { + /// Checked bytes for Q/K/V, normalized/residual rows, attention/partials, + /// selected wide forms and the one-matrix route table. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let partial = self.heads.checked_mul(self.segments)?.checked_mul(130)?; + let values = self + .hidden + .checked_mul(2)? + .checked_add(self.query.checked_mul(2)?)? + .checked_add(self.key_value.checked_mul(2)?)? + .checked_add(partial)?; + let floats = values.checked_mul(rows)?.checked_mul(4)?; + let route = super::ffn::dense_route_bytes(rows)?; + floats + .checked_add(self.input.bytes(rows, self.hidden)?)? + .checked_add(self.output.bytes(rows, self.query)?)? + .checked_add(route) + } +} diff --git a/crates/placement/src/kernels/ffn.rs b/crates/placement/src/kernels/ffn.rs new file mode 100644 index 000000000..0087d0343 --- /dev/null +++ b/crates/placement/src/kernels/ffn.rs @@ -0,0 +1,234 @@ +//! Buffer geometry of the common routed/shared FFN and prompt walk. + +use super::SiteAct; +use super::attention::Forms; +use crate::placement::workstation::TierLegScratch; + +/// The first unit size whose shared and dense blocks use the wide path. +pub const GATED_FROM: u64 = 9; + +/// Wide quantized forms and an owned f32 row, when a selected down reads one. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct WideForms { + pub quant: Forms, + pub f32: bool, +} + +impl WideForms { + /// Include a selected site's activation form. + #[must_use] + pub const fn with(self, reads: SiteAct) -> Self { + Self { + quant: self.quant.with(reads), + f32: self.f32 || matches!(reads, SiteAct::F32), + } + } + + /// Input f32 rows are borrowed from the caller. + #[must_use] + pub const fn without_f32(self) -> Self { + Self { f32: false, ..self } + } + + /// Checked allocation bytes of these forms. + #[must_use] + pub fn bytes(self, rows: u64, width: u64) -> Option { + let floats = if self.f32 { + rows.checked_mul(width)?.checked_mul(4)? + } else { + 0 + }; + self.quant.bytes(rows, width)?.checked_add(floats) + } +} + +/// One shared or dense block's wide scratch, with the actual selected read forms. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WideBlock { + pub hidden: u64, + pub ff: u64, + pub input: WideForms, + pub down: WideForms, +} + +impl WideBlock { + /// Route table, borrowed-input forms, owned activation forms, and gate/up outputs. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + dense_route_bytes(rows)? + .checked_add(self.input.without_f32().bytes(rows, self.hidden)?)? + .checked_add(self.down.bytes(rows, self.ff)?)? + .checked_add(rows.checked_mul(self.ff)?.checked_mul(8)?) + } +} + +/// A single-expert wide route table: slot ids, tiles, counts and zero ids. +#[must_use] +pub const fn dense_route_bytes(rows: u64) -> Option { + let bytes = (2 * rows as u128 + 2 * (rows as u128 / 64 + 1) + 2) * 4; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } +} + +/// The common card batch beside its leg: accumulation/base rows and selected places. +#[must_use] +pub fn card_batch_bytes(leg: TierLegScratch) -> Option { + leg.bytes(leg.downs) + .ok()? + .checked_add(leg.chunk.checked_mul(leg.n_embd)?.checked_mul(8)?)? + .checked_add(leg.chunk.checked_mul(leg.n_used)?.checked_mul(4)?) +} + +/// Places, join places and join weights of a prompt unit on an expert-tier load. +#[must_use] +pub const fn tier_places_bytes(rows: u64, used: u64) -> Option { + match rows.checked_mul(used) { + Some(slots) => slots.checked_mul(12), + None => None, + } +} + +/// Exact buffers shared by all units of the routed/shared prompt walk. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RoutedBatch { + pub hidden: u64, + pub experts: u64, + pub used: u64, + pub shared: WideBlock, + pub dense: Option, + pub card: Option, + pub tiered: bool, +} + +impl RoutedBatch { + /// Norm/router/shared/base rows, narrow activation rows, optional dense/card/wide/tier buffers. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let dense_ff = self.dense.map_or(0, |d| d.ff); + let mut values = self + .hidden + .checked_mul(3)? + .checked_add(self.experts)? + .checked_add(self.used.checked_mul(2)?)? + .checked_add(self.shared.ff.max(dense_ff))?; + if self.dense.is_some() { + values = values.checked_add(self.hidden.checked_mul(2)?)?; + } + let mut bytes = rows.checked_mul(values)?.checked_mul(4)?; + if rows >= GATED_FROM { + bytes = bytes.checked_add(self.shared.bytes(rows)?)?; + if let Some(dense) = self.dense { + bytes = bytes.checked_add(dense.bytes(rows)?)?; + } + } + if let Some(card) = self.card { + if card.chunk != rows { + return None; + } + bytes = bytes.checked_add(card_batch_bytes(card)?)?; + } + if self.tiered { + bytes = bytes.checked_add(tier_places_bytes(rows, self.used)?)?; + } + Some(bytes) + } +} + +/// Prompt walk units and the shared host-sum upload buffer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PromptWalk { + pub hidden: u64, + pub sets: u64, +} + +impl PromptWalk { + /// Each unit owns x/x1 plus positions/key counts; all units share one host-sum row set. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let unit = self.hidden.checked_mul(8)?.checked_add(8)?; + self.sets + .checked_mul(unit)? + .checked_add(self.hidden.checked_mul(4)?)? + .checked_mul(rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kernels::{DownAct, DownForms}; + + #[test] + fn wide_threshold_selected_forms_and_protection_are_independent() { + let wide = WideBlock { + hidden: 256, + ff: 256, + input: WideForms::default().with(SiteAct::F32), + down: WideForms::default().with(SiteAct::F32), + }; + let batch = RoutedBatch { + hidden: 256, + experts: 8, + used: 2, + shared: wide, + dense: None, + card: None, + tiered: false, + }; + assert_eq!(batch.bytes(8), Some(8 * 4 * (3 * 256 + 8 + 4 + 256))); + assert_eq!( + batch.bytes(9), + Some(9 * 4 * (3 * 256 + 8 + 4 + 256) + dense_route_bytes(9).unwrap() + 9 * 256 * 12) + ); + let leg = TierLegScratch { + n_embd: 256, + ff: 256, + n_used: 2, + chunk: 9, + downs: DownForms::of(DownAct::Q8Act), + input_scales: false, + }; + let direct = card_batch_bytes(leg).unwrap(); + let scaled = card_batch_bytes(TierLegScratch { + input_scales: true, + ..leg + }) + .unwrap(); + assert_eq!(scaled - direct, 4 * 9 * 2); + let q32 = WideBlock { + input: WideForms::default().with(SiteAct::Q8x32), + ..wide + }; + assert_eq!(q32.bytes(9).unwrap() - wide.bytes(9).unwrap(), 9 * 4 * 80); + assert_eq!(batch.bytes(u64::MAX), None); + } + + #[test] + fn walk_groups_only_duplicate_unit_rows() { + let one = PromptWalk { + hidden: 256, + sets: 1, + } + .bytes(512) + .unwrap(); + let two = PromptWalk { + hidden: 256, + sets: 2, + } + .bytes(512) + .unwrap(); + assert_eq!(one, 512 * (12 * 256 + 8)); + assert_eq!(two - one, 512 * (8 * 256 + 8)); + assert_eq!( + PromptWalk { + hidden: u64::MAX, + sets: 1 + } + .bytes(512), + None + ); + } +} diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 9599ba578..96e625d63 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -6387,7 +6387,7 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, tier_leg_batch_bytes}; let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); - let batch = tier_leg_batch_bytes(256, 256, 1, 2, 3, both).expect("tier bytes"); + let batch = tier_leg_batch_bytes(256, 256, 1, 2, 3, both, false).expect("tier bytes"); let leg = batch.leg.expect("a tier leg descriptor"); let rule = routed_rule(Program::Qwen35Body); for (downs, first) in [ @@ -6463,7 +6463,7 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, TierLegScratch, tier_leg_batch_bytes}; let forms = DownForms::of(DownAct::Q8Act); - let batch = tier_leg_batch_bytes(256, 256, 1, 1, 1, forms).expect("valid batch"); + let batch = tier_leg_batch_bytes(256, 256, 1, 1, 1, forms, false).expect("valid batch"); let good = batch.leg.expect("a leg descriptor"); let mut model = legged(&[[GgmlType::Q5_K; 3]]); model.experts_used = 1; @@ -6522,13 +6522,15 @@ mod tests { ); assert!( matches!( - tier_leg_batch_bytes(leg.n_embd, leg.ff, leg.n_used, leg.chunk, 1, leg.downs), + tier_leg_batch_bytes( + leg.n_embd, leg.ff, leg.n_used, leg.chunk, 1, leg.downs, leg.input_scales, + ), Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" ), "{leg:?}" ); } - assert!(tier_leg_batch_bytes(256, 256, 8, 1, 1 << 61, forms).is_err()); + assert!(tier_leg_batch_bytes(256, 256, 8, 1, 1 << 61, forms, false).is_err()); let inconsistent = workstation::TierBatchBytes { scratch: 0, ..batch diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 74c5d5dd0..64a29c305 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -437,6 +437,8 @@ pub struct TierLegScratch { pub n_used: u64, pub chunk: u64, pub downs: DownForms, + /// One f32 input-restoration scale per slot of the leg's chunk. + pub input_scales: bool, } impl TierLegScratch { @@ -460,7 +462,16 @@ impl TierLegScratch { { return Err(invalid()); } - try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs).ok_or_else(invalid) + let counted = || { + let bytes = try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs)?; + let scales = if self.input_scales { + 4u64.checked_mul(self.chunk)?.checked_mul(self.n_used)? + } else { + 0 + }; + bytes.checked_add(scales) + }; + counted().ok_or_else(invalid) } } @@ -568,6 +579,7 @@ pub fn tier_leg_batch_bytes( chunk: u64, cols: u64, downs: DownForms, + input_scales: bool, ) -> Result { let leg = TierLegScratch { n_embd, @@ -575,6 +587,7 @@ pub fn tier_leg_batch_bytes( n_used, chunk, downs, + input_scales, }; let scratch = leg.bytes(downs)?; let counted = || -> Option<(u64, u64)> { @@ -1693,6 +1706,39 @@ mod tests { plan_on, plan_room, plan_tiers, spec_of_device, tier_batch_bytes, unified_of, }; + #[test] + fn tier_input_scales_cost_one_f32_per_chunk_slot() { + use crate::kernels::{DownAct, DownForms}; + + let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + for chunk in [1, 8, 16] { + for downs in [DownForms::NONE, DownForms::of(DownAct::Q8Act), both] { + let direct = super::tier_leg_batch_bytes(256, 256, 6, chunk, 32, downs, false) + .expect("direct descriptor"); + let scaled = super::tier_leg_batch_bytes(256, 256, 6, chunk, 32, downs, true) + .expect("scaled descriptor"); + assert_eq!(scaled.staging, direct.staging); + assert_eq!(scaled.host, direct.host); + assert_eq!(scaled.scratch, direct.scratch + 4 * chunk * 6); + assert_eq!( + scaled.card().expect("scaled card"), + direct.card().expect("direct card") + 4 * chunk * 6, + ); + let leg = scaled.leg.expect("scaled leg"); + assert!(leg.input_scales); + assert_eq!( + leg.bytes(DownForms::NONE).expect("empty selected union"), + direct + .leg + .expect("direct leg") + .bytes(DownForms::NONE) + .expect("empty union") + + 4 * chunk * 6, + ); + } + } + } + /// One plan room: `tools/ref/plan-room.sh --room`, the room /// `records-refresh` makes V4.1's plans at, prints [`PLAN_ROOM`], and /// the workstation's host leaves a load that room diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 64ad6ee9b..e76c7c082 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,9 +2,9 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader** and a real common -256/top6 floor-router binding; it still has no Flint body, app session or serve seat. -No GPU, inference, performance or weight-content validation is claimed. +that snapshot. The integration based on **`e0c82e1779a8e754c53f4442800e6c3d61271a48`** +adds an actual `gpu-flint` body, application session and shared HTTP seat, not just architecture +admission. Device execution, GPU numerical acceptance and performance remain unverified. ## Host reader status @@ -15,15 +15,15 @@ It reuses the common metadata validators, tensor stems, classification and dimen GemmaEmbedding2's sliding mask: missing keys, scalars, wrong lengths and non-bool items are refused. Present `expert_weights_scale` uses `arch::finite_positive`; only an absent key defaults to 1, with that default recorded. -The common needs walk emits q/k normalization even on NoPE layers. `program_of(Flint)` remains -`None`; missing program, head128 sinks/SWA and NoPE normalization remain coverage refusals. -`Router.norm` and `RouterRule.norm` distinguish no normalization, the existing selected-sum -rule and Flint's selected-sum floor. The real floor-router binding is selectable in both bias -modes; selecting that operation does not supply a whole program or activated-value scaling. +The common needs walk emits q/k normalization even on NoPE layers. `program_of(Flint)` selects +`FlintBody`, whose coverage names the operations its loader actually binds: head128/group8 +sink attention, independent NoPE/NeoX and SWA masks, and the real floor router. +`Router.norm` and `RouterRule.norm` keep the old selected-sum rule distinct from Flint's floor. +The activated-value protection and explicit residual association are loaded FFN policies. Fault formatting uses `bloomery_device::fault`'s shared layout/decoder and the -`gpu::arch::step_order` registry. Flint's registry row uses neutral code order; -it does not describe a runnable step. +`gpu::arch::step_order` registry. The body delegates fault checks and captured execution to the +common chain runtime; the architecture registry's neutral order is not numerical evidence. Run `BLOOMERY_BOX_ENV="BLOOMERY_MODEL=/absolute/path/flint-header.gguf" BLOOMERY_BOX_LOCAL=1 just gate-flint-meta`. Set `BLOOMERY_ENV_FILE` to the local toolchain environment file if it is not `~/bloomery-env.sh`. @@ -45,16 +45,14 @@ remaining acceptance rows. This header gate is not an end-to-end model gate. ### Runtime handoff and host policy The design, common template support, tokenizer/four-EOG contract and reader (#676–#680) are merged. -The existing header recipe was exercised on the exact 5,470,528-byte prefix above: four reader tests and -the pinned-header test passed, still reporting the missing program and operation bindings. -No full weights or GPU were needed for that check. +The pinned-header consumer now also checks actual Program coverage and planning on the complete +local GGUF: 779 tensors, 49 layers, 2 dense/47 routed, 49 head norms, 35 rotations and 49 sinks. +This reads the header, not tensor contents; it does not prove CUDA execution. -The maintainer [assigned the model port and offered card runs](https://github.com/midagedev/bloomery/issues/632#issuecomment-6352066886), -and named a forthcoming head128 sink/SWA attention, QK RMS and sigmoid256/top6 train. -Those complete attention contracts are not in this main snapshot or the inspected public heads of -#726 (`1723485602c1`) and #747 (`f883e84f8d2c`). Existing ring and sink-merge ingredients do not -establish the combined attention operation. Obtain the train's fetchable ref before duplicating -those primitives. +The maintainer [assigned the model port and offered card runs](https://github.com/midagedev/bloomery/issues/632#issuecomment-6352066886). +The later public QK-RMS, sink/SWA and sigmoid train supplies reusable primitives. The new common +`attention_h128` binding combines those with actual ring append/attention kernels; it is not an +identity-RoPE substitute or a padded-head adaptation of another family. The common `route_body::sigmoid_floor_256_6` binding now supplies Flint's fixed `max(sum_selected, 2^-14)` rule. `RouterKernels::enqueue_router` and `enqueue_router_rows` @@ -108,10 +106,9 @@ Its nonnegative constant-weight case derives a scalar floating-operation bound; it is not a whole-model accuracy pin. The hardware case has not been run here. Existing program-specific admission remains unchanged: in particular the legacy -Qwen projection/shared dispatch does not gain this site automatically. Flint -still has no Program, attention/router binding, body or serve seat. Run the new -consumer gate and the affected regressions on NVIDIA before claiming device -support; reconcile full-model coverage only with the actual body integration. +Qwen projection/shared dispatch does not gain this site automatically. Flint's implemented +Program binds the real ordinary dense/shared operations. Run the new consumer gate and affected +regressions on NVIDIA before claiming device support; header coverage is not arithmetic proof. ### Selected expert arms and tier scratch Common card and tier wrappers resolve `LegArms` from their selected `Program`, rather than assuming @@ -133,18 +130,22 @@ family's activation contract. `LegTier` receives its activation converter at loa selected `Program` arms replace the redundant family arm hook. Qwen, Muse, MiMo and MiniMax sizing callers propagate checked tier descriptors rather than bypassing their validation. Share-budget floors use the same selected reserve calculation. +`TierLegScratch::input_scales` names an optional physical f32 scale per chunk +slot. Its checked byte formula includes `4 * chunk * n_used` when enabled, +including an empty selected-down union; legacy descriptors leave it disabled. +The same descriptor prices the loader's scale allocation without reserve padding. A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an -exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. +exact-head NVIDIA run. The integration's body/app/seat now consume these selected forms. ### Six-slot F32 tier handoff implementation `HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry beside the existing four- and eight-slot entries. Both host and tier layouts must have the same supported slot count and activation width; compressed tier images remain refused. The entry uses the existing copy/place/fault helpers and the existing -subsequent go/publication protocol; no Flint router or Program is registered. +subsequent go/publication protocol; the integration's configured routed block consumes it. The pure layout contract passed locally after an oxide build. `hw_f32_tier_handoffs_conserve_six_and_eight_consumed_routes`, owned by @@ -165,6 +166,37 @@ the new entry in the four existing architecture/capability tables. Offline no-JIT scans cannot supply that missing column, so no placeholder row is added. +### Integrated runtime consumer and NVIDIA handoff + +`crates/gpu-flint` loads the validated `ModelSpec`, actual selected projection/expert forms and +host/NVMe source through the common hosted loader. It implements the common chain, tier, +rollback and resident-slot interfaces. `app::flint` and `bloomery-serve --model flint` use the +existing hybrid/tiered seats, tokenizer, original chat template and stop handling. + +Head128 attention keeps NoPE and NeoX independent of the readable window, stores f16 KV, and +uses `Rows::Ring.count` for capacity and checked chunking. For the released model, per-slot KV is +`2048 * (14*C + 35*min(C,1024))` bytes. Slot state retains high-water positions: at a filled +1024-row ring a 512-token cut is retained, whereas 513 tokens require reset/refeed. + +The routed/shared block loads `(routed + shared) + residual` and the reference's activated-input +protection: `exp(ceil(log(amax / 1024) * log2(e)).clamp(0,127) * log(2))`. The down output is +restored before route weighting; this is not an activation clamp. Legacy models keep their +previous residual association and direct expert inputs without scale buffers or added launches. +Prompt and tier reservations include the actual optional scales and selected buffer unions. + +Run the owning model-free gates `gate-gpu-flint-attn` and `gate-gpu-flint-router`, then the +full-file `gate-gpu-flint-lib` with the complete GGUF passed through `BLOOMERY_BOX_ENV`. +Its ignored hardware case compares eager/captured logits, interleaved resident slots, +reset/refeed and retained/refused ring-boundary transitions. The test must be executed, not +counted as a pass because it compiles or remains ignored. + +NVIDIA acceptance also requires the affected shared expert/residency/tier regressions, exact-head +oracle comparisons and the existing per-architecture spill/JIT/resource checks. Ten new entries +currently lack genuine measurements in the four pinned architecture/capability tables; no zero, +placeholder or copied hardware pin is supplied. Keep the integration Draft until that evidence +is produced. No ROCm implementation or AMD execution is implied by this CUDA attachment. + + ## 1. Sources and observed artifact @@ -188,7 +220,10 @@ alignment padding, at byte **5,470,528**. Bytes after that boundary were ignored `fb0cc4b520f927f0b679193397ee0c36ca252426e7881283c5aa6de3e44a3a2b`. The existing Python header reader and Rust `gguf::inventory_of` agree on **779 tensors** and **25,049,218,176 encoded tensor bytes**. Shape products total **39,785,287,456 parameters** (derived). -No full model was downloaded or executed. +That initial design inspection fetched only the prefix. A later local CPU oracle run used the +complete 25,054,688,704-byte GGUF (SHA-256 +`63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f`). +Mainline CPU completion is reference evidence, not execution of this CUDA body. The official repository says Apache-2.0; this exact GGUF header says `other` / `flint-preview`. The config/header shapes and the byte-identical 2,328-byte chat templates establish those contracts, @@ -290,13 +325,13 @@ requires bias and derives width from a shared-count convention. Invalid mask typ norms/sinks/shared matrices, an out-of-range block, impossible KV grouping, zero dimensions, nonfinite epsilon/base, overflow and malformed optional bias all need named refusals. -At this snapshot there is no Flint architecture reader or selected program. A reader-only landing -must keep `program_of=None` and the existing **`Need::Program`** refusal in [coverage][coverage]. -Do not claim runtime support by adding the name to an enum or relaxing a shape/type row. +The original reader-only landing retained `program_of=None` and `Need::Program` refusals. +The integration now has actual operation consumers and Program admission; adding a name or +loosening a type row alone would still not establish runtime support. ### Concrete common prerequisites -| ID | Current limitation | Required common change before an executable attach | +| ID | Original gap (design snapshot above) | Required common change / acceptance contract | |---|---|---| | G1 | NoPE normalization-need generation is present; the executable head128 norm-only binding is missing | Preserve the emitted normalization **or** rotation need; bind head128 normalization with `mode=None` truthfully. No identity-RoPE table. | | G2 | Existing head128/group8 attention lacks the complete sink/SWA contract; MiMo's window pass is K192/V128 | Complete the described head128 operation in `rope_neox` / `flash_gqa`, including genuine NoPE append, sink merge, window/ring positions, prompt and slot paths. Never pad heads to 192 or copy MiMo's body. | diff --git a/justfile b/justfile index dfe31c8f9..48414e733 100644 --- a/justfile +++ b/justfile @@ -32,7 +32,7 @@ default: # 빠른 루프: 타입 검사만, 커널은 안 만든다. 의존을 고친 뒤 cargo가 박스 쪽 Cargo.lock을 고쳐 쓰면 lock-back.sh가 # 그것을 이 트리로 가져온다 — box.sh는 한 방향으로만 싣는다. check: - ./tools/box.sh 'cargo check --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' + ./tools/box.sh 'cargo check --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/flint,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' ./tools/lock-back.sh # check와 같은 이유로 `--features gpu`. 이 피처를 켠 것이 기준 계기다(R26). V4.1 op 게이트의 피처 @@ -40,14 +40,14 @@ check: # V4.1 커널의 컴파일러 결함은 여기가 아니라 op 게이트 빌드에서 드러난다. # lint. 에러 0이 계약이고 경고 수는 RESULTS/AGENTS에 적힌 기준선과 비교한다. lint: - ./tools/box.sh 'cargo clippy --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' + ./tools/box.sh 'cargo clippy --workspace --all-targets --features gpu,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/flint,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' # The build without `vision` of the bins that refuse an image request by name — serve_seats/ds41.rs:444 and glm.rs:825 (--mmproj), # decide.rs:545 (the Qwen3.5 seat's encoder), gate_glm5next_e2e.rs:5146 (--only media-seat). One bin takes one feature set (tools/recipes.py's # bin check), and every recipe that builds these bins builds them with `vision`, so only this check compiles the # `cfg(not(feature = "vision"))` branches. A cargo check writes no executable, so it replaces no bin another lane runs. mac-combos runs this shape. check-no-vision: - ./tools/box.sh 'cargo check -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41 --bin gate_glm5next_e2e' + ./tools/box.sh 'cargo check -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41 --bin gate_glm5next_e2e' # fmt는 맥에서 돈다. box.sh의 rsync가 단방향이라 박스에서 포맷하면 결과가 돌아오지 # 않고 다음 명령에 덮여 사라진다(2026-09-19에 그렇게 한 번 날렸다). cargo fmt는 컴파일을 @@ -387,7 +387,7 @@ gate-gpu-musespark-prefill: [group('solo')] [group('v41-load')] gate-gpu-musespark-twocard: - BLOOMERY_MODEL=muse-spark BLOOMERY_CARD=both ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_musespark_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_musespark_prefill --two-card && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_musespark_serve --dir $D' + BLOOMERY_MODEL=muse-spark BLOOMERY_CARD=both ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_musespark_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_musespark_prefill --two-card && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_musespark_serve --dir $D' # Muse Spark Open's load gate (gate_musespark_load's doc): the body's placed load of the gate plan proven against its plan — # every expert in one place and the stack bytes the file's, each segment on the card resident at the plan's buffers and nothing @@ -406,7 +406,7 @@ gate-gpu-musespark-load: [group('solo-real')] [group('v41-load')] gate-gpu-musespark-serve: - BLOOMERY_MODEL=muse-spark ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_musespark_serve --dir $D' + BLOOMERY_MODEL=muse-spark ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_musespark_serve --dir $D' # Solar-Open2-250B end-to-end gate (gate_solar2_e2e's doc): the 36 KDA layers (beta = 2 sigmoid, softplus decay) and 12 NoPE gated GQA @@ -459,7 +459,7 @@ gate-gpu-solar2-twocard: [group('solo-real')] [group('v41-load')] gate-gpu-solar2-serve: - BLOOMERY_MODEL=solar-open2 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_solar2_serve && D=target/solar2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=600 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_solar2_serve --dir $D' + BLOOMERY_MODEL=solar-open2 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_solar2_serve && D=target/solar2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=600 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_solar2_serve --dir $D' # The nemotron seat of bloomery-serve on Nemotron 3 Nano 4B Q4_K_M (gate_nemotron_serve's doc): `--plan` with no flag names its # default context (the ladder's rung, held by the ctx line and the rung above refused) and exits before any load, `--parallel 2` is @@ -468,14 +468,14 @@ gate-gpu-solar2-serve: # server's template renders), serves that turn on /v1/chat/completions (usage counts its ids), resends an edited prompt as a # fresh run that keeps nothing and an extended one that keeps the held positions. One card. gate-gpu-nemotron-serve: - BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' # The same seat gate on the moe body (gate_nemotron_serve's doc, every clause one path over both # bodies): the Bolt fixture under the fixture tier (nemotron_h_moe, the fixture's four layers, any # card), the arch word and the layer count read from the file, the session's prompt call the body's # batched `GpuModel::prefill` (several ids; one id is a step). One card. gate-gpu-nemotron-h-moe-serve: - BLOOMERY_TIER=fixture BLOOMERY_MODEL=nemotron_h_moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-h-moe-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + BLOOMERY_TIER=fixture BLOOMERY_MODEL=nemotron_h_moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-h-moe-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' # GLM-5.3-Flash 라우터(288개 중 8개, sigmoid, ×2.5 — `router::glm5next`)를 모델 없이 합성 입력으로 호스트 규칙과 맞춘다. # 규칙마다 절이 하나다: logits·ids·weights·batch는 비트, scores는 sigmoid 밴드, ties·guard·fault·graph는 명시한 값. @@ -1428,7 +1428,7 @@ gate-gpu-qwen35moe-mtp: # target). The card is the A6000: the DSpark draft's load holds the whole target on it, and Qwen3.6's # 29 GB does not fit the 24 GB card (the seat then refuses the draft by name). gate-gpu-qwen3-dspark: - BLOOMERY_MODEL=qwen35moe BLOOMERY_CARD=a6000 ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen3-dspark && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_dspark && D=target/qwen3-dspark-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && MINI=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/minicpmv47.sh) && test -n "$MINI" && Q30=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen3moe.sh) && test -n "$Q30" && bash tools/gpu-gate.sh gate_qwen3_dspark --qwen "$Q36" --qwen-draft /models/Qwen3.6-35B-A3B/dspark-redhat --sage /models/Sage-A1.5/Sage-A1.5-Q4_K_M.gguf --ornith /models/Ornith-1.5-35B-A3B/Ornith-1.5-35B-Q4_K_M.gguf --minicpm "$MINI" --minicpm-draft /models/MiniCPM-V-4.7-35B-A3B/dspark --qwen3moe "$Q30" --dir $D' + BLOOMERY_MODEL=qwen35moe BLOOMERY_CARD=a6000 ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen3-dspark && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_dspark && D=target/qwen3-dspark-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && MINI=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/minicpmv47.sh) && test -n "$MINI" && Q30=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen3moe.sh) && test -n "$Q30" && bash tools/gpu-gate.sh gate_qwen3_dspark --qwen "$Q36" --qwen-draft /models/Qwen3.6-35B-A3B/dspark-redhat --sage /models/Sage-A1.5/Sage-A1.5-Q4_K_M.gguf --ornith /models/Ornith-1.5-35B-A3B/Ornith-1.5-35B-Q4_K_M.gguf --minicpm "$MINI" --minicpm-draft /models/MiniCPM-V-4.7-35B-A3B/dspark --qwen3moe "$Q30" --dir $D' # Qwen3.6-35B-A3B's DSpark draft, RedHatAI's speculator (an HF directory on the box, `--draft DIR` names another): the dense layers on the card against a host f64 computation of the same layers from the file's bf16 # weights (a block of 8 rows over 3 and over 2,100 context rows, the window 2,048), the captured pass replaying to the @@ -1502,7 +1502,7 @@ gate-gpu-qwen35moe-media: [group('solo')] [group('v41-load')] gate-gpu-qwen38-media: - BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen38-media && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen38_media && D=target/qwen38-media-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=2400 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen38_media --dir $D' + BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen38-media && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen38_media && D=target/qwen38-media-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=2400 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen38_media --dir $D' # The decide seat of bloomery-serve (gate_clef_serve's doc): Clef-Flash at bartowski's Q5_K_M (6.9 GB, the # `qwen35` file of rev d7f376ea) with the release's head; the six refusals (a qwen35 file with no head, --model @@ -1520,7 +1520,7 @@ gate-gpu-qwen38-media: # adds the clauses that start the seat from the hub with no projector flag (it fetches the repo's own projector and # serves images) and with --no-mmproj (it serves none): a network read. Either card. gate-gpu-clef-serve *ARGS: - BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_clef_serve && D=target/clef-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_clef_serve --model /models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --clef-model /root/models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --image-model /models/clef-flash/Cloudflare_clef-flash-Q8_0.gguf --mmproj /root/models/clef-flash/mmproj-Cloudflare_clef-flash-bf16.gguf --dir $D {{ARGS}}' + BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_clef_serve && D=target/clef-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_clef_serve --model /models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --clef-model /root/models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --image-model /models/clef-flash/Cloudflare_clef-flash-Q8_0.gguf --mmproj /root/models/clef-flash/mmproj-Cloudflare_clef-flash-bf16.gguf --dir $D {{ARGS}}' # The decide seat of bloomery-serve on lev (gate_lev_serve's doc): ggml-org/lev-GGUF's Q4_K_M (3.0 GB, a `qwen35` # file with `qwen35.decision.type = lev`, fetched into /root/models/lev-4b/) with no head flag; five refusals before @@ -1530,7 +1530,7 @@ gate-gpu-clef-serve *ARGS: # prompts' ids; ARGS `--hf ggml-org/lev-GGUF:Q4_K_M` adds the clause that serves the file from the hub, no head flag, and # answers the first request as the `-m` server does (a network read). Either card. gate-gpu-lev-serve *ARGS: - BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_lev_serve && D=target/lev-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_lev_serve --model /root/models/lev-4b/lev-Q4_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --ref $BLOOMERY_DATA/lev/4b/ref --dir $D {{ARGS}}' + BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_lev_serve && D=target/lev-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_lev_serve --model /root/models/lev-4b/lev-Q4_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --ref $BLOOMERY_DATA/lev/4b/ref --dir $D {{ARGS}}' # clef_hidden on the box (ARGS as its doc: --model, --ids, --out, ...): a qwen35 file's prompt-only pass, every # position's final-norm hidden state to a file and a `clef hidden` record with the call's functional wall. @@ -1668,7 +1668,7 @@ gate-gpu-minimaxm3-prefill: [group('solo-real')] [group('v41-load')] gate-gpu-minimaxm3-serve: - BLOOMERY_MODEL=minimax-m3 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' + BLOOMERY_MODEL=minimax-m3 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' # MiniMax-M3.1-Flash on plan (b′) (gate_minimaxm3_prefill's doc, its `--arm bp`): the stage on the A6000 and the expert tier on the 3090, # both planned under one card budget the gate sets, against the same plan with every tier segment on the stage card (the A6000 alone, no @@ -1681,7 +1681,7 @@ gate-gpu-minimaxm3-serve: [group('solo')] [group('v41-load')] gate-gpu-minimaxm3-twocard: - BLOOMERY_MODEL=minimax-m3 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-minimaxm3-twocard && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features minimaxm3,checkpoint --release --bin gate_minimaxm3_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_minimaxm3_prefill --arm bp && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' + BLOOMERY_MODEL=minimax-m3 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-minimaxm3-twocard && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features minimaxm3,checkpoint --release --bin gate_minimaxm3_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_minimaxm3_prefill --arm bp && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' # Nemotron-H MoE (Lightning) with its 32-value experts on the card (Relu2Leg for the step, the 32-value GEMM for the # prompt): host, card (the 3090's room, split) and whole (the A6000's room, box.sh's pick) arms near the host path [derived: 8..12 min, @@ -1706,7 +1706,7 @@ gate-gpu-gemma-embedding-head256: # greedy tokens with embedding calls in flight, and named refusals; the runner holds one card. [group('pool')] gate-gpu-gemma-embedding2-serve *ARGS: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/gate.sh -p bloomery-gpu-gates --features hf --test sentence_prompts && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding2-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D" {{ARGS}} && if [ -z "{{ARGS}}" ]; then BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D/audio" --audio; fi' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/gate.sh -p bloomery-gpu-gates --features hf --test sentence_prompts && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding2-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D" {{ARGS}} && if [ -z "{{ARGS}}" ]; then BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D/audio" --audio; fi' # EmbeddingGemma 2's encoder (gpu-gemma-embedding2) end to end on the Q8_0 file the sets are dumped from # (refset's gemma_embedding2::MODEL, held to its sha256; the sets' `# model` line names the same path): every @@ -1718,7 +1718,7 @@ gate-gpu-gemma-embedding-e2e: BLOOMERY_MODEL=gemma-embedding ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e --v1' gate-gpu-gemma-embedding-serve: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --v1 --dir "$D"' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --v1 --dir "$D"' gate-gpu-gemma-embedding2-e2e *ARGS: ./tools/box.sh 'bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-gemma-embedding2 --lib -- --include-ignored' @@ -1764,10 +1764,14 @@ gate-gpu-mimo2-flash *ARGS: gate-gpu-flint-attn: ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_flint_attn && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_flint_attn' -# Flint's 256-expert top-6 sigmoid router. Model-free: one token, scale 1, zero bias. +# Flint's floor-normalized top-six router: optional selection bias, rows and captured replay. gate-gpu-flint-router: ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_flint_router && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_flint_router' +# Full-file eager/captured, resident-slot and ring-retention consumer transitions. +gate-gpu-flint-lib: + ./tools/box.sh 'bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-flint --lib -- --include-ignored' + # Muse Spark's 256-expert top-8 sigmoid router (`sigmoid_256_8`). Model-free like the Flint one, # plus the bias clause: the selection follows the biased scores, the weights the unbiased ones. gate-gpu-muse-router: @@ -2377,7 +2381,7 @@ gate-q38-serve-rules: # gate lock. [group('host')] gate-serve-draft-rules: - ./tools/box.sh 'bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41' + ./tools/box.sh 'bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41' # The V4.1 binaries' `--place` word (`generate::Place`, behind the deepseek41 feature like bind): each alias and its card # list (`a6000+3090` is `bp`) planning the machine its plan function makes, a list word planning its own cards, and every @@ -2869,7 +2873,7 @@ gate-gpu-qwen38-serve: # _decode_*): the qwen3moe file's --mmproj refusal and, on the Qwen3.6 file, the seat with its encoder. [group('pool')] gate-gpu-qwen3-serve: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/qwen3-serve-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && Q38D=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen38d.sh) && test -n "$Q38D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen3_serve --model "$BLOOMERY_REF_MODEL" --model "$Q36" --model "$Q38D" --dir $D --images' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/qwen3-serve-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && Q38D=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen38d.sh) && test -n "$Q38D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen3_serve --model "$BLOOMERY_REF_MODEL" --model "$Q36" --model "$Q38D" --dir $D --images' # bloomery-serve-qwen38 on the box, for a person to attach a client to (toktape records from it): the A6000 by # default (SERVE_PLACE=gate for the 3090), the port 8080 unless SERVE_PORT, a four-hour gate bound unless @@ -2988,7 +2992,7 @@ depth-gpu-glm5next *ARMS: # BLOOMERY_BOX_ENV=BLOOMERY_DRY=1 nothing is built and no lease is taken: the plan and each binary's state. depth-serve *ARMS: @[ -n "${BLOOMERY_MODEL:-}" ] || { echo "depth-serve: BLOOMERY_MODEL is unset: name the model profile (tools/ref/models/.sh), e.g. BLOOMERY_MODEL=mimo2 just depth-serve :" >&2; exit 64; } - ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && { here=$(basename "$PWD"); ours=; for a in {{ARMS}}; do t=${a#env=*:}; [ "${t%%:*}" != "$here" ] || ours=1; done; if [ -n "$ours" ] && [ -z "${BLOOMERY_DRY:-}" ]; then cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve; fi; } && bash tools/ref/depth-serve.sh {{ARMS}}' + ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && { here=$(basename "$PWD"); ours=; for a in {{ARMS}}; do t=${a#env=*:}; [ "${t%%:*}" != "$here" ] || ours=1; done; if [ -n "$ours" ] && [ -z "${BLOOMERY_DRY:-}" ]; then cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve; fi; } && bash tools/ref/depth-serve.sh {{ARMS}}' # The schemajump against Clef-Flash sitting: the Clef decide seat and a Qwen3.6 seat, both bloomery-serve, one lease on # the A6000, the arms alternating request by request (the design's §4; the cards are docs/cards/sj-*.card). Two windows @@ -3009,7 +3013,7 @@ depth-serve *ARMS: # bloomery-serve-ds41 picks, with SJ_DS41_MODEL; this recipe builds that binary when the windows name it) and SJ_SUITES # other suite files (`tools/ref/clef/suite-hard.jsonl`, scored against its `.gold.jsonl`); the header of the runner has both. sj-vs-clef: - ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && if [ -z "${BLOOMERY_DRY:-}" ]; then case " $(printf %s "${SJ_WINDOWS:-qwen:on qwen:off}" | tr , " ") " in *" qwen:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve;; esac && case " $(printf %s "${SJ_WINDOWS:-}" | tr , " ") " in *" ds41:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features deepseek41,vision,gemma-embedding2,hf --release --bin bloomery-serve-ds41;; esac; fi && bash tools/ref/sj-vs-clef.sh' + ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && if [ -z "${BLOOMERY_DRY:-}" ]; then case " $(printf %s "${SJ_WINDOWS:-qwen:on qwen:off}" | tr , " ") " in *" qwen:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve;; esac && case " $(printf %s "${SJ_WINDOWS:-}" | tr , " ") " in *" ds41:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features deepseek41,vision,gemma-embedding2,hf --release --bin bloomery-serve-ds41;; esac; fi && bash tools/ref/sj-vs-clef.sh' # Qwen3-30B-A3B decode-step kernel timeline (nsys, A6000, under the lease, lead-only): generate_qwen3moe's seed form at each # depth (default 6 4096), one prefill pass + BLOOMERY_NSYS_N - 1 replays, the cache height depth-qwen3moe.sh uses. The @@ -3572,7 +3576,7 @@ gate-gpu-glm5next-residency: [group('solo')] [group('v41-load')] gate-gpu-glm5next-serve: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-glm5next-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,hf,glm5next-hf --release --bin generate_glm5next && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-serve-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm plain --dir $D/plain && bash tools/gpu-gate.sh gate_glm5next_serve --arm drafted --dir $D/drafted' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-glm5next-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,hf,glm5next-hf --release --bin generate_glm5next && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-serve-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm plain --dir $D/plain && bash tools/gpu-gate.sh gate_glm5next_serve --arm drafted --dir $D/drafted' # The GLM seat's image input beside its draft (gate_glm5next_serve --arm media): one server of --parallel 2 under # BLOOMERY_DRAFT=mtp BLOOMERY_RESIDENCY=off with --mmproj, the GLM-5.3 projector, then an image chat on one slot and a @@ -3587,7 +3591,7 @@ gate-gpu-glm5next-serve: [group('solo')] [group('v41-load')] weekly-gpu-glm5next-media-serve: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-glm5next-media-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-media-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm media --dir $D --mmproj /models/mmproj/unsloth--GLM-5.3-Flash-GGUF/mmproj-BF16.gguf' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-glm5next-media-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-media-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm media --dir $D --mmproj /models/mmproj/unsloth--GLM-5.3-Flash-GGUF/mmproj-BF16.gguf' # The shipped `bloomery-serve` started once per image flag on the GLM-5.3 fixture (gate_serve_start's header has the clauses): (s1) # `-m ` with the serving defaults and one text chat; (s2) `--place a --mmproj `, the tower's @@ -3605,7 +3609,7 @@ weekly-gpu-glm5next-media-serve: # 8.6 GB set and the 1.2 GB projector in threads beside the first two starts: 53 to 98 s warm; the bound is the runner's 900 s. [group('solo')] gate-gpu-serve-start-glm: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-glm && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/glm && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family glm --dir $D' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-glm && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/glm && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family glm --dir $D' # The same gate on the V4.1 fixture (the file's architecture picks the seat, which has no `--plan`, so its plan is read from each # start's own log). The projector is the `# mmproj` line of the V4.1 vision set ($BLOOMERY_DATA/ref-vision/deepseek41v, `just dump-ref-vision`), @@ -3616,7 +3620,7 @@ gate-gpu-serve-start-glm: # beside the first two starts, a warm build: 72 to 132 s warm; the bound is the runner's 900 s. [group('solo')] gate-gpu-serve-start-ds41: - BLOOMERY_MODEL=deepseek41 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-ds41 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/ds41 && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family ds41 --dir $D' + BLOOMERY_MODEL=deepseek41 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-ds41 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/ds41 && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family ds41 --dir $D' # The MiMo seat of `bloomery-serve` (gate_mimo2_serve): the refusals before any load (--plan's default context, the one # slot, the tier card), one server at --ctx 4160 --parallel 1 holding its load and listening records, the three prompts' @@ -3633,7 +3637,7 @@ gate-gpu-serve-start-ds41: [group('solo')] [group('v41-load')] weekly-gpu-mimo2-serve: - BLOOMERY_MODEL=mimo2 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-mimo2-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_mimo2_serve && D=target/mimo2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1680 bash tools/gpu-gate.sh gate_mimo2_serve --dir $D' + BLOOMERY_MODEL=mimo2 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-mimo2-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_mimo2_serve && D=target/mimo2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1680 bash tools/gpu-gate.sh gate_mimo2_serve --dir $D' # glm5next decode CLI, functional run (no timing): generate_glm5next feeds --tokens one step per id, then greedy -n # tokens. The gate placement on the 3090 unless --place a (and BLOOMERY_CARD=a6000). 3090, gate lock. diff --git a/tools/families.tsv b/tools/families.tsv index ef853b823..e8eb4ec92 100644 --- a/tools/families.tsv +++ b/tools/families.tsv @@ -10,7 +10,7 @@ # fixture yes (the family has a fixture file, tools/fixture-dirs.tsv) or no deepseek2 F crates/model/src/arch/deepseek2/** crates/gpu/src/arch/deepseek2/** 7 no deepseek41 R crates/gpu-deepseek41/** crates/model/src/arch/deepseek41/** crates/refset/src/arch/deepseek41/** crates/refset/src/arch/deepseek41v/** crates/vision/src/arch/deepseek41v/** crates/gpu-gates/src/bin/gate_deepseek41_*.rs crates/gpu-gates/src/bin/gate_ds41_*.rs crates/gpu-gates/src/bin/gate_dspark_*.rs crates/gpu-gates/src/bin/shared/ds41_*.rs crates/gpu-gates/src/bin/shared/dspark_hf.rs crates/gpu-gates/src/bin/shared/serve_seats/ds41.rs crates/gpu-gates/src/bin/shared/routed_legs/deepseek41.rs 7 yes -flint F crates/model/src/arch/flint/** crates/model/tests/flint_meta.rs crates/gpu/src/flash_gqa_h128_swa.rs crates/gpu/src/qk_rms.rs crates/gpu-gates/src/bin/gate_flint_attn.rs crates/gpu-gates/src/bin/gate_flint_router.rs 7 no +flint F crates/model/src/arch/flint/** crates/model/tests/flint_meta.rs crates/gpu-flint/** crates/app/src/arch/flint.rs crates/gpu-gates/src/bin/shared/serve_seats/flint.rs crates/gpu/src/attention_h128.rs crates/gpu/src/attention_h128_ring.rs crates/gpu/src/flash_gqa_h128_swa.rs crates/gpu/src/qk_rms.rs crates/gpu-gates/src/bin/gate_flint_attn.rs crates/gpu-gates/src/bin/gate_flint_router.rs 7 no gemma-embedding F crates/model/src/arch/gemma_embedding/** crates/refset/src/arch/gemma_embedding/** crates/gpu-gates/src/bin/gate_gemma_embedding_*.rs 7 no gemma-embedding2 F crates/gpu-gemma-embedding2/** crates/model/src/arch/gemma_embedding2/** crates/refset/src/arch/gemma_embedding2/** crates/gpu-gates/src/bin/gate_gemma_embedding2_*.rs 7 no gemma4a F crates/gpu-gemma4a/** crates/vision/src/arch/gemma4a.rs crates/gpu-gates/src/bin/gate_gemma4a.rs 7 no diff --git a/tools/gate-class.tsv b/tools/gate-class.tsv index 51f3e8d67..6bf0f536b 100644 --- a/tools/gate-class.tsv +++ b/tools/gate-class.tsv @@ -75,6 +75,7 @@ gate-gpu-dspark-kv family deepseek41 draft drafts_dspark gate-gpu-e2e family deepseek2 e2e graph_fault+moe_route+weights_load gate-gpu-flint-attn family flint kernel attn_gqa gate-gpu-flint-router family flint kernel moe_route +gate-gpu-flint-lib family flint e2e graph_fault+attn_gqa+moe_route+tier_legs+placement_sizing gate-gpu-muse-router common kernel moe_route gate-gpu-gates-lib tooling harness - gate-gpu-gemm common kernel quant_gemm diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 14ab9d47e..9462bb612 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -113,7 +113,7 @@ tools/census/stubs.tsv derive the census manifest (refset::census::MANIFEST name # arch/coverage.rs and arch/qwen35moe/mtp.rs are carved out to their own rows below. crates/model/src/arch/{mod.rs,registry.rs,dspark.rs,classify.rs,stems.rs} derive gate-head gate-r8 gate-ptx-spill the dispatch's own files (Arch and spec, the registry, the shared architecture reads and the DSpark reader): every gate that links one, less the gguf-only kernel gates; the family modules behind the registry's rows are reached from them only by the gates that can open that family (tools/recipes.py, Reach) crates/model/src/arch/deepseek2/** derive gate-head gate-r8 the V2-Lite engine's reader and forward: the gates whose targets name it or call what it calls from another module (qwen35moe/mod.rs embeds through deepseek2::forward), and the gates that open a V2-Lite file (its profile, the names of their targets, the /models paths they name) -crates/model/src/arch/flint/** derive Flint's header-only reader and exact roles: its metadata gate and the common registry's architecture dispatch; no executable program or fixture +crates/model/src/arch/flint/** derive Flint's validated reader, placement and host expert adapter: metadata and the actual head128 body/app/serve consumers; no redistributed fixture crates/model/src/arch/qwen3moe/** derive the Qwen3 reader and plan: the gates whose targets name it, call it from another crate, or open a qwen3moe file (their profile, the names of their targets, the /models paths they name) crates/model/src/arch/glm5next/** derive the GLM reader and plan: the gates whose targets name it, call it from another crate (the GLM body's chains), or open a glm5next file (their profile, the names of their targets, the /models paths they name) crates/model/src/arch/mimo2/** derive weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve MiMo-V2.6-Flash's reader and plan: the gates that open a mimo2 file (its meta test, its flash gate, bloomery-serve's mimo2 seat), and the gates whose code calls it except gate-fixture: gate-fixture walks the registry's rows and runs no mimo2 reader on a file diff --git a/tools/hf-arch-check.py b/tools/hf-arch-check.py index 0cf29f393..af0111ddb 100755 --- a/tools/hf-arch-check.py +++ b/tools/hf-arch-check.py @@ -111,10 +111,10 @@ def load_by_path(name, *rel): "mimo2": ("mimo2",), } SERVES_VIA_FROM_NAME = {"glm": ("Glm5next",), "solar2": ("SolarOpen2",), "minimaxm3": ("MiniMaxM3",), "nemotron": ("NemotronH", "NemotronHMoe"), - "raon1": ("Raon1",), "musespark": ("MuseSpark",)} + "raon1": ("Raon1",), "musespark": ("MuseSpark",), "flint": ("Flint",)} # crates/gpu-gates/src/bin/bloomery_serve.rs:Model::ALL and Model::word — the generative seats in the order # the refusal of an unserved file lists them; crates/serve/src/decide.rs:WORD is the decide seat's word. -SEAT_ORDER = ("ds41", "qwen38", "glm", "qwen3", "mimo2", "solar2", "minimaxm3", "nemotron", "raon1", "musespark") +SEAT_ORDER = ("ds41", "qwen38", "glm", "qwen3", "mimo2", "solar2", "minimaxm3", "nemotron", "raon1", "musespark", "flint") DECIDE_WORD = "decide" # tools/arch-registry.tsv, the registry's print: FROM_NAME is crates/model/src/arch/mod.rs:Arch::from_name — variant -> the # strings that name it; SPEC is crates/model/src/arch/mod.rs:spec — the architectures the model reader takes (the @@ -1929,10 +1929,6 @@ def row_line(seat, arch, tensors=(), extra=(), vocab=0, quant="Q4_K_M", cards=No line, green, hub = row_line("glm", "glm5-next", small) check("a glm5-next file routes to the glm seat", green and "arch=glm5-next route=ok" in line, line) check("a tensor-type line parses", "embd=Q3_K exps=gate:Q4_K,down:Q6_K " in line, line) - line, green, _ = row_line("glm", "glm5nextx", small) - check("a glm5nextx file is refused by name", - not green and "arch=glm5nextx route=REFUSED: a glm5nextx file, which no seat serves; the seats are " - "--model ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, decide" in line, line) line, green, _ = row_line("glm", "deepseek41", small) check("a file the router seats elsewhere is refused for the row's seat", not green and "in the ds41 seat, and the README row says glm" in line, line) diff --git a/tools/levers-direct.txt b/tools/levers-direct.txt index c99b8511d..2c85c03aa 100644 --- a/tools/levers-direct.txt +++ b/tools/levers-direct.txt @@ -62,6 +62,7 @@ crates/model/tests/deepseek4_meta.rs BLOOMERY_REF_MODEL stays: the model file th crates/model/tests/ds41_meta.rs BLOOMERY_DSPARK_MODEL stays: the DSpark draft file crates/model-hf/tests/fixture.rs BLOOMERY_DSPARK_MODEL stays: the DSpark draft file crates/model/tests/flint_meta.rs BLOOMERY_MODEL stays: the pinned model file or header prefix the harness opens +crates/gpu-flint/src/tests.rs BLOOMERY_MODEL stays: cfg(test) full-file hardware consumer opens this exact local model path, not a production serving lever crates/model/tests/gemma_embedding_meta.rs BLOOMERY_MODEL stays: the model file the harness opens crates/model/tests/gemma_embedding2_meta.rs BLOOMERY_MODEL stays: the model file the harness opens crates/model/tests/mt.rs BLOOMERY_MT_CHILD_DUMP stays: the gate's handshake with the child it runs diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index 298e4b9c2..d46c3ed28 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -81,7 +81,7 @@ crates/gpu-gates/src/bin/shared/dspark_hf.rs qwen 1 baseline O 1: own path: an a crates/gpu-gates/src/bin/shared/emb2_image.rs gemma 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu-gates/src/bin/shared/route_trace38.rs qwen 3 baseline O 3: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu-gates/src/bin/shared/routed_legs.rs deepseek 8 baseline P 9: MK2 program by body -crates/gpu-gates/src/bin/shared/routed_legs.rs flint 2 PIN(2026-10-23) flint: family_of and skip_of match every models::Arch exhaustively; the header-only reader has no program or routed-legs adapter +crates/gpu-gates/src/bin/shared/routed_legs.rs flint 2 PIN(2026-10-25) flint: family_of and skip_of match every models::Arch exhaustively; no common routed-legs adapter is registered, and the full-file body transitions have their own owning gate crates/gpu-gates/src/bin/shared/routed_legs.rs gemma 4 baseline P 5: MK2 program by body crates/gpu-gates/src/bin/shared/routed_legs.rs glm 7 baseline P 8: MK2 program by body crates/gpu-gates/src/bin/shared/routed_legs.rs mimo 4 baseline P 5: MK2 program by body @@ -130,7 +130,7 @@ crates/gpu-vision/src/encoder.rs deepseek 2 baseline D 2: media design: VideoRul crates/gpu/src/arch/hybrid.rs nemotron 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu/src/card_slots.rs deepseek 24 PIN(2026-10-21) deepseek: baseline K 20 + 4, the four-slot entry `ds41_ffn_card_acc_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it crates/gpu/src/dspark/head.rs deepseek 3 baseline K 3: rename the kernel entries by what they compute (deferred) -crates/gpu/src/host/handoff.rs deepseek 38 PIN(2026-10-24) deepseek: baseline K 31 + 4 + 3, the four-slot tier entry `ds41_ffn_handoff_4_tier` (MiniMax-M3.1's bp) beside the four-slot entry `ds41_ffn_handoff_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it +crates/gpu/src/host/handoff.rs deepseek 39 PIN(2026-10-25) deepseek: fixed-slot tier preparation and launch select the existing generated ABI names beside the new six-slot entry; the per-entry module methods retain their pinned names until the deferred common-entry rename crates/hf/src/vocab.rs qwen 4 PIN(2026-10-20) qwen: the qwen35 tokenizer pipeline name and regex; tokenizer programs are not yet described by ModelSpec crates/metal/src/bin/bloomery_serve_metal.rs qwen 3 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string crates/metal/src/bin/generate_metal.rs qwen 4 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string @@ -163,7 +163,7 @@ crates/models/src/lib.rs mimo 2 baseline A 2: MK1a/MK1b ArchRow registry crates/models/src/lib.rs nemotron 4 baseline A 4: MK1a/MK1b ArchRow registry crates/models/src/lib.rs qwen 9 baseline A 9: MK1a/MK1b ArchRow registry crates/placement/src/accepts.rs deepseek 10 PIN(2026-10-21) deepseek: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet -crates/placement/src/accepts.rs flint 1 PIN(2026-10-23) flint: program_of matches every models::Arch; the header-only reader must explicitly return no program until a body is registered +crates/placement/src/accepts.rs flint 5 PIN(2026-10-25) flint: the implemented body registers its Program variant, name, architecture selection, expert arms and routed rule in the existing exhaustive program owners crates/placement/src/accepts.rs gemma 5 PIN(2026-10-21) gemma: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs glm 5 PIN(2026-10-21) glm: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs mimo 5 PIN(2026-10-21) mimo: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet @@ -216,3 +216,5 @@ crates/gpu-gates/src/record.rs muse 4 PIN(2026-10-23) muse: MK12 per-bin Kind st crates/model/src/arch/coverage.rs muse 25 PIN(2026-10-23) muse: the coverage rows, the type pins and the expert-stack match are keyed by Program, so each row the musespark body runs lists its variant; the program would own its rows instead of the list naming programs crates/hf/src/vocab.rs minimax 3 PIN(2026-10-23) minimax: the minimax-m2 pre-tokenizer's name, regex and pipeline row; tokenizer programs are not yet described by ModelSpec crates/gpu-gates/src/prompts.rs deepseek 1 PIN(2026-10-23) deepseek: the measured logit σ is named for the one family it was measured on (SIGMA_LOGIT_V41); other families carry it in their gate description until their own is measured +crates/gpu-gates/src/bin/bloomery_serve.rs flint 10 PIN(2026-10-25) flint: MK9 seat by program: the seat word, help/models list, architecture match and dispatch register the implemented shared hybrid seat until the description selects it +crates/model/src/arch/coverage.rs flint 11 PIN(2026-10-25) flint: the implemented Program selects its actual head128 attention, floor router, dense/shared sites, embedding/head and expert-stack capabilities in the existing coverage owners diff --git a/tools/recipes.py b/tools/recipes.py index 2ce417061..cfc84dce3 100755 --- a/tools/recipes.py +++ b/tools/recipes.py @@ -13031,8 +13031,6 @@ def _st_host_crates(fails: list[str], side: Side) -> None: ], f"host nightly: the real tree's set is {host_now} — a new member of the set is proven by a nightly run before this list takes it", ) - left = [n for n, _ in host_left_out(tree, lock_now)] - expect(left == ["bloomery-gpu", "bloomery-gpu-deepseek41", "bloomery-gpu-gates", "bloomery-gpu-gemma-embedding2", "bloomery-gpu-gemma4a", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-minimaxm3", "bloomery-gpu-solar2", "bloomery-gpu-vision"], f"host nightly: left out {left}") expect(not nightly_problems(tree, lock_now), f"host nightly: {nightly_problems(tree, lock_now)}") ran = {u for u, _ in host_units(tree, lock_now)} expect( diff --git a/tools/ref/dense-logits.py b/tools/ref/dense-logits.py index e3c185588..754416789 100644 --- a/tools/ref/dense-logits.py +++ b/tools/ref/dense-logits.py @@ -234,7 +234,7 @@ def native_build(args): env = os.environ.copy() env["CARGO_TARGET_DIR"] = str(ROOT / "target") argv = ["cargo", "oxide", "build", "--arch", args.arch, "--", "-p", "bloomery-gpu-gates", - "--features", "glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint", "--release", + "--features", "glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint", "--release", "--bin", "generate_qwen3moe", "--bin", "bloomery-serve"] subprocess.run(argv, cwd=ROOT, env=env, check=True, timeout=1800) if clean_head(ROOT) != head: diff --git a/tools/release/build.sh b/tools/release/build.sh index d48cd3277..0d999bf40 100755 --- a/tools/release/build.sh +++ b/tools/release/build.sh @@ -191,7 +191,7 @@ else bin_args=() for b in "${BINS[@]}"; do bin_args+=(--bin "$b"); done # The commit goes to cargo alone: the serve build script reads it, and a running binary never does. - BLOOMERY_GIT_COMMIT="$COMMIT" cargo oxide build --arch "$ARCH" -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release "${bin_args[@]}" + BLOOMERY_GIT_COMMIT="$COMMIT" cargo oxide build --arch "$ARCH" -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release "${bin_args[@]}" fi fail=0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 4af6e25ec..b6e058838 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -601,6 +601,9 @@ gguf 7 2 0 # block has its SAFETY line. # PIN(2026-10-25): gpu 2418 -> 2420 blocks, the six-slot F32 tier entry adds two # handoff body calls under its checked launch contract; the newmain baseline is preserved. +# PIN(2026-10-24): gpu 2420 -> 2475 blocks, 311 -> 315 fns, new code: checked +# head128 ring append/attention, optional-bias floor router, expert-input scaling +# and common loaded launchers. Existing entries retain their contracts; each new site states SAFETY. gpu 2640 334 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and @@ -624,7 +627,12 @@ gpu 2640 334 10 # PIN(2026-10-24): gpu-deepseek41 382 -> 384 blocks, new code, not old code relaxed: the four-slot tier card sum # (chain/ffn/batch.rs: ds41_ffn_card_acc_4_tier's card_acc_tier_at call and its store, 2 blocks), each block with its # SAFETY line. -gpu-deepseek41 384 44 0 +# PIN(2026-10-24): gpu-deepseek41 384 -> 386 blocks: load the configured +# residual module and launch its checked output rows, each with SAFETY. +gpu-deepseek41 386 44 0 +# PIN(2026-10-24): gpu-flint starts at 2 blocks, 0 fns and 0 impls: step and prompt +# call the checked common attention binding with staged distinct positions, each with SAFETY. +gpu-flint 2 0 0 # PIN(2026-10-01): gpu-gates 60 -> 61 blocks, new code, not old code relaxed: gate_glm5next_serve starts # bloomery-serve with PR_SET_PDEATHSIG set between fork and exec (one pre_exec block, its SAFETY line), as the # Qwen3.8 serve gate does, so a bound that ends the gate does not leave the server holding the card. @@ -690,7 +698,9 @@ gpu-deepseek41 384 44 0 # and the host flag's drop once the stream that waited on it is drained, each with its SAFETY line. # PIN(2026-10-24): gpu-gates 103 -> 104 blocks, new code, not old code relaxed: gate_qwen35moe_attn's launch of the # head-256 append at no query head over the rows rope_gate::run checks distinct, with its SAFETY line. -gpu-gates 104 5 1 +# PIN(2026-10-24): gpu-gates 104 -> 106 blocks: ring-attention consumers launch +# checked distinct positions and bounded chunks under their stated SAFETY invariants. +gpu-gates 106 5 1 # PIN(2026-10-15): gpu-gemma-embedding2 0 -> 1 blocks, new code, not old code relaxed: the head-512 # launcher call requires distinct Consts::pos positions, with its SAFETY line. # PIN(2026-10-15): gpu-gemma-embedding2 1 -> 2 blocks, new code, not old code relaxed: Encoder::pass's call of From 39c1a781fe7b220fa7b5f787c7f522c68b1b2f58 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:22:22 +0900 Subject: [PATCH 35/85] test(flint): align mixed expert fixtures with current model description --- crates/gpu/src/card_experts.rs | 2 ++ crates/gpu/src/site.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 5811138e5..3178bda9d 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -914,6 +914,8 @@ mod tests { top_k: USED as u32, expert_ff: FF as u32, latent: None, + pre_exp_norm: None, + post_exp_norm: None, act: models::Act::SwiGlu { limit: None }, router: Router { score: Score::Sigmoid, diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index f36ae5733..61b861fe2 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -1685,8 +1685,8 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, file_site, gemm_weight, - kgemv_order, swiglu_launch, + Form, SiteTy, SwigluLaunch, WideActs, WideForms, file_site, gemm_weight, kgemv_order, + swiglu_launch, }; use crate::kquant::Act; use gguf::Split; From 2802c4987de4c24486629f7518fe71e534a72211 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:27:29 +0900 Subject: [PATCH 36/85] test(flint): fix expert norm flag and record real-weight CPU consumers --- crates/gpu/src/card_experts.rs | 2 +- docs/research/flint-attach.md | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 3178bda9d..4fd16c23f 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -914,7 +914,7 @@ mod tests { top_k: USED as u32, expert_ff: FF as u32, latent: None, - pre_exp_norm: None, + pre_exp_norm: false, post_exp_norm: None, act: models::Act::SwiGlu { limit: None }, router: Router { diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index e76c7c082..1d4151727 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -196,8 +196,14 @@ currently lack genuine measurements in the four pinned architecture/capability t placeholder or copied hardware pin is supplied. Keep the integration Draft until that evidence is produced. No ROCm implementation or AMD execution is implied by this CUDA attachment. - - +Local CPU smoke on the complete artifact exercised all 47 routed layers with six fixed selected +experts, two columns and input amplitudes 1/8192. Single-column and union execution compared +385,024 f32 cells bit-for-bit (zero differences), with finite outputs and bounded protected +activations. This is actual expert-weight execution, not a whole-model decode or GPU oracle. +A separate public planner consumer checked group sizes 1/8, contexts 512/1024/2048 and slots 1/3; +all 12 plans conserved full/window KV bytes and repeated group configuration charged no second reserve. +The compiled CLI lists the Flint seat; opening it on this AMD-only machine stops at the named +NVIDIA R580+/`libcuda.so.1` prerequisite, so no HTTP generation run is reported. ## 1. Sources and observed artifact From 2ea4203b47c3eb6276b8f94652c80e710f7d0c4d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:37:51 +0900 Subject: [PATCH 37/85] fix(flint): include shared HTTP seat in published integration --- .../src/bin/shared/serve_seats/flint.rs | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 crates/gpu-gates/src/bin/shared/serve_seats/flint.rs diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs b/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs new file mode 100644 index 000000000..e74fdfcf8 --- /dev/null +++ b/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs @@ -0,0 +1,88 @@ +//! Flint's actual slot-capable body behind the common hybrid HTTP seat. +//! Chat, tools, reasoning and end-of-generation tokens use the canonical model contract. + +use bloomery_gpu_gates::generate::Place; +use bloomery_gpu_gates::{GateError, record, side_seat}; +use gguf::Split; +use model::arch::Arch; +use model::arch::flint::place; +use model::arch::place_hybrid::PlanInputs; +use model::placement::PlacementError; +use model::placement::workstation::TierBatchBytes; +use serve::ServeError; + +use super::hybrid::Family; + +const NAME: &str = "bloomery-serve-flint"; +const USAGE: &str = "usage: bloomery-serve --model flint [-m PATH | --hf [:]] \ + [--host H] [--port P] [--ctx C] [--parallel N] [--alias NAME] \ + [--chat-template-file PATH] [--place W] [--plan] [--queue-depth Q] \ + [--slot-save-path DIR] [--api-key KEY] [--api-key-file FNAME]"; + +/// The common placement, host load, residency, prompt-group and caller settings. +pub const ACTS_ON: &[&str] = &[ + bloomery_levers::CARD_BUDGET, + bloomery_levers::RESIDENCY, + bloomery_levers::PREFILL_GROUP, + bloomery_levers::HOST_POPULATE, + bloomery_levers::HOST_LOCK, + bloomery_levers::CARD_DONTNEED, + bloomery_levers::R8, + bloomery_levers::PIN_MAIN, + bloomery_levers::HOST_LANES, + bloomery_levers::SCREEN, + bloomery_levers::SCHEMAJUMP, +]; + +/// Architecture-specific facts; the common owner plans and serves every slot. +pub struct Flint; + +impl Family for Flint { + type Body = bloomery_gpu_flint::Body; + type H = (); + + const NAME: &'static str = self::NAME; + const USAGE: &'static str = self::USAGE; + const ACTS_ON: &'static [&'static str] = self::ACTS_ON; + const ALIAS: &'static str = "flint"; + const FAMILY: &'static str = "flint"; + const SERVES: &'static str = "flint"; + + fn accepts(arch: Arch) -> bool { + matches!(arch, Arch::Flint) + } + + fn read(split: &Split) -> Result, PlacementError> { + place::PlanInputs::read(split) + } + + fn configure(inputs: &mut PlanInputs<()>, group: usize) -> Result<(), GateError> { + Ok(place::configure_group(inputs, group)?) + } + + fn register() { + record::at_main(NAME, record::BLOOMERY_SERVE_HYBRID); + } + + fn listening() -> record::Record { + record::Record::new(&record::LISTENING_HYBRID) + } + + fn load_record() -> record::Record { + record::Record::new(&record::LOAD_HYBRID) + } + + fn tier_batch( + place: Place, + inputs: &PlanInputs<()>, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(inputs)) + .transpose()?) + } +} + +/// Opens the real Flint session and serves resident sequences by select and step. +pub fn run(args: &[String], side: &side_seat::Flags) -> Result { + super::hybrid::run::(args, side) +} From fb2c73da9b54a46c09ae433dcecb7dd066ef1f4d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Mon, 26 Oct 2026 03:44:17 +0900 Subject: [PATCH 38/85] test(gpu): pin measured Flint cc8.6 resources Record exact-head RTX 3090 gates and partial HTTP/oracle evidence. Pin only ten genuinely measured sm_86/cc8.6 spill and JIT-local rows; retain thirty cc12.1 gaps and the Korean greedy mismatch. No numerical or gate changes. --- docs/research/flint-attach.md | 58 +++++++++++++++++++++++++++++++++++ tools/ref/ptx-shapes.tsv | 14 +++++++++ 2 files changed, 72 insertions(+) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 1d4151727..941b4c78e 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -595,6 +595,64 @@ The Python reader intentionally skips array values. Their recorded values came f `gguf::inventory_of(...).meta`, not `None` values treated as absent GGUF facts. The recipe above revalidates the **whole header hash**, including those arrays, without a second parser. +## 10. Single-card NVIDIA acceptance and remaining numerical boundary + +The runtime at `313b9f0f88c90fcfae2365a4af53cd077762eb7c` was exercised on one RTX 3090, +cc8.6, driver 595.91.07 (CUDA API 13.2), with nightly-2026-08-28, cuda-oxide `87bc9518`, +LLVM 21.1.8 and CUDA toolkit 13.3.0 / ptxas 13.3.33. The complete 25,054,688,704-byte +GGUF was downloaded separately on that runner and its full SHA-256 matched +`63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f`. + +- `just gate-gpu-flint-attn` and `just gate-gpu-flint-router`: exit 0, including loaded + NoPE/NeoX ring consumers, floor/optional-bias routing, capture/replay and named faults. +- `just gate-gpu-flint-lib`: exit 0; the full-file capture/slots/reset/ring-boundary test + ran with **1 passed, 0 failed, 0 ignored**. Its bitwise consistency is not a mainline oracle. +- The real HTTP seat loaded 49 layers and two resident slots, exposed the original template, + generated English and Korean turns to EOG 150003, split reasoning, parsed a required + `get_weather` call with `{"location":"Seoul"}`, and preserved messages under concurrent + requests. The four stop spellings tokenized to the four expected IDs; natural emission of + all four stop IDs was not tested. +- A full `generate_ds41` scan and the pre-pin spill gate JITed all 436 entries without a + resource cache. The only ten violations were the new unpinned entries, each measured + `spill=0, jit_local=0`; the 26 vision entries passed. These observations supply only the + ten `sm_86`/cc8.6 rows. The three cc12.1 tables still need thirty genuine measurements. + +### Independent greedy comparison remains partly red + +The English raw continuation exactly matched pinned mainline CPU +`a37cc46104eb50900d5ab4a9ba02a0abdac039db`. The Korean continuation did not: the first +different generated ID is index 74, after the same 22 prompt IDs and 74 generated IDs +(next absolute position 96). Mainline chooses 26007 (`라`); Bloomery chooses 29263 (` 정`). +Both eventually answer `4`, which does not turn the comparison green. + +A standalone consumer linked to the preserved mainline library replayed that original +prompt-then-single-token trajectory with zero preceding raw-argmax mismatches. Its complete +150,144-logit rows at context 1024 and 2048 were bit-identical. At the divergence: + +| Observation | Margin `logit(26007) - logit(29263)` | Selected ID | +|---|---:|---:| +| Mainline CPU, original trajectory | +0.608115554 | 26007 | +| Bloomery probability diagnostic, original trajectory | -0.018128157 | 29263 | +| Bloomery, same 96 IDs supplied as one prefill | +0.361473739 | 26007 | + +The GPU margins are differences of reported pre-sampling log probabilities; the diagnostic +reproduced the original first 75 generated IDs. Probability collection reads each row and +uses plain per-token stepping, rather than establishing equality of every execution mode. + +This is not an exact tie or a demonstrated context-capacity defect. The common FFN owner +explicitly uses F32-activation Q8_0 GEMV for narrow/decode rows and quantized-activation +GEMM for wide prompt units (`shared_routed.rs`, `GATED_FROM=9`). Mainline also quantizes +the output head's activations and BF16 router operands differently. These are source-backed +numerical differences, not a proven localization of the Korean flip. No precision mode, +compiler pin, tie rule, tolerance or acceptance criterion was changed to force matching text. +Comparable follow-up traces must retain the original prompt/decode trajectory and can use +the existing 49-layer `Body::taps` and mainline `l_out` callbacks. + +Keep the integration Draft. Korean numerical/oracle acceptance, cc12.1 resource rows, +affected common-owner coverage beyond the exercised consumers, two-card operation, dynamic +residency/NVMe, streamed HTTP and spontaneous tool selection are not established by this run. + + [needs]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/models/src/need.rs#L505-L531 [stems]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/model/src/arch/stems.rs#L61-L125 [coverage]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/model/src/arch/coverage.rs#L1654-L1770 diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 06b852b20..491794d66 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -506,6 +506,20 @@ generate_ds41 swiglu_rule_rows 0 0 # PIN(2026-09-25): ds41_attn_seg_stage is the staged prefix segment pass (prefill only, never on the decode path); its ring ⧺ staging source keeps a few registers more than ds41_attn_seg, which already sits at the 128-register cap of a 512-thread block, and ptxas spills 8 bytes. ds41_attn_seg_sel_stage fits in 112 with none. # PIN(2026-10-21): raon_hc_route (Raon's mHC route, one block a token and layer) keeps its 4x3 and 4x4 route matrices in per-thread arrays the while loops index at run time: hz 48 B + e 64 B = 112 B of JIT local; the rest of its work is shared memory. ptxas spills nothing. +# PIN(2026-10-25): Flint entries measured at 313b9f0f88c90fcfae2365a4af53cd077762eb7c: +# sm_86, ptxas 13.3.33, RTX 3090 cc8.6, driver 595.91.07 (CUDA API 13.2). +# Full generate_ds41 JIT: 436 entries, cache off; each new entry has measured spill=0/jit_local=0. +generate_ds41 ffn_handoff_6_tier 0 0 +generate_ds41 gqa_flash_merge_h128_swa_ring 0 0 +generate_ds41 gqa_flash_seg_h128_swa_ring 0 0 +generate_ds41 head_norm_neox_append_ring 0 0 +generate_ds41 normalize 0 0 +generate_ds41 qk_rms_append_ring 0 0 +generate_ds41 restore 0 0 +generate_ds41 routed_shared_residual 0 0 +generate_ds41 sigmoid_floor_256_6_router 0 0 +generate_ds41 sigmoid_floor_256_6_router_pick 0 0 + # gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. # PIN(2026-10-15): the seven Gemma4 entries have zero sm_86 ptxas spill; jit_local=0 is a derived cc8.6 ceiling, # not a cc8.6 driver measurement. Both sm_86 and sm_121 PTX have zero JIT local bytes on GB10 cc12.1. From c691a90270a0ac4bb6176e6f1abda3dc5c6e493c Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Mon, 26 Oct 2026 05:29:24 +0900 Subject: [PATCH 39/85] test(flint): capture forced trajectories and cover real two-card consumers --- crates/gpu-flint/examples/trace_flint.rs | 473 ++++++++++++++++++ crates/gpu-flint/src/lib.rs | 3 + crates/gpu-flint/src/two_card_tests.rs | 397 +++++++++++++++ .../src/bin/gate_glm5next_twocard.rs | 3 +- .../src/bin/gate_minimaxm3_prefill.rs | 9 +- .../src/bin/gate_musespark_prefill.rs | 7 +- .../src/bin/gate_qwen3moe_twocard.rs | 3 +- .../gpu-gates/src/bin/gate_solar2_twocard.rs | 3 +- .../gpu-gates/src/bin/shared/mimo2_twocard.rs | 8 +- .../src/bin/shared/minimaxm3_seat.rs | 11 +- .../gpu-gates/src/bin/shared/tier_twocard.rs | 64 +-- crates/placement/src/placement.rs | 275 ++++++++++ docs/research/flint-attach.md | 110 ++++ justfile | 8 +- tools/gate-batch-resources.tsv | 5 + tools/gate-class.tsv | 1 + tools/gate-paths.tsv | 1 + tools/levers-direct.txt | 1 + tools/ref/flint_trace.cpp | 355 +++++++++++++ tools/unsafe-ratchet.txt | 8 +- 20 files changed, 1653 insertions(+), 92 deletions(-) create mode 100644 crates/gpu-flint/examples/trace_flint.rs create mode 100644 crates/gpu-flint/src/two_card_tests.rs create mode 100644 tools/ref/flint_trace.cpp diff --git a/crates/gpu-flint/examples/trace_flint.rs b/crates/gpu-flint/examples/trace_flint.rs new file mode 100644 index 000000000..9bfa0b6ed --- /dev/null +++ b/crates/gpu-flint/examples/trace_flint.rs @@ -0,0 +1,473 @@ +//! Forced-ID Flint trajectory capture. Diagnostic artifacts, not an accuracy gate or refset. + +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io::{BufWriter, Write}; +use std::num::NonZeroUsize; +use std::path::{Path, PathBuf}; + +use bloomery_gpu::host::served::prompt_pass; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::model::StepMode; +use bloomery_gpu_flint::{Body, FlintModel, prefill, set_prefill_group, set_taps}; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::{self, PlanInputs}; +use model::placement::{Plan, PlanLevers, workstation}; + +type Error = Box; +type Result = std::result::Result; + +const WHAT: &str = "trace_flint"; +const WIDTH: usize = 2048; +const LAYERS: usize = 49; +const VOCAB: usize = 150_144; +const PROMPT: usize = 22; +const TOTAL: usize = 96; +const CALLS: usize = TOTAL - PROMPT + 1; +const CTX: usize = 1024; +const SLOTS: usize = 2; +const USAGE: &str = "Usage: trace_flint -m MODEL --ids FILE --prompt 22 --out NEWDIR --ctx 1024 --slots 2 --mode eager|graph [--taps]\n\ +Captures flint-trajectory-v1: one 22-ID batched prefill, then 74 forced one-ID steps.\n\ +FILE must contain exactly 96 decimal token IDs; no tokenization, sampling or EOG stopping.\n\ +--ctx is per-slot (total context 2048); exactly one CUDA device must be visible.\n\ +--taps captures each call's last-position layer outputs and result_norm.\n\ +Without --taps those tensors are explicitly absent; full logits are always captured.\n\ +The output directory must not exist. MANIFEST.tsv is published only after all files finish.\n\ +-h, --help print this usage without opening CUDA. --levers prints the existing lever table."; + +struct Args { + model: PathBuf, + ids: PathBuf, + out: PathBuf, + mode: StepMode, + taps: bool, + plan_levers: PlanLevers, +} + +fn assign(slot: &mut Option, value: T, name: &str) -> Result<()> { + if slot.replace(value).is_some() { + return Err(format!("{name}: duplicate option").into()); + } + Ok(()) +} + +fn parse() -> Result> { + let mut args = std::env::args_os().skip(1); + let (mut model, mut ids, mut out, mut mode) = (None, None, None, None); + let (mut prompt, mut ctx, mut slots) = (None, None, None); + let mut taps = false; + while let Some(arg) = args.next() { + let option = arg.to_str().ok_or("option is not UTF-8")?; + if matches!(option, "-h" | "--help") { + println!("{USAGE}"); + return Ok(None); + } + if option == "--levers" { + continue; + } + if option == "--taps" { + if taps { + return Err("--taps: duplicate option".into()); + } + taps = true; + continue; + } + if !matches!( + option, + "-m" | "--ids" | "--out" | "--mode" | "--prompt" | "--ctx" | "--slots" + ) { + return Err(format!("unknown option {option:?}\n{USAGE}").into()); + } + let value: OsString = args + .next() + .ok_or_else(|| format!("{option}: missing value"))?; + match option { + "-m" => assign(&mut model, PathBuf::from(value), option)?, + "--ids" => assign(&mut ids, PathBuf::from(value), option)?, + "--out" => assign(&mut out, PathBuf::from(value), option)?, + "--mode" => { + let value = match value.to_str() { + Some("eager") => StepMode::Eager, + Some("graph") => StepMode::Graph, + _ => return Err("--mode: expected eager or graph".into()), + }; + assign(&mut mode, value, option)?; + } + "--prompt" | "--ctx" | "--slots" => { + let value = value.to_str().ok_or("numeric option is not UTF-8")?; + if value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { + return Err(format!("{option}: expected an unsigned decimal integer").into()); + } + let value: usize = value.parse()?; + let slot = match option { + "--prompt" => &mut prompt, + "--ctx" => &mut ctx, + _ => &mut slots, + }; + assign(slot, value, option)?; + } + _ => unreachable!(), + } + } + // This exits through the common parser before any CUDA call. + let levers = bloomery_levers::at_main(&[bloomery_levers::CARD_BUDGET])?; + if prompt != Some(PROMPT) || ctx != Some(CTX) || slots != Some(SLOTS) { + return Err( + format!("required: --prompt {PROMPT} --ctx {CTX} --slots {SLOTS}\n{USAGE}").into(), + ); + } + Ok(Some(Args { + model: model.ok_or("required: -m MODEL")?, + ids: ids.ok_or("required: --ids FILE")?, + out: out.ok_or("required: --out NEWDIR")?, + mode: mode.ok_or("required: --mode eager|graph")?, + taps, + plan_levers: PlanLevers::from_levers(&levers)?, + })) +} + +fn tsv_path(path: &Path) -> Result<&str> { + let text = path.to_str().ok_or("artifact path is not UTF-8")?; + if text.contains(['\t', '\n', '\r']) { + return Err("artifact path contains a TSV delimiter".into()); + } + Ok(text) +} + +fn read_ids(path: &Path) -> Result> { + let text = fs::read_to_string(path)?; + let ids = text + .split_whitespace() + .map(|word| { + if !word.bytes().all(|b| b.is_ascii_digit()) { + return Err(format!("invalid decimal token ID {word:?}").into()); + } + let id: u32 = word.parse()?; + if id as usize >= VOCAB { + return Err(format!("token ID {id} is outside vocabulary {VOCAB}").into()); + } + Ok(id) + }) + .collect::>>()?; + if ids.len() != TOTAL { + return Err(format!("expected exactly {TOTAL} IDs, found {}", ids.len()).into()); + } + Ok(ids) +} + +fn create(path: &Path) -> Result> { + Ok(BufWriter::new( + OpenOptions::new().write(true).create_new(true).open(path)?, + )) +} + +fn finish(mut file: BufWriter) -> Result<()> { + file.flush()?; + file.get_ref().sync_all()?; + Ok(()) +} + +fn floats(file: &mut BufWriter, name: &str, values: &[f32], len: usize) -> Result<()> { + if values.len() != len { + return Err(format!("{name}: expected {len} floats, found {}", values.len()).into()); + } + if let Some(index) = values.iter().position(|x| !x.is_finite()) { + return Err(format!("{name}: non-finite float at {index}").into()); + } + for value in values { + file.write_all(&value.to_le_bytes())?; + } + Ok(()) +} + +fn placement(out: &Path, plan: &Plan<'_>, m: &FlintModel) -> Result<()> { + let mut rows = create(&out.join("placement.tsv"))?; + writeln!( + rows, + "tensor\tshard\tlayer\trole\tfile_type\tdims\tstage\tdevice\tformat\texpert_ids\tresident_bytes\tread_bytes" + )?; + for row in &plan.rows { + let tensor = &plan.model.tensors[row.tensor]; + for segment in &row.segments { + writeln!( + rows, + "{}\t{}\t{:?}\t{:?}\t{:?}\t{:?}\t{}\t{:?}\t{:?}\t{}\t{}\t{}", + tensor.name, + tensor.shard, + tensor.layer, + tensor.role, + tensor.ty, + tensor.dims, + row.stage, + segment.device, + segment.format, + segment + .experts + .as_ref() + .map_or_else(|| "absent".into(), ToString::to_string), + segment.resident_bytes, + row.read_bytes + )?; + } + } + finish(rows)?; + let body = m.body(WHAT)?; + let map = body.hybrid().slots(); + let mut slots = create(&out.join("expert-slots.tsv"))?; + writeln!(slots, "layer\texpert_id\tactual_slot")?; + for layer in map.layers() { + for id in 0..map.n_expert() { + let id = u32::try_from(id)?; + writeln!( + slots, + "{layer}\t{id}\t{:?}", + map.slot(layer, id).ok_or("missing loaded expert slot")? + )?; + } + } + finish(slots)?; + let mut details = create(&out.join("placement.txt"))?; + writeln!( + details, + "machine={:#?}\ncard_totals={:#?}\nhost_totals={:#?}\ncard_budget={:?}\nbudget_from_pool={}\nn_l={:?}\ntier_n_l={:?}\nnvme_bytes={}\nloaded_ctx={}\nloaded_store_bytes_per_slot={}\nloaded_scratch_bytes={}\nloaded_prompt_bytes={}", + plan.machine, + plan.cards, + plan.host, + plan.card_budget, + plan.budget_from_pool, + plan.n_l, + plan.tier_n_l, + plan.nvme_bytes, + body.ctx(), + body.store_bytes(), + body.scratch_bytes(), + body.prompt_bytes() + )?; + finish(details) +} + +struct Capture { + steps: BufWriter, + graph: BufWriter, + counters: BufWriter, + logits: BufWriter, + taps: Option<(BufWriter, BufWriter)>, + row: Vec, + graph_nodes: Option, +} + +impl Capture { + fn new(out: &Path, taps: bool) -> Result { + let mut steps = create(&out.join("steps.tsv"))?; + writeln!(steps, "call\tprocessed_pos\tpredict_pos\tinput_id\targmax")?; + let mut graph = create(&out.join("graph.tsv"))?; + writeln!(graph, "call\tmode\thas_capture\tdecode_nodes")?; + let mut counters = create(&out.join("counters.tsv"))?; + writeln!(counters, "call\thost_stats\ttier_stats")?; + Ok(Self { + steps, + graph, + counters, + logits: create(&out.join("logits.f32"))?, + taps: if taps { + Some(( + create(&out.join("layers.f32"))?, + create(&out.join("norm.f32"))?, + )) + } else { + None + }, + row: vec![0.0; VOCAB], + graph_nodes: None, + }) + } + + fn call(&mut self, m: &FlintModel, call: usize, input: u32, argmax: u32) -> Result<()> { + let processed = PROMPT - 1 + call; + if m.pos() as usize != processed + 1 || m.n_vocab()? != VOCAB || argmax as usize >= VOCAB { + return Err(format!("call {call}: invalid position, vocabulary or argmax").into()); + } + // Read the last call's taps before another prompt/step can overwrite them. + if let Some((layers, norm)) = &mut self.taps { + let values = m.body(WHAT)?.taps(m.gpu())?; + floats(layers, "layers", &values, LAYERS * WIDTH)?; + floats(norm, "result_norm", &m.head_normed()?, WIDTH)?; + } + m.logits_into(&mut self.row)?; + floats(&mut self.logits, "logits", &self.row, VOCAB)?; + let best = self.row.iter().copied().fold(f32::NEG_INFINITY, f32::max); + if self.row[argmax as usize] != best { + return Err(format!( + "call {call}: device argmax {argmax} does not maximize saved logits" + ) + .into()); + } + let captured = m.has_capture(); + if m.mode() == StepMode::Graph && call != 0 && !captured { + return Err(format!("call {call}: graph mode did not capture a decode chain").into()); + } + if m.mode() == StepMode::Eager && captured { + return Err(format!("call {call}: eager mode unexpectedly holds a capture").into()); + } + let nodes = if captured { + if let Some(nodes) = self.graph_nodes { + nodes + } else { + let nodes = m.step_graph_nodes()?.len(); + self.graph_nodes = Some(nodes); + nodes + } + } else { + 0 + }; + if captured && nodes == 0 { + return Err("captured decode graph is empty".into()); + } + writeln!( + self.steps, + "{call}\t{processed}\t{}\t{input}\t{argmax}", + processed + 1 + )?; + writeln!(self.graph, "{call}\t{:?}\t{captured}\t{nodes}", m.mode())?; + let hybrid = m.body(WHAT)?.hybrid(); + writeln!( + self.counters, + "{call}\t{:?}\t{:?}", + hybrid.stats(), + hybrid.tier_stats() + )?; + Ok(()) + } + + fn finish(self) -> Result<()> { + finish(self.steps)?; + finish(self.graph)?; + finish(self.counters)?; + finish(self.logits)?; + if let Some((layers, norm)) = self.taps { + finish(layers)?; + finish(norm)?; + } + Ok(()) + } +} + +fn run(args: Args) -> Result<()> { + let model = fs::canonicalize(&args.model)?; + let ids_path = fs::canonicalize(&args.ids)?; + tsv_path(&model)?; + tsv_path(&ids_path)?; + let ids = read_ids(&ids_path)?; + let file = Split::open(&model)?; + let mut inputs = PlanInputs::read(&file)?; + if inputs.spec.layers.len() != LAYERS + || inputs.spec.hidden as usize != WIDTH + || inputs.spec.vocab as usize != VOCAB + { + return Err("expected the full 49-layer, width-2048, vocab-150144 Flint model".into()); + } + place::configure_group(&mut inputs, 1)?; + // Atomic directory creation rejects existing directories, files and symlinks before CUDA is touched. + fs::create_dir(&args.out)?; + let out = fs::canonicalize(&args.out)?; + let mut tokens = create(&out.join("tokens.ids"))?; + for id in &ids { + writeln!(tokens, "{id}")?; + } + finish(tokens)?; + let census = bloomery_gpu::census()?; + let [device] = census.as_slice() else { + return Err(format!( + "exactly one CUDA device must be visible, found {}", + census.len() + ) + .into()); + }; + let mut hardware = create(&out.join("device.txt"))?; + writeln!(hardware, "{device:#?}")?; + finish(hardware)?; + let machine = workstation::plan_on(workstation::spec_of_device(device), LAYERS); + let slots = NonZeroUsize::new(SLOTS).ok_or("zero resident slots")?; + let plan = inputs.plan_with_slots(&machine, CTX as u64, &args.plan_levers, slots)?; + let host = HostCfg { + populate: false, + lock: false, + card_dontneed: false, + r8: false, + route_trace: false, + }; + let mut m = Body::open_placed_slots(file, &plan, &inputs, 0, host, Residency::Off, slots)?; + m.add_slots(SLOTS)?; + if m.body(WHAT)?.ctx() != CTX { + return Err("loaded per-slot context is not 1024".into()); + } + set_prefill_group(&mut m, 1)?; + if args.taps { + set_taps(&mut m, true)?; + } + // Arming taps selects eager and invalidates captures; the requested arm is selected afterwards. + m.set_mode(args.mode); + placement(&out, &plan, &m)?; + let mut capture = Capture::new(&out, args.taps)?; + let argmax = prompt_pass(&mut m, |m| { + m.run_rows(PROMPT, WHAT, |gpu, weights, body, head, at| { + prefill(gpu, weights, body, head, &ids[..PROMPT], at) + }) + })? + .ok_or("batched prompt produced no final argmax")?; + capture.call(&m, 0, ids[PROMPT - 1], argmax)?; + for (offset, &id) in ids[PROMPT..].iter().enumerate() { + let argmax = m.step(&[id])?; + capture.call(&m, offset + 1, id, argmax)?; + } + let mut nodes = create(&out.join("graph-nodes.tsv"))?; + writeln!(nodes, "node\tdescriptor")?; + if args.mode == StepMode::Graph { + for (index, node) in m.step_graph_nodes()?.iter().enumerate() { + writeln!(nodes, "{index}\t{node:?}")?; + } + } + finish(nodes)?; + capture.finish()?; + let mut manifest = create(&out.join("MANIFEST.tsv.partial"))?; + let mode = match args.mode { + StepMode::Eager => "eager", + StepMode::Graph => "graph", + }; + let instrumentation = if args.taps { + "layer-head-taps" + } else { + "no-taps" + }; + let layers_file = if args.taps { + "layers.f32" + } else { + "absent:not-requested" + }; + let norm_file = if args.taps { + "norm.f32" + } else { + "absent:not-requested" + }; + writeln!( + manifest, + "protocol\tflint-trajectory-v1\nmodel\t{}\nids_source\t{}\nengine\tbloomery-gpu-flint\nmode\t{mode}\ninstrumentation\t{instrumentation}\nctx_per_slot\t{CTX}\nctx_total\t{}\nslots\t{SLOTS}\nactive_slot\t0\nwidth\t{WIDTH}\nlayers\t{LAYERS}\nvocab\t{VOCAB}\nprompt\t{PROMPT}\ntotal_ids\t{TOTAL}\ncalls\t{CALLS}\nlogits\tlogits.f32\nlayers_file\t{layers_file}\nnorm_file\t{norm_file}\nlogits_layout\tf32le[call][150144]\nlayers_layout\tf32le[call][49][2048]\nnorm_layout\tf32le[call][2048]\nresidency\toff\nhost_populate\tfalse\nhost_lock\tfalse\ncard_dontneed\tfalse\nr8\tfalse\nroute_trace\tfalse\nprefill_group\t1\nprompt_submission\tone-batched-prefill\nprompt_mode\teager-batched\ndecode_graph_captured\t{}\nplacement\tplacement.tsv\nloaded_expert_slots\texpert-slots.tsv\nplacement_counters\tplacement.txt\ndevice\tdevice.txt\ngraph_state\tgraph.tsv\ngraph_nodes\tgraph-nodes.tsv\nruntime_counters\tcounters.tsv\nprovenance\texternal-receipt-required\ncomplete\t1", + tsv_path(&model)?, + tsv_path(&ids_path)?, + CTX * SLOTS, + m.has_capture() + )?; + finish(manifest)?; + fs::rename(out.join("MANIFEST.tsv.partial"), out.join("MANIFEST.tsv"))?; + File::open(&out)?.sync_all()?; + Ok(()) +} + +fn main() { + let result = parse().and_then(|args| args.map_or(Ok(()), run)); + if let Err(error) = result { + eprintln!("{WHAT}: {error}"); + std::process::exit(1); + } +} diff --git a/crates/gpu-flint/src/lib.rs b/crates/gpu-flint/src/lib.rs index 57ceacb09..c8a33bb62 100644 --- a/crates/gpu-flint/src/lib.rs +++ b/crates/gpu-flint/src/lib.rs @@ -11,3 +11,6 @@ pub use prefill::{prefill, set_prefill_group}; #[cfg(test)] mod tests; + +#[cfg(test)] +mod two_card_tests; diff --git a/crates/gpu-flint/src/two_card_tests.rs b/crates/gpu-flint/src/two_card_tests.rs new file mode 100644 index 000000000..ccc90411c --- /dev/null +++ b/crates/gpu-flint/src/two_card_tests.rs @@ -0,0 +1,397 @@ +//! The real six-slot tier protocol against the same plan's one-card expert union. + +use std::path::Path; + +use bloomery_gpu::fault::{Fault, FaultSite}; +use bloomery_gpu::host::served::prompt_pass; +use bloomery_gpu::host::tier::TierStats; +use bloomery_gpu::model::StepMode; +use bloomery_gpu::{GpuError, hybrid::HOST}; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::{PlanInputs, tier_batch}; +use model::placement::{Plan, PlanLevers, union_plan, workstation}; +use models::Ffn; + +use crate::{Body, FlintModel, set_taps}; + +const WHAT: &str = "flint two-card full-file acceptance"; +const CTX: usize = 1024; +const PROMPT: usize = 22; +const IDS: usize = 96; +const MODEL_BYTES: u64 = 25_054_688_704; +// PIN(2026-10-25): full-file SHA from the pinned Flint diagnostic model receipt. +const MODEL_SHA256: &str = "63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f"; + +type Error = Box; + +#[derive(Debug, PartialEq, Eq)] +struct Observation { + position: u32, + argmax: u32, + logits: Vec, + layers: Vec, +} + +fn observe(m: &FlintModel, argmax: u32) -> Result { + let logits = m.logits()?; + let layers = m.body(WHAT)?.taps(m.gpu())?; + assert_eq!(logits.len(), 150_144, "{WHAT}: full vocabulary"); + assert_eq!( + layers.len(), + 49 * 2048, + "{WHAT}: every layer's merge output" + ); + assert!( + logits.iter().chain(&layers).all(|v| v.is_finite()), + "{WHAT}: completed finite outputs, not an unfilled tier row" + ); + Ok(Observation { + position: m.pos(), + argmax, + logits: logits.into_iter().map(f32::to_bits).collect(), + layers: layers.into_iter().map(f32::to_bits).collect(), + }) +} + +fn same(label: &str, want: &[Observation], got: &[Observation]) { + assert_eq!(got.len(), want.len(), "{label}: completed calls"); + for (call, (w, g)) in want.iter().zip(got).enumerate() { + assert_eq!( + (g.position, g.argmax), + (w.position, w.argmax), + "{label}: call {call} position and token" + ); + for (name, a, b) in [ + ("logits", &g.logits, &w.logits), + ("layer outputs", &g.layers, &w.layers), + ] { + let first = a.iter().zip(b).position(|(a, b)| a != b); + assert!( + a.len() == b.len() && first.is_none(), + "{label}: call {call} {name} first differing bits {first:?}" + ); + } + } +} + +fn prompt(m: &mut FlintModel, ids: &[u32]) -> Result { + prompt_pass(m, |m| { + m.run_rows(ids.len(), WHAT, |gpu, w, body, head, at| { + crate::prefill(gpu, w, body, head, ids, at) + }) + })? + .ok_or(GpuError::State { + what: WHAT, + missing: "the batched prompt's last token", + }) +} + +fn tier_stats(m: &FlintModel) -> Result { + let tiers = m.body(WHAT)?.hybrid().tiers(); + assert_eq!(tiers.len(), 1, "{WHAT}: one actual expert tier"); + Ok(tiers[0].stats()) +} + +fn sent_every(before: &TierStats, after: &TierStats, layers: &[usize], first: usize) { + for &layer in layers { + let i = layer - first; + assert!( + after.layer_hits[i] > before.layer_hits[i], + "{WHAT}: real routing sent no tier slot at layer {layer}" + ); + } +} + +/// Read the actual router's six ranked picks and the handoff's published tier places. +fn published(m: &FlintModel, layer: usize) -> Result, GpuError> { + let gpu = m.gpu(); + gpu.context().bind_to_thread()?; + let body = m.body(WHAT)?; + let picks = body.ffn.route_ids().to_host_vec(gpu.stream())?; + assert_eq!(picks.len(), 6, "{WHAT}: six real routed slots"); + assert!(picks.iter().all(|&id| id < 256), "{WHAT}: valid expert ids"); + for (i, id) in picks.iter().enumerate() { + assert!(!picks[..i].contains(id), "{WHAT}: distinct top-six picks"); + } + let side = body.ffn.tier().expect("the stage has a tier join"); + let places = side.tsel(0)?.to_host_vec(gpu.stream())?; + let tier = &body.hybrid().tiers()[0]; + let ids = tier + .set() + .ids(layer) + .expect("last routed layer in tier set"); + let expected: Vec = picks + .iter() + .map(|id| ids.iter().position(|i| i == id).map_or(HOST, |i| i as u32)) + .collect(); + assert_eq!( + places, expected, + "{WHAT}: rank-preserving six-slot publication" + ); + let at = body.s.pos.to_host_vec(gpu.stream())?; + assert_eq!( + at, + [m.pos() - 1], + "{WHAT}: replay refreshed its position input" + ); + gpu.stream().synchronize()?; + Ok(picks) +} + +fn history( + m: &mut FlintModel, + ids: &[u32], + mode: StepMode, + tier_layers: &[usize], +) -> Result, GpuError> { + m.reset()?; + m.set_mode(mode); + let tiered = !tier_layers.is_empty(); + let first = if tiered { + m.body(WHAT)?.hybrid().tiers()[0].set().layers().start + } else { + 0 + }; + let before = tiered.then(|| tier_stats(m)).transpose()?; + let token = prompt(m, &ids[..PROMPT])?; + let mut out = vec![observe(m, token)?]; + let after_prompt = tiered.then(|| tier_stats(m)).transpose()?; + if let (Some(b), Some(a)) = (&before, &after_prompt) { + sent_every(b, a, tier_layers, first); + let batch = m.body(WHAT)?.hybrid().tier_batch_stats(); + assert_eq!(batch.len(), 1, "{WHAT}: actual tier batch port"); + assert!( + batch[0].served > 0 && batch[0].settles > 0, + "{WHAT}: batched prompt published and completed tier work" + ); + } + let mut previous_picks = None; + let mut changed_picks = false; + let mut graph_nodes = None; + for &id in &ids[PROMPT..] { + let token = m.step(&[id])?; + out.push(observe(m, token)?); + if mode == StepMode::Graph { + assert!(m.has_capture(), "{WHAT}: graph mode really captured"); + let nodes = m.step_graph_nodes()?.len(); + assert!(nodes > 0, "{WHAT}: nonempty captured decode"); + if let Some(previous) = graph_nodes { + assert_eq!( + nodes, previous, + "{WHAT}: changed inputs reuse the captured chain" + ); + } + graph_nodes = Some(nodes); + } + if let Some(&last) = tier_layers.last() { + let picks = published(m, last)?; + changed_picks |= previous_picks.as_ref().is_some_and(|p| *p != picks); + previous_picks = Some(picks); + } + } + assert_eq!( + m.pos(), + IDS as u32, + "{WHAT}: consumed the whole teacher-forced history" + ); + if let Some(before) = after_prompt { + let after = tier_stats(m)?; + sent_every(&before, &after, tier_layers, first); + assert!( + after.issued > before.issued && after.settles > before.settles, + "{WHAT}: decode published and completed actual tier work" + ); + assert!( + changed_picks, + "{WHAT}: replay refreshed actual routing, not just tensor shapes" + ); + if mode == StepMode::Graph { + assert!( + after.captures > before.captures && after.replays > before.replays, + "{WHAT}: the second physical card captured and replayed its leg" + ); + } + } + Ok(out) +} + +fn open(path: &Path, plan: &Plan<'_>, inputs: &PlanInputs) -> Result { + let host = HostCfg { + populate: false, + lock: false, + card_dontneed: false, + r8: false, + route_trace: false, + }; + let mut m = Body::open_placed(Split::open(path)?, plan, inputs, 0, host)?; + set_taps(&mut m, true)?; + Ok(m) +} + +#[test] +#[ignore = "needs exactly two visible CUDA devices, the complete pinned Flint GGUF in BLOOMERY_MODEL, and host RAM for its host experts"] +fn hw_two_card_full_file_union_capture_publication_fault_reset() -> Result<(), Error> { + let path = std::env::var_os("BLOOMERY_MODEL") + .ok_or("flint two-card acceptance: BLOOMERY_MODEL must name the complete pinned GGUF")?; + let path = Path::new(&path); + assert_eq!( + std::fs::metadata(path)?.len(), + MODEL_BYTES, + "{WHAT}: complete file" + ); + assert_eq!( + gguf::digest::sha256_file(path)?, + MODEL_SHA256, + "{WHAT}: pinned model bytes" + ); + let file = Split::open(path)?; + let inputs = PlanInputs::read(&file)?; + assert_eq!( + ( + inputs.spec.layers.len(), + inputs.spec.hidden, + inputs.spec.vocab + ), + (49, 2048, 150_144), + "{WHAT}: released geometry" + ); + assert_eq!( + inputs.model.experts_used, 6, + "{WHAT}: production top-six program" + ); + let census = bloomery_gpu::census()?; + let [stage, tier] = census.as_slice() else { + return Err(format!( + "{WHAT}: exactly two CUDA devices must be visible, found {}", + census.len() + ) + .into()); + }; + let stage = workstation::spec_of_device(stage); + let tier = workstation::spec_of_device(tier); + assert!( + !stage.same_device(&tier), + "{WHAT}: two distinct physical devices" + ); + // A reduced budget makes real tier assignments while leaving room for the union reference. + // It is bounded by each actual free reading, not an assumed card model or the file's length. + let budget = (8u64 << 30) + .min(stage.free_bytes.unwrap_or(stage.usable_bytes()) / 3) + .min(tier.free_bytes.unwrap_or(tier.usable_bytes()) / 2); + let machine = workstation::plan_tiers(49, stage, &[tier], None, tier_batch(&inputs)?)?; + let mut flat = machine.clone(); + flat.tiers.clear(); + let levers = PlanLevers { + card_budget_bytes: Some(budget).into(), + }; + let plan = inputs.plan(&machine, CTX as u64, &levers)?; + let routed: Vec = inputs + .spec + .layers + .iter() + .enumerate() + .filter_map(|(i, l)| matches!(l.ffn, Ffn::Moe(_)).then_some(i)) + .collect(); + assert_eq!(plan.tier_n_l.len(), 1, "{WHAT}: one planned tier"); + for &l in &routed { + assert!( + plan.n_l[l] > 0 && plan.tier_n_l[0][l] > 0, + "{WHAT}: layer {l} must assign actual experts to both physical cards, budget {budget}" + ); + } + let union = union_plan(&plan, &flat)?; + assert!( + union.cards[0].headroom_bytes >= i128::from(flat.cards[0].margin_bytes), + "{WHAT}: the actual stage must fit the union and its reserves" + ); + let mut state = 0x4f1a_29c3u32; + let ids: Vec = (0..IDS) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + state % inputs.spec.vocab + }) + .collect(); + assert!( + ids.windows(2).all(|w| w[0] != w[1]), + "{WHAT}: changed decode inputs" + ); + let reference = { + let mut m = open(path, &union, &inputs)?; + assert!( + m.body(WHAT)?.hybrid().tiers().is_empty(), + "{WHAT}: reference computes the union on the stage" + ); + history(&mut m, &ids, StepMode::Eager, &[])? + }; + let mut m = open(path, &plan, &inputs)?; + let actual_stage = m.gpu().device_id()?; + let actual_tier = m.body(WHAT)?.hybrid().tiers()[0].gpu()?.device_id()?; + assert_ne!( + actual_stage.uuid, actual_tier.uuid, + "{WHAT}: not same-GPU wrappers" + ); + assert_eq!(actual_stage, stage.device.expect("census stage identity")); + assert_eq!(actual_tier, tier.device.expect("census tier identity")); + let eager = history(&mut m, &ids, StepMode::Eager, &routed)?; + same("two-card eager versus same-plan union", &reference, &eager); + let graph = history(&mut m, &ids, StepMode::Graph, &routed)?; + same( + "two-card capture versus same-plan union", + &reference, + &graph, + ); + + let fault = Fault::at(routed[0] as u32, FaultSite::QuantColumn); + m.gpu().context().bind_to_thread()?; + assert_eq!(m.gpu().fault()?, None, "{WHAT}: stage starts clean"); + { + let gpu = m.body(WHAT)?.hybrid().tiers()[0].gpu()?; + gpu.context().bind_to_thread()?; + let x = cuda_core::DeviceBuffer::from_host(gpu.stream(), &[f32::NAN; 256])?; + let mut act = bloomery_gpu::Q8Act::with_slots(gpu.stream(), 1, 256)?; + gpu.enqueue_quantize_q8_1_layer(&x, &mut act, routed[0])?; + gpu.stream().synchronize()?; + assert_eq!(gpu.fault()?, Some(fault), "{WHAT}: real tier kernel fault"); + } + m.gpu().context().bind_to_thread()?; + assert_eq!( + m.gpu().fault()?, + None, + "{WHAT}: tier fault is not a stage injection" + ); + let failed = m.step(&[ids[0]]); + assert!( + matches!(failed, Err(GpuError::Fault { fault: got, .. }) if got == fault), + "{WHAT}: actual readback must carry the tier kernel fault, got {failed:?}" + ); + assert_eq!( + m.poisoned(), + Some(fault), + "{WHAT}: fault poisoned the model" + ); + let position = m.pos(); + assert!( + matches!(m.step(&[ids[1]]), Err(GpuError::Poisoned { fault: got, .. }) if got == fault), + "{WHAT}: poisoned model must refuse further work" + ); + assert_eq!(m.pos(), position, "{WHAT}: refusal advanced no sequence"); + m.reset()?; + assert_eq!(m.poisoned(), None, "{WHAT}: reset lifted fault poison"); + assert_eq!(m.pos(), 0, "{WHAT}: reset rewound sequence"); + m.set_mode(StepMode::Graph); + let token = prompt(&mut m, &ids[..PROMPT])?; + let mut recovered = vec![observe(&m, token)?]; + let token = m.step(&ids[PROMPT..PROMPT + 1])?; + recovered.push(observe(&m, token)?); + same( + "fault/reset/refeed versus clean union", + &reference[..2], + &recovered, + ); + assert!( + m.has_capture(), + "{WHAT}: recovered model has a real decode capture" + ); + Ok(()) +} diff --git a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs index 3d30bbeda..6e72e0d28 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs @@ -192,7 +192,8 @@ mod gate { use crate::glm5next_tier; use crate::quiet::Quiet; - use crate::tier_twocard::{self, Leg, Sent, plus, same, sent_between, sent_every, union_plan}; + use crate::tier_twocard::{self, Leg, Sent, plus, same, sent_between, sent_every}; + use model::placement::union_plan; const NAME: &str = "gate_glm5next_twocard"; /// The card budget both plans are made under (module header: the union diff --git a/crates/gpu-gates/src/bin/gate_minimaxm3_prefill.rs b/crates/gpu-gates/src/bin/gate_minimaxm3_prefill.rs index ba0461931..e2eea9a2d 100644 --- a/crates/gpu-gates/src/bin/gate_minimaxm3_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_minimaxm3_prefill.rs @@ -37,7 +37,7 @@ //! The two-card arm (`--arm bp`, `just gate-gpu-minimaxm3-twocard` under `BLOOMERY_CARD=both`): plan (b′) (the A6000 the //! stage card, the 3090 an expert tier holding each layer's next ids after the stage's prefix, the host the rest), planned //! under one card budget no environment moves (`seat::TWO_BUDGET`), in the real tier only. Before the common clauses it holds, against the reference — the plan with every tier segment on the stage card -//! (`tier_twocard::union_plan`), the A6000 alone, opened first and released: +//! (`model::placement::union_plan`), the A6000 alone, opened first and released: //! - `two_card_load`: the plan puts experts on the tier, and the load's tier opened on the 3090 beside the stage card with //! the plan's experts on each layer; //! - `two_card_first_tokens_are_union`: [`TWO_WALK`] decode steps of the walk read the reference's ids and logits rows bit @@ -490,12 +490,7 @@ mod gate { ) -> Result<(), GateError> { // The walk's positions too: the reference and the two-card load hold the same cache rows. let ctx = ctx.max(TWO_WALK + 1); - let mut two = seat::two_card( - path, - (ctx, tier_now), - (TWO_WALK, crate::e2e::walk), - crate::tier_twocard::union_plan, - )?; + let mut two = seat::two_card(path, (ctx, tier_now), (TWO_WALK, crate::e2e::walk))?; let (pass, layers) = two_card_clauses(&mut two)?; let mut loaded = two.loaded; let dims = loaded.dims(); diff --git a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs index 5137e5828..62dc44357 100644 --- a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs @@ -83,7 +83,7 @@ //! ids and logits bits, every logit finite, and the walk sent every tier layer a routed slot (or the run proves nothing //! there and fails by name); //! - `two_card_first_token_near_union`: the first token's layers against the reference — the plan with every tier -//! segment on the stage card ([`tier_twocard::union_plan`]), the A6000 alone, opened first and released — each layer +//! segment on the stage card ([`model::placement::union_plan`]), the A6000 alone, opened first and released — each layer //! teacher-forced: the step runs that layer alone (`GpuModel::set_layer_span`) on the reference's `l_out` of the layer //! before it (`GpuModel::set_input_override`; layer 0 reads the embedding), keeps the reference's picks and keeps //! `ffn_moe_out` and `l_out` within [`TWO_STEP_BAND`]; the token sent the tier a slot on one of the tier's layers. The @@ -155,9 +155,7 @@ mod gate { use crate::gate_card; use crate::lcpp_e2e::{self, Dims, Seat, fnv1a}; use crate::prefill_clauses::{self, Case}; - use crate::tier_twocard::{ - Sent, sent_between, sent_every, tier_counts, tier_on_3090, union_plan, - }; + use crate::tier_twocard::{Sent, sent_between, sent_every, tier_counts, tier_on_3090}; use app::Prompt; use bloomery_gpu::arch::musespark::{Body, LayerNodes, LayerRoute}; use bloomery_gpu::host::served::TierBody; @@ -171,6 +169,7 @@ mod gate { use model::arch::Arch; use model::arch::musespark::hparams::Hparams; use model::arch::musespark::place::{self, PlanInputs}; + use model::placement::union_plan; use model::placement::{Machine, Plan, PlanLevers, workstation}; type Model = GpuModel; diff --git a/crates/gpu-gates/src/bin/gate_qwen3moe_twocard.rs b/crates/gpu-gates/src/bin/gate_qwen3moe_twocard.rs index 5a2d8dd1b..ee6ee7139 100644 --- a/crates/gpu-gates/src/bin/gate_qwen3moe_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_qwen3moe_twocard.rs @@ -87,8 +87,9 @@ mod gate { use crate::e2e::walk; use crate::q3place::PlaceQ3; use crate::tier_twocard::{ - Leg, Sent, same, sent_between, sent_every, tier_counts, tier_on_3090, union_plan, + Leg, Sent, same, sent_between, sent_every, tier_counts, tier_on_3090, }; + use model::placement::union_plan; const NAME: &str = "gate_qwen3moe_twocard"; /// The card budget both plans are made under in the real tier (module header). diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index df267bdaf..a660da817 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -88,7 +88,8 @@ mod gate { use crate::e2e::walk; use crate::quiet::Quiet; - use crate::tier_twocard::{Leg, Sent, same, sent_between, sent_every, union_plan}; + use crate::tier_twocard::{Leg, Sent, same, sent_between, sent_every}; + use model::placement::union_plan; const NAME: &str = "gate_solar2_twocard"; /// The card budget both plans are made under in the real tier (module header: the union it leaves fits the A6000). diff --git a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs index cbf918642..ea316e6e5 100644 --- a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs +++ b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs @@ -2,7 +2,7 @@ //! `BLOOMERY_CARD=both`): plan (b′) (`workstation::plan_bp`: the A6000 the stage card, the 3090 an expert tier holding //! each routed layer's next ids after the stage's prefix, the host the rest), planned under one card budget no //! environment moves ([`TWO_BUDGET`]), against the reference — the same plan with every tier segment on the stage card -//! (`tier_twocard::union_plan`), the A6000 alone, opened first and released. Its clauses: +//! (`model::placement::union_plan`), the A6000 alone, opened first and released. Its clauses: //! - `two_card_load`: the plan puts experts on the tier, and the load's tier opened on the 3090 beside the stage card //! with the plan's experts on each layer; //! - `two_card_first_tokens_are_union`: [`TWO_WALK`] decode steps of the walk read the reference's ids and logits rows @@ -26,11 +26,9 @@ use bloomery_gpu_mimo2::{Body, Mimo2Model, T_MAX, prefill, set_prefill_group}; use gguf::Split; use model::arch::mimo2::place::PlanInputs; use model::placement::workstation; -use model::placement::{Machine, PlanLevers}; +use model::placement::{Machine, PlanLevers, union_plan}; -use crate::tier_twocard::{ - Leg, Sent, same, sent_between, sent_every, tier_counts, tier_on_3090, union_plan, -}; +use crate::tier_twocard::{Leg, Sent, same, sent_between, sent_every, tier_counts, tier_on_3090}; /// What the arm's errors name. const WHAT: &str = "gate_mimo2_e2e two-card"; diff --git a/crates/gpu-gates/src/bin/shared/minimaxm3_seat.rs b/crates/gpu-gates/src/bin/shared/minimaxm3_seat.rs index 1573268fd..df9603efe 100644 --- a/crates/gpu-gates/src/bin/shared/minimaxm3_seat.rs +++ b/crates/gpu-gates/src/bin/shared/minimaxm3_seat.rs @@ -27,7 +27,7 @@ use gguf::Split; use model::arch::minimaxm3::hparams::FfnKind; use model::arch::minimaxm3::place::PlanInputs; use model::placement::workstation::{self, HostRead, PLAN_ROOM}; -use model::placement::{Machine, Plan, PlanLevers}; +use model::placement::{Machine, PlanLevers}; use refset::arch::minimaxm3; use std::num::NonZeroUsize; use std::path::Path; @@ -235,15 +235,11 @@ pub struct TwoCard { pub opened: std::time::Instant, } -/// A plan's union on the stage card alone (the gate's `tier_twocard::union_plan`): every tier segment joined to the stage -/// card's, `flat` the machine without the tier. -pub type Union = for<'a, 'b> fn(&'b Plan<'a>, &'a Machine) -> Result, GateError>; - /// The steps' token walk: `walk(steps, vocab)` gives the ids the reference decodes. pub type Walk = fn(usize, usize) -> Vec; /// The two-card arm's loads at `ctx` positions, one after the other: plan (b′) (`workstation::plan_bp`, the stage the A6000, -/// the tier the 3090) under one card budget no environment moves ([`TWO_BUDGET`]); the reference, that plan's `union` on the A6000 alone, its ids and logits rows read over the walk +/// the tier the 3090) under one card budget no environment moves ([`TWO_BUDGET`]); the reference, that plan's union on the A6000 alone, its ids and logits rows read over the walk /// `walk(steps, vocab)` and released; then plan (b′) itself. A budget the environment sets, a union past the A6000's usable /// bytes, a reference opened elsewhere or holding a tier, and the fixture tier (its cut's routed layers fit the stage card, /// so the tier would hold nothing) are refused by name. @@ -251,7 +247,6 @@ pub fn two_card( file: &Path, (ctx, tier): (usize, Tier), (steps, walk): (usize, Walk), - union: Union, ) -> Result { if tier == Tier::Fixture { return Err("the two-card arm runs in the real tier: the fixture cut's routed layers fit the stage card".into()); @@ -302,7 +297,7 @@ pub fn two_card( } let host = levers.host(); let reference = { - let uplan = union(&plan, &flat)?; + let uplan = model::placement::union_plan(&plan, &flat)?; let mut r = Body::open_placed(minimaxm3_folder::model_at(file)?, &uplan, &inputs, 0, host)?; r.set_mode(StepMode::Graph); if !r.body("minimaxm3 seat")?.hybrid().tiers().is_empty() { diff --git a/crates/gpu-gates/src/bin/shared/tier_twocard.rs b/crates/gpu-gates/src/bin/shared/tier_twocard.rs index a70c17b5c..7eb661e39 100644 --- a/crates/gpu-gates/src/bin/shared/tier_twocard.rs +++ b/crates/gpu-gates/src/bin/shared/tier_twocard.rs @@ -1,7 +1,7 @@ //! What the two-card tier gates share (`gate_glm5next_twocard`, `gate_solar2_twocard`): plan (b′) against the same //! plan with every tier segment on the stage card, the legs' reads, and the per-layer counters of what the tier was sent. //! The reference of a two-card run is the stage card opened alone over the union of its experts and the tier's -//! ([`union_plan`]): the tier's slots computed by the stage card's own launches, so the two-card run must read the same +//! ([`model::placement::union_plan`]): the tier's slots computed by the stage card's own launches, so the two-card run must read the same //! tokens and logits rows bit for bit ([`same`]). A gate includes it by `#[path]`, with `#[allow(dead_code)]` for the //! helpers it does not read. @@ -12,68 +12,6 @@ use bloomery_gpu::host::run::HostRun; use bloomery_gpu::hybrid::Hybrid; use bloomery_gpu_gates::GateError; use model::placement::workstation; -use model::placement::{self, Device, ExpertList, Machine, Plan, Role, Row}; - -/// `plan` with each routed stack's tier segment joined to its stage-card -/// segment on card 0 (`placement::routed_row` over the two lists), the -/// host's segments as they are, on `flat` — the plan's machine with no -/// tier card: the stage card holds the union, and no segment sits on a -/// tier. -pub fn union_plan<'a>(plan: &Plan<'a>, flat: &'a Machine) -> Result, GateError> { - let model = plan.model; - let tier = Device::Card(plan.machine.cards.len()); - let mut out = plan.clone(); - for row in &mut out.rows { - let t = &model.tensors[row.tensor]; - if t.role != Role::RoutedExperts { - continue; - } - let ids = |d: Device| -> Vec { - row.segments - .iter() - .filter(|s| s.device == d) - .filter_map(|s| s.experts.as_ref()) - .flat_map(|e| e.ids().iter().copied()) - .collect() - }; - let on_tier = ids(tier); - if on_tier.is_empty() { - continue; - } - let mut union = ids(Device::Card(0)); - union.extend(on_tier); - let n = union.len(); - let card = placement::routed_row( - row.tensor, - t, - 0, - ExpertList::new(union, model.experts)?, - model, - )?; - let segments = card - .segments - .into_iter() - .filter(|s| s.device == Device::Card(0)) - .chain( - row.segments - .iter() - .filter(|s| !matches!(s.device, Device::Card(_))) - .cloned(), - ) - .collect(); - *row = Row { - segments, - ..row.clone() - }; - if let Some(l) = t.layer.and_then(|l| out.n_l.get_mut(l)) { - *l = u64::try_from(n)?; - } - } - out.machine = flat; - out.cards.truncate(flat.cards.len()); - out.tier_n_l.clear(); - Ok(out) -} /// Every token, kept count and logits row a leg read, in order. #[derive(Default)] diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 96e625d63..6d66ea1f8 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -1208,6 +1208,139 @@ pub struct Plan<'a> { pub budget_from_pool: bool, } +/// The one-stage reference of a one-expert-tier plan: the stage holds the union +/// of its routed ids and the tier's, and the host segments are unchanged. +/// `flat` is the same stage machine with the tier removed. The reference uses +/// the stage's actual capacity, not the artificial budget that split the sets. +pub fn union_plan<'a>(plan: &Plan<'a>, flat: &'a Machine) -> Result, PlacementError> { + if plan.machine.cards.len() != 1 + || plan.machine.expert_tiers().count() != 1 + || flat.cards.len() != 1 + || !flat.tiers.is_empty() + || !plan.machine.cards[0].same_device(&flat.cards[0]) + { + return Err(PlacementError::Stages( + "union reference needs one stage, one expert tier and the same stage without tiers" + .into(), + )); + } + let model = plan.model; + let tier = Device::Card(plan.machine.cards.len()); + let mut out = plan.clone(); + for row in &mut out.rows { + let t = &model.tensors[row.tensor]; + if t.role != Role::RoutedExperts { + continue; + } + let ids = |d: Device| -> Vec { + row.segments + .iter() + .filter(|s| s.device == d) + .filter_map(|s| s.experts.as_ref()) + .flat_map(|e| e.ids().iter().copied()) + .collect() + }; + let on_tier = ids(tier); + if on_tier.is_empty() { + continue; + } + let mut union = ids(Device::Card(0)); + union.extend(on_tier); + let n = union.len() as u64; + let mut format = None; + for segment in row + .segments + .iter() + .filter(|s| s.device == Device::Card(0) || s.device == tier) + { + let Format::Card(selected) = segment.format else { + return Err(PlacementError::tensor( + t, + "union reference needs resident card expert segments", + )); + }; + if format.is_some_and(|f| f != selected) { + return Err(PlacementError::tensor( + t, + "union reference cannot join different selected card formats", + )); + } + format = Some(selected); + } + let card = routed_row_on_as( + row.tensor, + t, + 0, + vec![(0, ExpertList::new(union, model.experts_of(t))?)], + model, + format, + )?; + let segments = card + .segments + .into_iter() + .filter(|s| s.device == Device::Card(0)) + .chain( + row.segments + .iter() + .filter(|s| !matches!(s.device, Device::Card(_))) + .cloned(), + ) + .collect(); + *row = Row { + segments, + ..row.clone() + }; + if let Some(l) = t.layer.and_then(|l| out.n_l.get_mut(l)) { + *l = n; + } + } + out.machine = flat; + out.cards.truncate(1); + out.tier_n_l.clear(); + out.card_budget = card_budget::CardBudget::default(); + out.budget_from_pool = false; + let (heaps, _) = card_heaps(model, flat, &out.rows); + let card = &flat.cards[0]; + let total = &mut out.cards[0]; + total.expert_bytes = out + .rows + .iter() + .flat_map(|r| { + r.segments.iter().filter_map(|s| { + (model.tensors[r.tensor].role == Role::RoutedExperts && s.device == Device::Card(0)) + .then_some(s.resident_bytes) + }) + }) + .sum(); + total.experts = out.n_l.iter().sum(); + total.rounding_bytes = heaps[0].taken - total.dense_bytes - total.expert_bytes; + total.reserve_bytes = card.selected_reserve_bytes(model, &out.n_l)?; + let used = card + .floor_with_reserves( + heaps[0].taken, + total.kv_bytes, + total.stage_rows_bytes, + total.reserve_bytes, + ) + .ok_or_else(|| card.bytes_past_u64())?; + total.headroom_bytes = i128::from(capped(card, None)) - i128::from(used); + let limit = capped(card, None).saturating_sub(card.margin_bytes); + if used > limit { + return Err(PlacementError::Metadata { + key: "union reference".into(), + detail: Violation::CardOver { + card: card.name.clone(), + total: used, + limit, + } + .to_string(), + }); + } + out.host.headroom_bytes = workstation::HostNeed::of_totals(flat.host(), &out.host, false, 0) + .headroom(workstation::HostNeed::machine_room(flat.host())); + Ok(out) +} + /// Why a placement could not be built; each variant names what it refused. #[derive(Debug, thiserror::Error)] pub enum PlacementError { @@ -6376,6 +6509,148 @@ mod tests { assert_eq!(plan.n_l, vec![8]); } + #[test] + fn a_union_recounts_uploads_selected_forms_and_capacity() { + use crate::kernels::{DownAct, DownForms}; + use workstation::{TIER_BATCH_RESERVE, tier_leg_batch_bytes}; + + let mut model = legged(&[ + [GgmlType::IQ3_S, GgmlType::IQ3_S, GgmlType::Q5_K], + [GgmlType::IQ4_XS, GgmlType::IQ4_XS, GgmlType::Q8_0], + ]); + model.experts_used = 6; + let forms = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + let batch = tier_leg_batch_bytes(256, 256, 6, 2, 3, forms, false).unwrap(); + let mut stage = bytes_card("stage", 1 << 30, 0..2); + stage.tier_leg = batch.leg; + stage + .reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + let flat = Machine { + cards: vec![stage], + tiers: Vec::new(), + nodes: vec![Node { + host: host(), + unified: None, + }], + }; + let machine = Machine { + tiers: vec![bytes_card("tier", 1 << 30, 0..0)], + ..flat.clone() + }; + let rule = crate::accepts::routed_rule(crate::accepts::Program::FlintBody); + let (index, iq3) = model + .tensors + .iter() + .enumerate() + .find(|(_, t)| t.role == Role::RoutedExperts && t.ty == GgmlType::IQ3_S) + .unwrap(); + assert!(matches!( + routed_row( + index, + iq3, + 0, + ExpertList::new(vec![0], model.experts).unwrap(), + &model + ), + Err(PlacementError::NoCardFormat { .. }) + )); + let mut rows = plan_routed(&model, &flat, 1, &NoKv, &PlanLevers::default(), rule) + .unwrap() + .rows; + for row in &mut rows { + let tensor = &model.tensors[row.tensor]; + if tensor.role == Role::RoutedExperts { + let device = usize::from(tensor.layer == Some(1)); + *row = routed_row_on_as( + row.tensor, + tensor, + 0, + vec![(device, ExpertList::new(vec![0], model.experts).unwrap())], + &model, + rule.format(tensor).unwrap(), + ) + .unwrap(); + } + } + let plan = totals( + &model, + &machine, + 1, + rows, + &[(0, 0, 0); 2], + (vec![1, 0], vec![vec![0, 1]]), + card_budget::CardBudget::default(), + ) + .unwrap(); + let joined = union_plan(&plan, &flat).unwrap(); + assert_eq!(joined.n_l, [1, 1]); + assert!(joined.tier_n_l.is_empty()); + assert_eq!( + joined.cards[0].expert_bytes, + plan.cards[0].expert_bytes + plan.cards[1].expert_bytes + ); + assert_eq!(joined.cards[0].experts, 2); + let heap = card_heaps(&model, &flat, &joined.rows).0.remove(0); + assert_eq!( + joined.cards[0].rounding_bytes, + heap.taken - joined.cards[0].dense_bytes - joined.cards[0].expert_bytes + ); + let reserve = batch.staging + batch.leg.unwrap().bytes(forms).unwrap(); + assert_eq!(joined.cards[0].reserve_bytes, reserve); + assert_eq!( + joined.card_reserve(0, TIER_BATCH_RESERVE).unwrap(), + Some(reserve) + ); + assert!(reserve > plan.cards[0].reserve_bytes); + assert_eq!(joined.cards[0].kv_bytes, plan.cards[0].kv_bytes); + for (before, after) in plan.rows.iter().zip(&joined.rows) { + let host = |row: &Row| { + row.segments + .iter() + .filter(|s| !matches!(s.device, Device::Card(_))) + .map(|s| { + ( + s.device, + s.resident_bytes, + s.experts.as_ref().map(|e| e.ids().to_vec()), + ) + }) + .collect::>() + }; + assert_eq!(host(before), host(after)); + } + let mut conflicting = plan.clone(); + let row = conflicting + .rows + .iter_mut() + .find(|r| r.segments.iter().any(|s| s.device == Device::Card(1))) + .unwrap(); + let mut other = row + .segments + .iter() + .find(|s| s.device == Device::Card(1)) + .unwrap() + .clone(); + other.device = Device::Card(0); + other.format = Format::Card(CardFormat::F32); + row.segments.push(other); + assert!(matches!( + union_plan(&conflicting, &flat), + Err(PlacementError::Tensor { .. }) + )); + let mut short = flat.clone(); + short.cards[0].usable_bytes = joined.cards[0].bytes() - 1; + assert!(matches!(union_plan(&plan, &short), + Err(PlacementError::Metadata { key, .. }) if key == "union reference")); + let mut wrong = flat.clone(); + wrong.cards.clear(); + assert!(matches!( + union_plan(&plan, &wrong), + Err(PlacementError::Stages(_)) + )); + } + /// Scratch enters each expert candidate's footprint monotonically. /// A boundary that admits one down form cannot pay for an unused second /// form, and adding the first expert of that form pays its exact bytes. diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 941b4c78e..7666cd1c4 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -652,6 +652,116 @@ Keep the integration Draft. Korean numerical/oracle acceptance, cc12.1 resource affected common-owner coverage beyond the exercised consumers, two-card operation, dynamic residency/NVMe, streamed HTTP and spontaneous tool selection are not established by this run. +## 11. Current-main integration and trajectory instruments + +The linear integration candidate starts at main +`43073091be21e48cda40d79fe6dae1dfcfd643d5`; it preserves the published behavioral +and resource evidence in section 10 as historical evidence, not acceptance of a new head. +The placement integration keeps both the cached upload `Footprint` and candidate-selected +tier scratch/reserves. No compiler pin, numerical band or model admission was widened. + +### The common union must retain selected formats and recount its allocation + +`model::placement::union_plan` now owns the shared one-stage/one-tier reference transform. +It retains the actual selected card format, refuses incompatible formats, preserves host +assignments and KV, and recomputes expert bytes, upload-heap rounding, selected reserves and +actual-stage headroom. The artificial budget that split the experts is not the reference's +physical capacity. The old shared helper is removed and its consumers use the common owner. +The standalone DS41 and Qwen3.8 transforms have different contracts and remain separate. + +A public host-only consumer over the complete pinned GGUF reproduced the former generic +transform's `NoCardFormat` refusal on `blk.2.ffn_gate_exps.weight` (`IQ3_S`). The corrected +transform preserves the program-selected format rather than admitting IQ3_S generically. +With the workstation's 3090 stage/A6000 tier figures, 1024 context and an 8 GiB per-card +planning budget, its union contains 5,197 experts and 9,959,514,112 expert bytes, equal to +the assigned rows, with no plan violations. This is real-file planning, not device loading. +The pure regression also covers mixed selected down forms, heap/reserve recounting, +unchanged generic refusal, conflicting formats and insufficient union capacity. + +### CPU and GPU use one forced-ID diagnostic artifact layout + +`tools/ref/flint_trace.cpp` and the `bloomery-gpu-flint` example `trace_flint` consume exactly +96 whitespace-separated decimal IDs: one 22-ID batched prefill, then 74 one-ID calls. +They do not retokenize, sample, or stop at an input EOG. Each observation saves the latest +processed row: prompt position 21, then positions 22 through 95, predicting positions +22 through 96. Earlier prompt rows are not claimed as captured history. + +The diagnostic format is `flint-trajectory-v1`, not a reference-set family or accuracy gate: + +- `tokens.ids` and `steps.tsv`: exact inputs and each call's processed/predicted position + and raw argmax. +- `logits.f32`: little-endian F32 `[75][150144]` in every arm. +- With taps: `layers.f32` `[75][49][2048]` and `norm.f32` `[75][2048]`. + Untapped arms explicitly mark those tensors absent. +- CPU `tensors.tsv` records actual node types, dimensions, strides and selected rows. + GPU artifacts retain the actual expert assignments, loaded slots, device/plan facts, + graph nodes and completion counters. +- A new output directory is required. `MANIFEST.tsv` is published only after successful + capture; a partial run cannot masquerade as complete. External receipts identify the + model, source/diagnostic overlay, toolchain, libraries and binary hashes. + +Build the CPU consumer against the preserved mainline revision +`a37cc46104eb50900d5ab4a9ba02a0abdac039db`, with shell variables naming its source, +built library directory and the new diagnostic executable: + +```sh +c++ -std=c++17 -O2 -I"$LLAMA_SRC/include" -I"$LLAMA_SRC/ggml/include" \ + tools/ref/flint_trace.cpp -L"$LLAMA_LIB" -Wl,-rpath,"$LLAMA_LIB" \ + -lllama -lggml -lggml-cpu -lggml-base -o "$CPU_TRACE" +CUDA_VISIBLE_DEVICES= "$CPU_TRACE" -m "$MODEL" --ids "$IDS" \ + --prompt 22 --ctx 1024 --out "$CPU_OUT" +``` + +The CPU callback arm was actually run on the complete model: 75 calls, 3,675 layer vectors, +zero preceding argmax mismatches, and final argmax 26007. Its complete final-logit SHA-256 +is `083c8aa95aab025c0183c2ee3c4788642499533d5a2ee4e280c1125302c23bf9`, bit-identical +to the preserved callback-free original row. The model was rehashed in this round. + +The GPU build uses the pinned oxide toolchain. Actual capture runs belong inside an +authorized runner configuration with its normal card locks and bounds; `TRACE` names +the compiled example in that checkout's target directory: + +```sh +cargo oxide build --arch sm_86 -- -p bloomery-gpu-flint --release --example trace_flint +bash tools/gpu-gate.sh --executable "$TRACE" -m "$MODEL" --ids "$IDS" \ + --prompt 22 --ctx 1024 --slots 2 --mode graph --out "$GPU_OUT" +``` + +Use separate new output directories for Graph/untapped, Eager/untapped and Eager/tapped +(`--taps`) arms. Arming taps forces Eager; the consumer explicitly restores the requested +mode afterwards, and Graph decode must actually capture. The instrument admits only +the existing card-budget lever; do not assume other inherited levers were applied. +Context 1024 is per slot, two slots total 2048, matching the original HTTP setup. +Compare the actual per-layer placement, not just model identity or aggregate counts. + +Diagnose the original `313b9f0` production source with the separately recorded example +overlay first; then bridge the same controls onto the integration candidate. A new binary +on a larger card is not automatically the original 3090 trajectory. A control mismatch +stops causal attribution. If layer outputs agree but the final norm/head does not, isolate +that operation on matching inputs; intended quantization is neither a demonstrated bug +nor automatic permission to waive the Korean flip. + +### Dedicated real two-card consumer + +`gate-gpu-flint-twocard` runs +`two_card_tests::hw_two_card_full_file_union_capture_publication_fault_reset`. +It requires the complete pinned GGUF in inner `BLOOMERY_MODEL`, two distinct visible +devices, and actual expert assignments to both cards at every routed layer. It compares +batched prompt and changed-input decode logits/layer rows with the same plan's one-card +union, checks ranked six-slot publication and stage/tier capture/completion, then raises +a real quantizer fault on the tier GPU while the stage is clean and checks propagation, +poison, refusal, reset and refeed. No unsafe fault-injection API is added. + +The one-card library recipe keeps its exact original test filter. Both full-file recipes +refuse the fixture tier. The two-card recipe takes the existing `both`/big-load runner +path and has a derived host-resource ceiling, not a measured RSS claim. It does not prove +dynamic residency/NVMe, loss/deadline recovery or a Korean oracle result. + +The new GPU diagnostic and two-card case require fresh NVIDIA execution. Hikari's existing +work was explicitly left undisturbed; the new window waits for its owner. No cc12.1 values +were inferred and the remaining resource/oracle acceptance stays open. + + [needs]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/models/src/need.rs#L505-L531 [stems]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/model/src/arch/stems.rs#L61-L125 diff --git a/justfile b/justfile index 48414e733..ada93be8e 100644 --- a/justfile +++ b/justfile @@ -1770,7 +1770,13 @@ gate-gpu-flint-router: # Full-file eager/captured, resident-slot and ring-retention consumer transitions. gate-gpu-flint-lib: - ./tools/box.sh 'bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-flint --lib -- --include-ignored' + ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-flint-lib && bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-flint --lib -- --include-ignored --exact tests::hw_full_file_capture_slots_reset_and_ring_boundary' + +# Real two-card top-six prompt/decode, union reference, capture and tier fault recovery. +[group('solo')] +[group('v41-load')] +gate-gpu-flint-twocard: + BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-flint-twocard && export BLOOMERY_GATE_V41_LOAD=1 && bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-flint --lib -- --include-ignored --exact two_card_tests::hw_two_card_full_file_union_capture_publication_fault_reset' # Muse Spark's 256-expert top-8 sigmoid router (`sigmoid_256_8`). Model-free like the Flint one, # plus the bias clause: the selection follows the biased scores, the weights the unbiased ones. diff --git a/tools/gate-batch-resources.tsv b/tools/gate-batch-resources.tsv index e4fdb0f90..de831a023 100644 --- a/tools/gate-batch-resources.tsv +++ b/tools/gate-batch-resources.tsv @@ -36,6 +36,11 @@ # so the e2e's 187e9 bounds its host bytes [derived]. # qwen3moe-twocard: unmeasured ceiling, the qwen38-twocard row's figure (qwen4exp --place gate need, 92,054,154,240 B). # The two loads run one after the other. Not a measurement of the Qwen3-30B or Qwen3.6 plan. +# flint-twocard: [derived] pinned GGUF, 1024 context, one sequence, residency/R8 off: +# HostNeed with every routed file tensor on the host is 24,168,079,360 B; +# four histories of 75 * (49 * 2048 + 150144) f32 bound retained readbacks and temporaries, +# 300,595,200 B. Sequential model loads: total ceiling 24,468,674,560 B, not measured RSS. +gate-gpu-flint-twocard - 24468674560 gate-gpu-ds41-callstream - 237600000000 gate-gpu-ds41-long m 237600000000 gate-gpu-ds41-twocard - 237600000000 diff --git a/tools/gate-class.tsv b/tools/gate-class.tsv index 6bf0f536b..9c45e4087 100644 --- a/tools/gate-class.tsv +++ b/tools/gate-class.tsv @@ -76,6 +76,7 @@ gate-gpu-e2e family deepseek2 e2e graph_fault+moe_route+weights_load gate-gpu-flint-attn family flint kernel attn_gqa gate-gpu-flint-router family flint kernel moe_route gate-gpu-flint-lib family flint e2e graph_fault+attn_gqa+moe_route+tier_legs+placement_sizing +gate-gpu-flint-twocard family flint tier tier_legs+node_link+placement_sizing+prompt_walks+graph_fault gate-gpu-muse-router common kernel moe_route gate-gpu-gates-lib tooling harness - gate-gpu-gemm common kernel quant_gemm diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 9462bb612..9a5ad942d 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -138,6 +138,7 @@ crates/gpu/src/arch/qwen3moe/tier_leg.rs derive the Qwen3 and Qwen3.6 expert-tie crates/gpu/src/site.rs derive gate-ds41-bind gate-gpu-gemm gate-gpu-nvtier gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp the weight sites' launches by type: read by the Qwen bodies and the Clef hidden gate, the bodies that read a site in more than one type crates/gpu/src/arch/deepseek2/** derive gate-ds41-bind the V2-Lite body: the V2-Lite GPU gates and AnyEngine (gate_e2e) crates/gpu/src/arch/musespark/** derive the muse-spark card body: the end-to-end gate names it, and every bin that links bloomery-gpu reaches it through the lib +crates/gpu-flint/** derive Flint's head128 body, resident slots and actual single/two-card consumers; the two-card gate retains its independent runtime filter # Derived rows: each row's recipes are `python3 tools/recipes.py reach --row ` at the time of writing, the # gates whose targets can run code of the files through another lib (tools/recipes.py, Reach); files grouped only diff --git a/tools/levers-direct.txt b/tools/levers-direct.txt index 2c85c03aa..98aec4038 100644 --- a/tools/levers-direct.txt +++ b/tools/levers-direct.txt @@ -63,6 +63,7 @@ crates/model/tests/ds41_meta.rs BLOOMERY_DSPARK_MODEL stays: the DSpark draft fi crates/model-hf/tests/fixture.rs BLOOMERY_DSPARK_MODEL stays: the DSpark draft file crates/model/tests/flint_meta.rs BLOOMERY_MODEL stays: the pinned model file or header prefix the harness opens crates/gpu-flint/src/tests.rs BLOOMERY_MODEL stays: cfg(test) full-file hardware consumer opens this exact local model path, not a production serving lever +crates/gpu-flint/src/two_card_tests.rs BLOOMERY_MODEL stays: cfg(test) two-card full-file input, not a production serving lever crates/model/tests/gemma_embedding_meta.rs BLOOMERY_MODEL stays: the model file the harness opens crates/model/tests/gemma_embedding2_meta.rs BLOOMERY_MODEL stays: the model file the harness opens crates/model/tests/mt.rs BLOOMERY_MT_CHILD_DUMP stays: the gate's handshake with the child it runs diff --git a/tools/ref/flint_trace.cpp b/tools/ref/flint_trace.cpp new file mode 100644 index 000000000..db66e5e93 --- /dev/null +++ b/tools/ref/flint_trace.cpp @@ -0,0 +1,355 @@ +// Flint forced-ID diagnostic trajectory, not a reference-set family or accuracy gate. +// Only the last input requests logits: the prompt's last layer is therefore already cropped. +// Node taps change scheduler segmentation; --no-taps is the same-engine logits-only control. +// Model SHA256 and binary/source identity belong in the external execution receipt. +#include "llama.h" +#include "ggml.h" +#include "ggml-backend.h" +#include "ref_io.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void require(bool ok, const std::string & message) { + if (!ok) throw std::runtime_error(message); +} + +static std::ofstream output(const fs::path & path, bool binary = false) { + std::ofstream stream; + stream.exceptions(std::ios::badbit | std::ios::failbit); + stream.open(path, std::ios::out | (binary ? std::ios::binary : std::ios::openmode(0))); + return stream; +} + +static void write_row(std::ofstream & out, const float * row, size_t width) { + out.write(reinterpret_cast(row), static_cast(width * sizeof(float))); +} + +static std::string metadata(llama_model * model, const char * key) { + char value[256]; + const int n = llama_model_meta_val_str(model, key, value, sizeof(value)); + require(n >= 0 && static_cast(n) < sizeof(value), std::string("missing/oversized model metadata: ") + key); + return value; +} + +static std::string actual_path(const std::string & path) { + const std::string result = fs::canonical(path).string(); + require(result.find_first_of("\t\r\n") == std::string::npos, "path contains a TSV separator: " + path); + return result; +} + +struct backend_guard { + backend_guard() { llama_backend_init(); } + ~backend_guard() { llama_backend_free(); } +}; + +struct batch_guard { + llama_batch batch = llama_batch_init(32, 0, 1); + ~batch_guard() { llama_batch_free(batch); } +}; + +struct tap_state { + int width; + int layers; + int call = -1; + int call_tokens = 0; + bool active = false; + std::vector names; + std::vector seen; + std::vector rows; + std::vector staging; + std::ofstream * tensors; + std::string error; + + tap_state(int width_, int layers_, std::ofstream * tensors_) : + width(width_), layers(layers_), seen(layers_ + 1), rows(static_cast(layers_ + 1) * width_), + tensors(tensors_) { + for (int i = 0; i < layers; ++i) names.push_back("l_out-" + std::to_string(i)); + names.push_back("result_norm"); + staging.reserve(static_cast(width) * sizeof(float)); + } + + int selected(const char * name) const { + const auto it = std::find(names.begin(), names.end(), name); + return it == names.end() ? -1 : static_cast(it - names.begin()); + } + + void begin(int call_, int tokens_) { + call = call_; + call_tokens = tokens_; + active = true; + std::fill(seen.begin(), seen.end(), 0); + error.clear(); + } + + void capture(const ggml_tensor * t, int index) { + const std::string label = "call " + std::to_string(call) + " tensor " + t->name; + require(!seen[index], label + ": duplicate capture"); + const int expected_rows = index < layers - 1 ? call_tokens : 1; + require(t->ne[0] == width && t->ne[1] == expected_rows && t->ne[2] == 1 && t->ne[3] == 1, + label + ": invalid shape (expected [" + std::to_string(width) + "," + + std::to_string(expected_rows) + ",1,1])"); + size_t element_bytes; + switch (t->type) { + case GGML_TYPE_F32: element_bytes = sizeof(float); break; + case GGML_TYPE_F16: element_bytes = sizeof(ggml_fp16_t); break; + case GGML_TYPE_BF16: element_bytes = sizeof(ggml_bf16_t); break; + default: throw std::runtime_error(label + ": unsupported type " + ggml_type_name(t->type)); + } + require(t->buffer && t->data, label + ": unallocated tensor"); + require(t->nb[0] >= element_bytes, label + ": invalid element stride"); + const size_t selected_row = static_cast(t->ne[1] - 1); + const size_t bytes = ggml_nbytes(t); + require(selected_row == 0 || t->nb[1] <= bytes / selected_row, label + ": invalid row stride"); + const size_t offset = selected_row * t->nb[1]; + require(offset <= bytes && element_bytes <= bytes - offset, label + ": row outside tensor"); + require(static_cast(width - 1) <= (bytes - offset - element_bytes) / t->nb[0], + label + ": element strides outside tensor"); + const size_t span = static_cast(width - 1) * t->nb[0] + element_bytes; + staging.resize(span); + // Copy while the scheduler still owns this live output, including view strides. + ggml_backend_tensor_get(t, staging.data(), offset, span); + float * dest = rows.data() + static_cast(index) * width; + for (int i = 0; i < width; ++i) { + const uint8_t * src = staging.data() + static_cast(i) * t->nb[0]; + float value; + if (t->type == GGML_TYPE_F32) { + std::memcpy(&value, src, sizeof(value)); + } else if (t->type == GGML_TYPE_F16) { + ggml_fp16_t half; + std::memcpy(&half, src, sizeof(half)); + value = ggml_fp16_to_fp32(half); + } else { + ggml_bf16_t half; + std::memcpy(&half, src, sizeof(half)); + value = ggml_bf16_to_fp32(half); + } + if (!std::isfinite(value)) { + throw std::runtime_error(label + ": nonfinite element " + std::to_string(i)); + } + dest[i] = value; + } + *tensors << call << '\t' << t->name << '\t' << ggml_type_name(t->type); + for (int i = 0; i < 4; ++i) *tensors << '\t' << t->ne[i]; + for (int i = 0; i < 4; ++i) *tensors << '\t' << t->nb[i]; + *tensors << '\t' << bytes << '\t' << selected_row << '\t' << ggml_op_name(t->op) + << '\t' << (ggml_is_contiguous(t) ? 1 : 0) << '\n'; + seen[index] = 1; + } + + void finish() { + active = false; + require(error.empty(), error); + for (int i = 0; i <= layers; ++i) { + require(seen[i], "call " + std::to_string(call) + " tensor " + names[i] + ": missing capture"); + } + } +}; + +static bool on_node(ggml_tensor * tensor, bool ask, void * user) { + auto & state = *static_cast(user); + if (!state.active) return !ask; + const int index = state.selected(tensor->name); + if (ask) return index >= 0; + // Never unwind a C callback or cancel a partially computed graph as if it succeeded. + if (index >= 0 && state.error.empty()) { + try { + state.capture(tensor, index); + } catch (const std::exception & e) { + state.error = e.what(); + } + } + return true; +} + +int main(int argc, char ** argv) { + try { + const char * usage = "usage: flint_trace -m MODEL --ids FILE --prompt 22 --out NEWDIR --ctx 1024 [--no-taps]\n"; + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "--help") == 0 || std::strcmp(argv[i], "-h") == 0) { + std::fputs(usage, stdout); + std::fputs("Read exactly 96 whitespace-separated decimal token IDs; never retokenize or stop on EOG.\n" + "Default: CPU layer/norm node taps and full logits. --no-taps: full logits only.\n" + "NEWDIR must not exist; MANIFEST.tsv is installed only after complete capture.\n", stdout); + return 0; + } + } + std::string model_arg, ids_arg, out_arg; + int prompt = 22, capacity = 1024; + bool taps = true; + for (int i = 1; i < argc; ++i) { + const bool more = i + 1 < argc; + if (std::strcmp(argv[i], "-m") == 0 && more) model_arg = argv[++i]; + else if (std::strcmp(argv[i], "--ids") == 0 && more) ids_arg = argv[++i]; + else if (std::strcmp(argv[i], "--out") == 0 && more) out_arg = argv[++i]; + else if (std::strcmp(argv[i], "--prompt") == 0 && more) { + if (!parse_i32("flint_trace", "--prompt", argv[++i], 1, prompt)) return 2; + } else if (std::strcmp(argv[i], "--ctx") == 0 && more) { + if (!parse_i32("flint_trace", "--ctx", argv[++i], 1, capacity)) return 2; + } else if (std::strcmp(argv[i], "--no-taps") == 0) taps = false; + else throw std::runtime_error(std::string("unknown/incomplete argument: ") + argv[i]); + } + require(!model_arg.empty() && !ids_arg.empty() && !out_arg.empty(), usage); + require(prompt == 22 && capacity == 1024, "flint-trajectory-v1 requires --prompt 22 --ctx 1024"); + const uint32_t endian = 1; + require(*reinterpret_cast(&endian) == 1 && sizeof(float) == 4 && + std::numeric_limits::is_iec559, "requires little-endian IEEE754 f32 host"); + const std::string model_path = actual_path(model_arg); + const std::string ids_path = actual_path(ids_arg); + require(out_arg.find_first_of("\t\r\n") == std::string::npos, "output path contains a TSV separator"); + struct stat model_stat {}; + require(stat(model_path.c_str(), &model_stat) == 0 && S_ISREG(model_stat.st_mode) && + model_stat.st_size == 25054688704LL, "model file is not the complete Flint UD-Q4_K_XL release size"); + + std::vector ids; + ids.reserve(96); + std::ifstream source(ids_path); + require(bool(source), "cannot read IDs"); + std::string record; + while (source >> record) { + int32_t id; + require(!record.empty() && record[0] >= '0' && record[0] <= '9' && parse_token_id(record, id), + "invalid decimal token ID at position " + std::to_string(ids.size()) + ": " + record); + require(ids.size() < 96, "ID file must contain exactly 96 IDs: extra token at position 96"); + ids.push_back(id); + } + require(source.eof() && !source.bad(), "cannot finish reading IDs"); + require(ids.size() == 96, "ID file must contain exactly 96 IDs, found " + std::to_string(ids.size())); + + backend_guard backend; + auto mp = llama_model_default_params(); + mp.n_gpu_layers = 0; + mp.load_mode = LLAMA_LOAD_MODE_MMAP; + mp.use_extra_bufts = false; + std::unique_ptr model( + llama_model_load_from_file(model_path.c_str(), mp), llama_model_free); + require(bool(model), "cannot load model"); + const int width = llama_model_n_embd(model.get()); + const int layers = llama_model_n_layer(model.get()); + const int vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get())); + const std::string arch = metadata(model.get(), "general.architecture"); + const std::string model_name = metadata(model.get(), "general.name"); + require(arch == "flint" && model_name == "Flint Preview" && width == 2048 && layers == 49 && vocab == 150144 && + metadata(model.get(), "general.file_type") == "15", "model metadata is not the Flint Preview UD-Q4_K_XL release"); + for (size_t i = 0; i < ids.size(); ++i) { + require(ids[i] >= 0 && ids[i] < vocab, "token outside model vocabulary at position " + std::to_string(i)); + } + const int calls = 1 + static_cast(ids.size()) - prompt; + require(fs::create_directory(out_arg), "output directory must be new: " + out_arg); + const fs::path out = fs::canonical(out_arg); + auto manifest = output(out / "MANIFEST.tsv.partial"); + auto steps = output(out / "steps.tsv"); + auto tokens = output(out / "tokens.ids"); + auto logits_out = output(out / "logits.f32", true); + auto tensors = output(out / "tensors.tsv"); + std::ofstream layers_out, norm_out; + if (taps) { + layers_out = output(out / "layers.f32", true); + norm_out = output(out / "norm.f32", true); + } + manifest << "protocol\tflint-trajectory-v1\nmodel\t" << model_path << "\narch\t" << arch + << "\nmodel_name\t" << model_name << "\nmodel_bytes\t" << model_stat.st_size + << "\nmodel_identity_validation\tmetadata-and-file-size;SHA256-in-external-receipt\n" + << "mode\tcpu\ninstrumentation\t" << (taps ? "node-taps" : "no-taps") + << "\nids_source\t" << ids_path << "\nwidth\t" << width << "\nlayers\t" << layers + << "\nvocab\t" << vocab << "\nprompt\t" << prompt << "\ntotal_ids\t" << ids.size() + << "\ncalls\t" << calls << "\nlogits\tlogits.f32\nlayers_file\t" << (taps ? "layers.f32" : "absent:not-requested") + << "\nnorm_file\t" << (taps ? "norm.f32" : "absent:not-requested") + << "\nlogits_layout\t[call][vocab];little-endian-f32\nlayers_layout\t[call][layer][width];little-endian-f32" + << "\nnorm_layout\t[call][width];little-endian-f32\ntensor_metadata\ttensors.tsv" + << "\nngpu\t0\nload_mode\tmmap\nuse_extra_bufts\tfalse\nthreads\t4\nthreads_batch\t4" + << "\noffload_kqv\tfalse\nop_offload\tfalse\nswa_full\tfalse\nkv_unified\tfalse" + << "\ntype_k\tf16\ntype_v\tf16\nflash_attn\tauto\nno_perf\ttrue\n"; + manifest.flush(); + tensors << "call\tname\ttype\tne0\tne1\tne2\tne3\tnb0\tnb1\tnb2\tnb3\tbytes\tselected_row\top\tcontiguous\n"; + tap_state state(width, layers, &tensors); + auto cp = llama_context_default_params(); + cp.n_ctx = capacity; + cp.n_seq_max = 1; + cp.n_batch = cp.n_ubatch = 32; + cp.n_threads = cp.n_threads_batch = 4; + cp.offload_kqv = false; + cp.op_offload = false; + cp.no_perf = true; + cp.swa_full = false; + cp.kv_unified = false; + cp.type_k = cp.type_v = GGML_TYPE_F16; + cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_AUTO; + if (taps) { + cp.cb_eval = on_node; + cp.cb_eval_user_data = &state; + } + std::unique_ptr ctx(llama_init_from_model(model.get(), cp), llama_free); + require(bool(ctx), "cannot create CPU context"); + require(llama_n_ctx_seq(ctx.get()) == static_cast(capacity) && llama_n_seq_max(ctx.get()) == 1 && + llama_n_batch(ctx.get()) == 32 && llama_n_ubatch(ctx.get()) == 32, "actual context/batch capacities differ from requested protocol"); + manifest << "slots\t" << llama_n_seq_max(ctx.get()) << "\nctx_per_slot\t" << llama_n_ctx_seq(ctx.get()) + << "\nctx_total\t" << llama_n_ctx(ctx.get()) << "\nn_batch\t" << llama_n_batch(ctx.get()) + << "\nn_ubatch\t" << llama_n_ubatch(ctx.get()) << '\n'; + manifest.flush(); + for (llama_token id : ids) tokens << id << '\n'; + tokens.close(); + steps << "call\tprocessed_pos\tpredict_pos\tinput_id\targmax\n"; + batch_guard batch; + for (int call = 0; call < calls; ++call) { + const int at = call == 0 ? 0 : prompt + call - 1; + const int count = call == 0 ? prompt : 1; + auto & b = batch.batch; + b.n_tokens = count; + for (int i = 0; i < count; ++i) { + b.token[i] = ids[at + i]; + b.pos[i] = at + i; + b.n_seq_id[i] = 1; + b.seq_id[i][0] = 0; + b.logits[i] = i == count - 1; + } + if (taps) state.begin(call, count); + const int status = llama_decode(ctx.get(), b); + if (taps) state.finish(); + require(status == 0, "llama_decode failed at call " + std::to_string(call) + ": " + std::to_string(status)); + const float * row = llama_get_logits_ith(ctx.get(), -1); + require(row != nullptr, "missing logits at call " + std::to_string(call)); + int argmax = 0; + for (int i = 0; i < vocab; ++i) { + if (!std::isfinite(row[i])) { + throw std::runtime_error("nonfinite logits at call " + std::to_string(call) + " lane " + std::to_string(i)); + } + if (row[i] > row[argmax]) argmax = i; + } + write_row(logits_out, row, vocab); + if (taps) { + write_row(layers_out, state.rows.data(), static_cast(layers) * width); + write_row(norm_out, state.rows.data() + static_cast(layers) * width, width); + } + const int processed = at + count - 1; + steps << call << '\t' << processed << '\t' << processed + 1 << '\t' << ids[processed] << '\t' << argmax << '\n'; + } + logits_out.close(); + steps.close(); + tensors.close(); + if (taps) { + layers_out.close(); + norm_out.close(); + } + manifest << "complete\t1\n"; + manifest.close(); + fs::rename(out / "MANIFEST.tsv.partial", out / "MANIFEST.tsv"); + std::fprintf(stderr, "flint_trace: %d calls, width=%d layers=%d vocab=%d taps=%s into %s\n", + calls, width, layers, vocab, taps ? "on" : "off", out.c_str()); + return 0; + } catch (const std::exception & e) { + std::fprintf(stderr, "flint_trace: %s\n", e.what()); + return 1; + } +} diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index b6e058838..1f9d41116 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -604,7 +604,13 @@ gguf 7 2 0 # PIN(2026-10-24): gpu 2420 -> 2475 blocks, 311 -> 315 fns, new code: checked # head128 ring append/attention, optional-bias floor router, expert-input scaling # and common loaded launchers. Existing entries retain their contracts; each new site states SAFETY. -gpu 2640 334 10 +# PIN(2026-10-25): gpu 2621 -> 2678 blocks, 326 -> 330 fns: current-main integration +# retains the six-slot F32 tier entry, head128 ring attention, floor router and +# expert-input scaling above; recounted by tools/check-unsafe.sh, with each new site's SAFETY contract. +# PIN(2026-10-26): gpu 2640 -> 2697 blocks, 334 -> 338 fns: Flint's same 57 blocks +# and four functions over af74ea4's MXFP8/HC epilogue baseline, recounted by tools/check-unsafe.sh. +# Both histories above retain their SAFETY contracts; no spare allowance is added. +gpu 2697 338 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. From d3d5cebe0fc2b49a86c4acd479242680b1f3c645 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:23:03 +0900 Subject: [PATCH 40/85] docs(flint): bind rebased candidate and instrumentation controls --- docs/research/flint-attach.md | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 7666cd1c4..50ddf296d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -654,9 +654,12 @@ residency/NVMe, streamed HTTP and spontaneous tool selection are not established ## 11. Current-main integration and trajectory instruments -The linear integration candidate starts at main -`43073091be21e48cda40d79fe6dae1dfcfd643d5`; it preserves the published behavioral -and resource evidence in section 10 as historical evidence, not acceptance of a new head. +The first linear integration candidate, `4dadee0195fb652608be59ffecc033a4643ad1eb`, +starts at main `43073091be21e48cda40d79fe6dae1dfcfd643d5`. The follow-up replay +starts at main `af74ea46b7f3e47692c2afae09eb64fdddf71461`; neither replay replaces +the published #799 head. Fixed-source numerical comparisons keep the original production +and first-candidate images separate from the later replay, and receipts name each image. +Section 10 remains historical evidence, not acceptance of either integration candidate. The placement integration keeps both the cached upload `Footprint` and candidate-selected tier scratch/reserves. No compiler pin, numerical band or model admission was widened. @@ -716,6 +719,10 @@ The CPU callback arm was actually run on the complete model: 75 calls, 3,675 lay zero preceding argmax mismatches, and final argmax 26007. Its complete final-logit SHA-256 is `083c8aa95aab025c0183c2ee3c4788642499533d5a2ee4e280c1125302c23bf9`, bit-identical to the preserved callback-free original row. The model was rehashed in this round. +The separate untapped control also ran: all 75 complete 150,144-logit rows are +bit-identical to the callback arm, whole-file SHA-256 +`7f2e476bd7fbf33faf6548eedadbb89b1844c50e0a55d7fc8aa25b297fbaff4a`. +These are results of the first candidate's CPU consumer, not new GPU results. The GPU build uses the pinned oxide toolchain. Actual capture runs belong inside an authorized runner configuration with its normal card locks and bounds; `TRACE` names @@ -727,12 +734,19 @@ bash tools/gpu-gate.sh --executable "$TRACE" -m "$MODEL" --ids "$IDS" \ --prompt 22 --ctx 1024 --slots 2 --mode graph --out "$GPU_OUT" ``` -Use separate new output directories for Graph/untapped, Eager/untapped and Eager/tapped -(`--taps`) arms. Arming taps forces Eager; the consumer explicitly restores the requested -mode afterwards, and Graph decode must actually capture. The instrument admits only -the existing card-budget lever; do not assume other inherited levers were applied. +Use separate new output directories for all four mode/instrumentation combinations: +Eager/untapped, Eager/tapped, Graph/untapped and Graph/tapped (`--taps`). +Check full-logit neutrality within each mode as well as Eager/Graph agreement. +Arming taps forces Eager; the consumer explicitly restores the requested mode afterwards, +and Graph decode must actually capture. The instrument admits only the existing +card-budget lever; do not assume other inherited levers were applied. Context 1024 is per slot, two slots total 2048, matching the original HTTP setup. Compare the actual per-layer placement, not just model identity or aggregate counts. +The diagnostic's all-false `HostCfg` matches the original `r8=off` and `HOST_LOCK=0` +settings, but page population/discard differ from the original HTTP defaults. Record that +difference; it is neither an established numerical cause nor permission to skip direct-B +boundary reproduction. `norm.f32` is the weighted RMS output before head activation +quantization, not an RMS-only row or quantized activation bytes. Diagnose the original `313b9f0` production source with the separately recorded example overlay first; then bridge the same controls onto the integration candidate. A new binary @@ -757,9 +771,10 @@ refuse the fixture tier. The two-card recipe takes the existing `both`/big-load path and has a derived host-resource ceiling, not a measured RSS claim. It does not prove dynamic residency/NVMe, loss/deadline recovery or a Korean oracle result. -The new GPU diagnostic and two-card case require fresh NVIDIA execution. Hikari's existing -work was explicitly left undisturbed; the new window waits for its owner. No cc12.1 values -were inferred and the remaining resource/oracle acceptance stays open. +Require fresh NVIDIA evidence tied to each candidate's actual source, model, placement +and executed hardware case; compiled/ignored tests are not such evidence. Preserve +existing workloads unless their owner authorizes a separate window. Physical cc12.1 +resource values must be measured, never inferred from an sm_86 build or cc8.6 run. From 412e5b3ccdc481cdc211af89d049cab75ff779dd Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:51:10 +0900 Subject: [PATCH 41/85] test(gpu): load mixed expert fixtures through placed rows --- crates/gpu/src/card_experts.rs | 39 +++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 4fd16c23f..4b898e316 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -885,6 +885,7 @@ mod tests { use gguf::GgmlType; use gguf::write::{Layout, TensorDecl, Writer}; use model::arch::models::{self, Mixer, Moe, Residual, Router, RouterNorm, Score}; + use model::placement::{self, ExpertList, ModelTensor, ModelTensors, Role}; use crate::card_leg::tests::stack_bytes; use crate::host::leg_tier::LegTier; @@ -936,10 +937,21 @@ mod tests { fn weights(gpu: &Gpu, triples: &[[GgmlType; 3]]) -> Weights { let mut decls = Vec::new(); let mut data = Vec::new(); + let mut tensors = Vec::new(); for (l, triple) in triples.iter().enumerate() { for (part, (name, &ty)) in (FAMILY.stacks)(l).into_iter().zip(triple).enumerate() { let (rows, k) = if part == 2 { (N, FF) } else { (FF, N) }; let bytes = stack_bytes(ty, CARD * rows, k, 7 + (3 * l + part) as u64); + tensors.push(ModelTensor { + name: name.clone(), + shard: 0, + layer: Some(l), + role: Role::RoutedExperts, + ty, + dims: vec![k as u64, rows as u64, CARD as u64], + file_bytes: bytes.len() as u64, + gathered_rows: None, + }); decls.push(TensorDecl { name, dims: vec![k as u64, rows as u64, CARD as u64], @@ -963,7 +975,32 @@ mod tests { } writer.finish().expect("a finished file"); let file = gguf::Split::open(&path).expect("the file"); - let weights = Weights::load_where(gpu.stream(), &file, |_| true).expect("resident stacks"); + // IQ3_S has a selected expert row arm, not a generic device format. + // Load the role-qualified placement as a real resident expert load does. + let model = ModelTensors { + tensors, + layers: triples.len(), + experts: EXPERTS as u64, + experts_used: USED as u64, + bf16_words: |_| false, + }; + let rows = model + .tensors + .iter() + .enumerate() + .map(|(i, t)| { + placement::routed_row( + i, + t, + 0, + ExpertList::prefix(CARD as u64).expect("the card prefix"), + &model, + ) + .expect("the resident expert row") + }) + .collect::>(); + let weights = Weights::load_rows(gpu.stream(), &file, &model, &rows, 0) + .expect("resident expert stacks"); std::fs::remove_file(path).expect("remove the fixture"); weights } From 89378a80dbd8d3ebcb8f1e3ef2a097eb41fee6d4 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:32:00 +0900 Subject: [PATCH 42/85] fix(flint): match HTTP diagnostic boundary and selected fixture formats --- crates/gpu-flint/examples/trace_flint.rs | 20 ++++++------ crates/gpu/src/card_experts.rs | 40 ++++++++++++++++++------ docs/research/flint-attach.md | 26 ++++++++++----- tools/ref/flint_trace.cpp | 18 ++++++----- 4 files changed, 70 insertions(+), 34 deletions(-) diff --git a/crates/gpu-flint/examples/trace_flint.rs b/crates/gpu-flint/examples/trace_flint.rs index 9bfa0b6ed..b592c2cda 100644 --- a/crates/gpu-flint/examples/trace_flint.rs +++ b/crates/gpu-flint/examples/trace_flint.rs @@ -28,7 +28,7 @@ const CALLS: usize = TOTAL - PROMPT + 1; const CTX: usize = 1024; const SLOTS: usize = 2; const USAGE: &str = "Usage: trace_flint -m MODEL --ids FILE --prompt 22 --out NEWDIR --ctx 1024 --slots 2 --mode eager|graph [--taps]\n\ -Captures flint-trajectory-v1: one 22-ID batched prefill, then 74 forced one-ID steps.\n\ +Captures flint-trajectory-v2: 21-ID prefix prefill, then the last prompt ID and 74 forced one-ID steps.\n\ FILE must contain exactly 96 decimal token IDs; no tokenization, sampling or EOG stopping.\n\ --ctx is per-slot (total context 2048); exactly one CUDA device must be visible.\n\ --taps captures each call's last-position layer outputs and result_norm.\n\ @@ -304,7 +304,7 @@ impl Capture { .into()); } let captured = m.has_capture(); - if m.mode() == StepMode::Graph && call != 0 && !captured { + if m.mode() == StepMode::Graph && !captured { return Err(format!("call {call}: graph mode did not capture a decode chain").into()); } if m.mode() == StepMode::Eager && captured { @@ -410,16 +410,16 @@ fn run(args: Args) -> Result<()> { m.set_mode(args.mode); placement(&out, &plan, &m)?; let mut capture = Capture::new(&out, args.taps)?; - let argmax = prompt_pass(&mut m, |m| { - m.run_rows(PROMPT, WHAT, |gpu, weights, body, head, at| { - prefill(gpu, weights, body, head, &ids[..PROMPT], at) + // The HTTP Slot::close_call prefills all but the last prompt ID, then calls next. + prompt_pass(&mut m, |m| { + m.run_rows(PROMPT - 1, WHAT, |gpu, weights, body, head, at| { + prefill(gpu, weights, body, head, &ids[..PROMPT - 1], at) }) })? - .ok_or("batched prompt produced no final argmax")?; - capture.call(&m, 0, ids[PROMPT - 1], argmax)?; - for (offset, &id) in ids[PROMPT..].iter().enumerate() { + .ok_or("batched prompt prefix produced no final argmax")?; + for (call, &id) in ids[PROMPT - 1..].iter().enumerate() { let argmax = m.step(&[id])?; - capture.call(&m, offset + 1, id, argmax)?; + capture.call(&m, call, id, argmax)?; } let mut nodes = create(&out.join("graph-nodes.tsv"))?; writeln!(nodes, "node\tdescriptor")?; @@ -452,7 +452,7 @@ fn run(args: Args) -> Result<()> { }; writeln!( manifest, - "protocol\tflint-trajectory-v1\nmodel\t{}\nids_source\t{}\nengine\tbloomery-gpu-flint\nmode\t{mode}\ninstrumentation\t{instrumentation}\nctx_per_slot\t{CTX}\nctx_total\t{}\nslots\t{SLOTS}\nactive_slot\t0\nwidth\t{WIDTH}\nlayers\t{LAYERS}\nvocab\t{VOCAB}\nprompt\t{PROMPT}\ntotal_ids\t{TOTAL}\ncalls\t{CALLS}\nlogits\tlogits.f32\nlayers_file\t{layers_file}\nnorm_file\t{norm_file}\nlogits_layout\tf32le[call][150144]\nlayers_layout\tf32le[call][49][2048]\nnorm_layout\tf32le[call][2048]\nresidency\toff\nhost_populate\tfalse\nhost_lock\tfalse\ncard_dontneed\tfalse\nr8\tfalse\nroute_trace\tfalse\nprefill_group\t1\nprompt_submission\tone-batched-prefill\nprompt_mode\teager-batched\ndecode_graph_captured\t{}\nplacement\tplacement.tsv\nloaded_expert_slots\texpert-slots.tsv\nplacement_counters\tplacement.txt\ndevice\tdevice.txt\ngraph_state\tgraph.tsv\ngraph_nodes\tgraph-nodes.tsv\nruntime_counters\tcounters.tsv\nprovenance\texternal-receipt-required\ncomplete\t1", + "protocol\tflint-trajectory-v2\nmodel\t{}\nids_source\t{}\nengine\tbloomery-gpu-flint\nmode\t{mode}\ninstrumentation\t{instrumentation}\nctx_per_slot\t{CTX}\nctx_total\t{}\nslots\t{SLOTS}\nactive_slot\t0\nwidth\t{WIDTH}\nlayers\t{LAYERS}\nvocab\t{VOCAB}\nprompt\t{PROMPT}\ntotal_ids\t{TOTAL}\ncalls\t{CALLS}\nlogits\tlogits.f32\nlayers_file\t{layers_file}\nnorm_file\t{norm_file}\nlogits_layout\tf32le[call][150144]\nlayers_layout\tf32le[call][49][2048]\nnorm_layout\tf32le[call][2048]\nresidency\toff\nhost_populate\tfalse\nhost_lock\tfalse\ncard_dontneed\tfalse\nr8\tfalse\nroute_trace\tfalse\nprefill_group\t1\nprompt_submission\tprefix-prefill-then-next\nprefix_prefill_ids\t21\nfirst_observed_call\tone-id-next\nprompt_mode\teager-batched-prefix\ndecode_graph_captured\t{}\nplacement\tplacement.tsv\nloaded_expert_slots\texpert-slots.tsv\nplacement_counters\tplacement.txt\ndevice\tdevice.txt\ngraph_state\tgraph.tsv\ngraph_nodes\tgraph-nodes.tsv\nruntime_counters\tcounters.tsv\nprovenance\texternal-receipt-required\ncomplete\t1", tsv_path(&model)?, tsv_path(&ids_path)?, CTX * SLOTS, diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 4b898e316..2203bed88 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -885,7 +885,10 @@ mod tests { use gguf::GgmlType; use gguf::write::{Layout, TensorDecl, Writer}; use model::arch::models::{self, Mixer, Moe, Residual, Router, RouterNorm, Score}; - use model::placement::{self, ExpertList, ModelTensor, ModelTensors, Role}; + use model::placement::{ + Device, ExpertList, ExpertPart, Format, ModelTensor, ModelTensors, Role, RoutedRule, Row, + Segment, + }; use crate::card_leg::tests::stack_bytes; use crate::host::leg_tier::LegTier; @@ -976,7 +979,7 @@ mod tests { writer.finish().expect("a finished file"); let file = gguf::Split::open(&path).expect("the file"); // IQ3_S has a selected expert row arm, not a generic device format. - // Load the role-qualified placement as a real resident expert load does. + // Use the selected program's formats, not the generic type/role table. let model = ModelTensors { tensors, layers: triples.len(), @@ -984,19 +987,36 @@ mod tests { experts_used: USED as u64, bf16_words: |_| false, }; + let RoutedRule::Leg { gate_up, down } = accepts::routed_rule(FAMILY.program) else { + panic!("the fixture family must select a card leg"); + }; let rows = model .tensors .iter() .enumerate() .map(|(i, t)| { - placement::routed_row( - i, - t, - 0, - ExpertList::prefix(CARD as u64).expect("the card prefix"), - &model, - ) - .expect("the resident expert row") + let format = match ExpertPart::of_name(&t.name).expect("a routed part") { + ExpertPart::Gate | ExpertPart::Up => gate_up(t.ty), + ExpertPart::Down => down(t.ty), + } + .expect("the selected arm's format"); + let mut segment = Segment { + device: Device::Card(0), + format: Format::Card(format), + experts: Some(ExpertList::prefix(CARD as u64).expect("the card prefix")), + resident_bytes: 0, + }; + segment.resident_bytes = segment + .buffer_bytes(t, model.experts_of(t)) + .expect("the selected format's buffers") + .into_iter() + .sum(); + Row { + tensor: i, + segments: vec![segment], + read_bytes: t.file_bytes, + stage: 0, + } }) .collect::>(); let weights = Weights::load_rows(gpu.stream(), &file, &model, &rows, 0) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 50ddf296d..9cf42239e 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -684,12 +684,17 @@ unchanged generic refusal, conflicting formats and insufficient union capacity. ### CPU and GPU use one forced-ID diagnostic artifact layout `tools/ref/flint_trace.cpp` and the `bloomery-gpu-flint` example `trace_flint` consume exactly -96 whitespace-separated decimal IDs: one 22-ID batched prefill, then 74 one-ID calls. -They do not retokenize, sample, or stop at an input EOG. Each observation saves the latest -processed row: prompt position 21, then positions 22 through 95, predicting positions -22 through 96. Earlier prompt rows are not claimed as captured history. - -The diagnostic format is `flint-trajectory-v1`, not a reference-set family or accuracy gate: +96 whitespace-separated decimal IDs. The original HTTP `Slot::close_call` in +`crates/serve/src/genloop.rs` prefills the first 21 prompt IDs, then calls `next` for the +22nd ID before the 74 forced continuation IDs. Both consumers reproduce that boundary. +They do not retokenize, sample, or stop at an input EOG. The 75 observations save processed +positions 21 through 95, predicting positions 22 through 96; the prefix itself is not an +observation. `--prompt 22` names the complete prompt, not a 22-ID batched submission. + +The format is `flint-trajectory-v2`, not a reference-set family or accuracy gate. +The former v1 consumer submitted all 22 prompt IDs as one batch: its controls remain +historical evidence but do not reproduce the actual HTTP prompt/step boundary. Do not +relabel v1 arrays or compare them as if the submission contracts matched. - `tokens.ids` and `steps.tsv`: exact inputs and each call's processed/predicted position and raw argmax. @@ -715,7 +720,7 @@ CUDA_VISIBLE_DEVICES= "$CPU_TRACE" -m "$MODEL" --ids "$IDS" \ --prompt 22 --ctx 1024 --out "$CPU_OUT" ``` -The CPU callback arm was actually run on the complete model: 75 calls, 3,675 layer vectors, +The preserved **v1** CPU callback arm was actually run on the complete model: 75 calls, 3,675 layer vectors, zero preceding argmax mismatches, and final argmax 26007. Its complete final-logit SHA-256 is `083c8aa95aab025c0183c2ee3c4788642499533d5a2ee4e280c1125302c23bf9`, bit-identical to the preserved callback-free original row. The model was rehashed in this round. @@ -724,6 +729,13 @@ bit-identical to the callback arm, whole-file SHA-256 `7f2e476bd7fbf33faf6548eedadbb89b1844c50e0a55d7fc8aa25b297fbaff4a`. These are results of the first candidate's CPU consumer, not new GPU results. +The corrected **v2** CPU callback and callback-free consumers also ran on the complete +model. Their 75 full-logit rows are bit-identical, SHA-256 +`4ef35ee7390341e640a6d38c39a010643f51a853fe743b29a5da989c7b8357b5`. +At next position96 the argmax is26007 and its margin over29263 is+0.238318443. +V1/v2 CPU arrays are not bit-identical; call16 has a different argmax. This is evidence +that the submission boundary matters, not a GPU fix or permission to waive the oracle. + The GPU build uses the pinned oxide toolchain. Actual capture runs belong inside an authorized runner configuration with its normal card locks and bounds; `TRACE` names the compiled example in that checkout's target directory: diff --git a/tools/ref/flint_trace.cpp b/tools/ref/flint_trace.cpp index db66e5e93..36bf894f8 100644 --- a/tools/ref/flint_trace.cpp +++ b/tools/ref/flint_trace.cpp @@ -180,6 +180,7 @@ int main(int argc, char ** argv) { if (std::strcmp(argv[i], "--help") == 0 || std::strcmp(argv[i], "-h") == 0) { std::fputs(usage, stdout); std::fputs("Read exactly 96 whitespace-separated decimal token IDs; never retokenize or stop on EOG.\n" + "Matches HTTP: 21-ID prefix prefill, last prompt ID as next, then 74 forced steps.\n" "Default: CPU layer/norm node taps and full logits. --no-taps: full logits only.\n" "NEWDIR must not exist; MANIFEST.tsv is installed only after complete capture.\n", stdout); return 0; @@ -201,7 +202,7 @@ int main(int argc, char ** argv) { else throw std::runtime_error(std::string("unknown/incomplete argument: ") + argv[i]); } require(!model_arg.empty() && !ids_arg.empty() && !out_arg.empty(), usage); - require(prompt == 22 && capacity == 1024, "flint-trajectory-v1 requires --prompt 22 --ctx 1024"); + require(prompt == 22 && capacity == 1024, "flint-trajectory-v2 requires --prompt 22 --ctx 1024"); const uint32_t endian = 1; require(*reinterpret_cast(&endian) == 1 && sizeof(float) == 4 && std::numeric_limits::is_iec559, "requires little-endian IEEE754 f32 host"); @@ -258,13 +259,14 @@ int main(int argc, char ** argv) { layers_out = output(out / "layers.f32", true); norm_out = output(out / "norm.f32", true); } - manifest << "protocol\tflint-trajectory-v1\nmodel\t" << model_path << "\narch\t" << arch + manifest << "protocol\tflint-trajectory-v2\nmodel\t" << model_path << "\narch\t" << arch << "\nmodel_name\t" << model_name << "\nmodel_bytes\t" << model_stat.st_size << "\nmodel_identity_validation\tmetadata-and-file-size;SHA256-in-external-receipt\n" << "mode\tcpu\ninstrumentation\t" << (taps ? "node-taps" : "no-taps") << "\nids_source\t" << ids_path << "\nwidth\t" << width << "\nlayers\t" << layers << "\nvocab\t" << vocab << "\nprompt\t" << prompt << "\ntotal_ids\t" << ids.size() << "\ncalls\t" << calls << "\nlogits\tlogits.f32\nlayers_file\t" << (taps ? "layers.f32" : "absent:not-requested") + << "\nprompt_submission\tprefix-prefill-then-next\nprefix_prefill_ids\t21\nfirst_observed_call\tone-id-next" << "\nnorm_file\t" << (taps ? "norm.f32" : "absent:not-requested") << "\nlogits_layout\t[call][vocab];little-endian-f32\nlayers_layout\t[call][layer][width];little-endian-f32" << "\nnorm_layout\t[call][width];little-endian-f32\ntensor_metadata\ttensors.tsv" @@ -302,9 +304,10 @@ int main(int argc, char ** argv) { tokens.close(); steps << "call\tprocessed_pos\tpredict_pos\tinput_id\targmax\n"; batch_guard batch; - for (int call = 0; call < calls; ++call) { - const int at = call == 0 ? 0 : prompt + call - 1; - const int count = call == 0 ? prompt : 1; + // Match HTTP Slot::close_call: the prefix has no observation; the last prompt ID is next. + for (int call = -1; call < calls; ++call) { + const int at = call < 0 ? 0 : prompt - 1 + call; + const int count = call < 0 ? prompt - 1 : 1; auto & b = batch.batch; b.n_tokens = count; for (int i = 0; i < count; ++i) { @@ -314,10 +317,11 @@ int main(int argc, char ** argv) { b.seq_id[i][0] = 0; b.logits[i] = i == count - 1; } - if (taps) state.begin(call, count); + if (taps && call >= 0) state.begin(call, count); const int status = llama_decode(ctx.get(), b); - if (taps) state.finish(); + if (taps && call >= 0) state.finish(); require(status == 0, "llama_decode failed at call " + std::to_string(call) + ": " + std::to_string(status)); + if (call < 0) continue; const float * row = llama_get_logits_ith(ctx.get(), -1); require(row != nullptr, "missing logits at call " + std::to_string(call)); int argmax = 0; From d6c372774f919983cd95a4c971b28fbd30f04e72 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 04:17:22 +0900 Subject: [PATCH 43/85] test(gpu): remove obsolete Solar2 Q6_K rejection --- crates/gpu/src/card_experts.rs | 65 ++++++++++++++++------------------ docs/research/flint-attach.md | 8 +++++ 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 2203bed88..8f5eb6fd4 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -1235,40 +1235,37 @@ mod tests { } assert!(gpu.take_fault().expect("the fault word").is_none()); } - for (triple, program) in [ - ([IQ3_S, Q5_K, Q8_0], Program::Qwen35Body), - ([Q5_K, Q5_K, Q6_K], Program::Solar2Body), - ] { - let w = weights(&gpu, &[triple]); - let specs = layers(1); - let map = SlotMap::prefix(0..1, EXPERTS, CARD).expect("a map"); - let family = CardFamily { program, ..FAMILY }; - assert!( - matches!( - CardExperts::new(&gpu, &w, family, &specs, &map, (N, FF), CAP), - Err(GpuError::Tensor { .. }), - ), - "{program:?}: {triple:?}" - ); - let set = - TierSet::new(0..1, EXPERTS, vec![(0..CARD as u32).collect()]).expect("a tier set"); - assert!( - matches!( - LegTier::new( - &gpu, - &specs, - &set, - &w, - family.shape(N, FF, 2), - program, - |moe| Ok(Act::SwigluClamp { - limit: moe.act.swiglu_limit() - }), - ), - Err(GpuError::Tensor { .. }), + let triple = [IQ3_S, Q5_K, Q8_0]; + let program = Program::Qwen35Body; + let w = weights(&gpu, &[triple]); + let specs = layers(1); + let map = SlotMap::prefix(0..1, EXPERTS, CARD).expect("a map"); + let family = CardFamily { program, ..FAMILY }; + assert!( + matches!( + CardExperts::new(&gpu, &w, family, &specs, &map, (N, FF), CAP), + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); + let set = + TierSet::new(0..1, EXPERTS, vec![(0..CARD as u32).collect()]).expect("a tier set"); + assert!( + matches!( + LegTier::new( + &gpu, + &specs, + &set, + &w, + family.shape(N, FF, 2), + program, + |moe| Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit() + }), ), - "{program:?}: {triple:?}" - ); - } + Err(GpuError::Tensor { .. }), + ), + "{program:?}: {triple:?}" + ); } } diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 9cf42239e..2b3fd0fc4 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -681,6 +681,14 @@ the assigned rows, with no plan violations. This is real-file planning, not devi The pure regression also covers mixed selected down forms, heap/reserve recounting, unchanged generic refusal, conflicting formats and insufficient union capacity. +The common mixed card/tier hardware consumer stages its fixture through `Weights::load_rows` +using `accepts::routed_rule(FAMILY.program)` and the shared segment buffer arithmetic. +Neither `Weights::load_where` nor the generic `routed_row` admits selected-only IQ3_S: +assigning a routed role alone is insufficient. The fixture keeps all mixed-format +step/pass/batch/tier bit comparisons and byte conservation, without widening generic +admission. The obsolete Solar2 Q6_K-down rejection is removed: the current selected owner +already admits that arm. Mismatched gate/up rejection remains exercised. + ### CPU and GPU use one forced-ID diagnostic artifact layout `tools/ref/flint_trace.cpp` and the `bloomery-gpu-flint` example `trace_flint` consume exactly From a0e1e901a258e1c43fc7bdc4567c05f0e328751d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:19:09 +0900 Subject: [PATCH 44/85] Reconcile selected scratch test contracts and format migrated callers --- .../src/bin/shared/serve_seats/minimaxm3.rs | 3 ++- crates/gpu/src/arch/qwen3moe/tier_leg.rs | 3 --- crates/gpu/src/card_experts.rs | 8 ++++++- crates/model/src/arch/glm5next/place.rs | 4 ++-- crates/model/src/arch/minimaxm3/place.rs | 24 +++---------------- crates/placement/src/accepts.rs | 7 +++++- crates/placement/src/placement.rs | 3 ++- crates/placement/src/placement/workstation.rs | 7 +----- 8 files changed, 23 insertions(+), 36 deletions(-) diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs index 27cd31f38..096093e07 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs @@ -505,7 +505,8 @@ impl Tiered for MiniMax { place: Place, inputs: &PlanInputs, ) -> Result, GateError> { - Ok((!place.tier_cards().is_empty()).then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs))) + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs))) } /// The tags the model's description names for its think span. diff --git a/crates/gpu/src/arch/qwen3moe/tier_leg.rs b/crates/gpu/src/arch/qwen3moe/tier_leg.rs index 5539b6dff..de8854b21 100644 --- a/crates/gpu/src/arch/qwen3moe/tier_leg.rs +++ b/crates/gpu/src/arch/qwen3moe/tier_leg.rs @@ -70,7 +70,6 @@ fn qwen_act(moe: &Moe) -> Result { } } - /// The host views of the routed layers `run`, the refusal naming the run as /// `word`'s. `n_layer` is the description's layer count, so a trunk layer is /// not refused for sitting past a next-token block. @@ -155,7 +154,6 @@ impl LegFamily for Qwen3 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - } /// Qwen3.6's facts for the common tier node service. The description's @@ -214,7 +212,6 @@ impl LegFamily for Qwen35 { fn tier_act(moe: &Moe) -> Result { qwen_act(moe) } - } /// Qwen3-30B's tier node service ([`leg_tier::node_family`]). diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 23266831f..33e8afca2 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -1214,7 +1214,13 @@ mod tests { assert!( matches!( LegTier::new( - &gpu, &specs, &set, &w, family.shape(N, FF, 2), program, tier_act, + &gpu, + &specs, + &set, + &w, + family.shape(N, FF, 2), + program, + tier_act, ), Err(GpuError::Tensor { .. }), ), diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 82efa5b6a..bca8663f6 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -1880,7 +1880,7 @@ mod tests { use GgmlType::{IQ3_S, IQ4_XS, NVFP4, Q4_K, Q5_K, Q6_K, Q8_0}; let q8 = DownForms::of(DownAct::Q8Act); let b32 = DownForms::of(DownAct::Q8Blocks32); - let base = tier_batch(&inputs()); + let base = tier_batch(&inputs()).expect("valid tier batch"); let hp = inputs().hp; let leg = |forms| { leg_bytes( @@ -1908,7 +1908,7 @@ mod tests { q8.with(DownAct::Q8Blocks32), ), ] { - let batch = tier_batch(&routed_types(&layers)); + let batch = tier_batch(&routed_types(&layers)).expect("valid selected tier batch"); assert_eq!(batch.scratch, leg(forms), "{layers:?}"); assert_eq!(batch.staging, base.staging, "{layers:?}"); assert_eq!(batch.host, base.host, "{layers:?}"); diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 6cf405c84..309200f48 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -418,31 +418,13 @@ mod tests { )); } - /// A machine that names an expert tier card (plan (b′)) is planned: the body serves one. The - /// card keeps the prompt batch's reserve on either machine and, on the tiered one, the tier - /// places' copies beside it ([`tier_places_bytes`]: 3 × 4 B × 256 rows × 2 slots). The tier's batch bytes are the common leg's: runs of - /// [`TIER_CHUNK`] tokens over blocks of the unit's `cols` rows, in the forms the leg's routed - /// downs read: the header's F32 downs run on no leg, so the 128-value form - /// ([`workstation::tier_leg_batch_bytes`]). The plan reaches the placement, which refuses the - /// tier idle: the header's experts all fit the stage card. Mutant: the tier reserve left out - /// of the description (the plan stops at [`PlaceError::Tier`]), the forms of a file with no - /// leg down left empty (the scratch holds no slot columns), or the run's tokens and the - /// block's rows swapped. + /// A tiered machine retains the stage's prompt reserve and the tier places beside it. + /// The fixture's experts fit on the stage, so placement refuses the idle tier rather + /// than rejecting the description as a body without tier support. #[test] fn a_tier_card_is_planned_with_the_leg_batch() { const COLS: u64 = 3 * TIER_CHUNK; let inputs = described("mm3-place-tier"); - let hp = &inputs.hp; - assert_eq!( - tier_batch(&inputs, COLS), - workstation::tier_leg_batch_bytes( - hp.n_embd as u64, - hp.experts.ff as u64, - hp.experts.n_used as u64, - TIER_CHUNK, - COLS, - ) - ); let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs, COLS)); let plain = workstation::plan_gate(LAYERS); let places = 3 * 4 * 256 * 2; diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index c06b86c81..f6bd1f8c4 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -508,7 +508,12 @@ pub fn leg_runs(p: Program, [gate, up, down]: [GgmlType; 3]) -> Option { pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> kernels::DownForms { use crate::placement::{ExpertPart, Role}; - let RoutedRule::Leg { gate_up, down, pair } = routed_rule(p) else { + let RoutedRule::Leg { + gate_up, + down, + pair, + } = routed_rule(p) + else { return kernels::DownForms::NONE; }; let mut forms = kernels::DownForms::NONE; diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index b512fa89d..2c09b7f87 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -3165,7 +3165,8 @@ impl Fill<'_, '_> { .map(|l| self.model.layer_experts(l)) .collect(); spread(n_l, held, self.eligible, &caps, budget, |n| { - footprint.at(n)? + footprint + .at(n)? .checked_add(scratch(n)?) .ok_or_else(|| card.bytes_past_u64()) }) diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 4e1a18893..7df753d30 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -529,12 +529,7 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// columns in the 128-value form ([`leg_bytes`]), in place of the tile path's /// scratch. Selected-form callers use [`tier_leg_batch_bytes_of`]. #[must_use] -pub const fn tier_chunk_scratch_bytes( - n_embd: u64, - ff: u64, - n_used: u64, - chunk: u64, -) -> u64 { +pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) } From 29a29b15451641c33399ac419e289ed1ff7796e4 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:19:09 +0900 Subject: [PATCH 45/85] Format the combined tier handoff guard --- crates/gpu/src/host/handoff.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 474158494..84f35cdc5 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -1634,8 +1634,7 @@ fn check_tier(lay: &HandoffLayout, tl: &TierImageLayout) -> Result Date: Tue, 27 Oct 2026 05:19:09 +0900 Subject: [PATCH 46/85] Keep IQ4 regression checks behavioral and format the merged sites --- crates/gpu-deepseek41/src/block.rs | 5 +++- crates/gpu-deepseek41/src/shared_routed.rs | 3 +- crates/gpu-gates/src/act_rule.rs | 35 ++++------------------ crates/gpu/src/site.rs | 15 ++++++++-- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index f5c634118..d35f5b674 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -676,7 +676,10 @@ mod tests { assert_eq!(plan([Q6K, Q6K, Q5K]).unwrap().gate_up, split(Q6K, Q6K)); assert_eq!(plan([Q4K, Q5K, Q4K]).unwrap().gate_up, split(Q4K, Q5K)); assert_eq!(plan([Q3K, Q3K, Q6K]).unwrap().gate_up, split(Q3K, Q3K)); - assert_eq!(plan([Iq4Xs, Iq4Xs, Iq4Xs]).unwrap().gate_up, split(Iq4Xs, Iq4Xs)); + assert_eq!( + plan([Iq4Xs, Iq4Xs, Iq4Xs]).unwrap().gate_up, + split(Iq4Xs, Iq4Xs) + ); assert_eq!( plan([Mxfp8, Mxfp8, Mxfp8]).unwrap().gate_up, GateUp::Site { diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index d3eb066d7..29bfb4742 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -1189,7 +1189,8 @@ fn no_batch() -> GpuError { mod tests { use super::SharedWide; use crate::block::{ - Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, enqueue_block_cols_act, + Block, BlockIo, BlockKernels, BlockPlan, BlockScratch, KBlockKernels, + enqueue_block_cols_act, }; use crate::experts::ExpertKernels; use bloomery_gpu::Gpu; diff --git a/crates/gpu-gates/src/act_rule.rs b/crates/gpu-gates/src/act_rule.rs index b819959ae..589e235fc 100644 --- a/crates/gpu-gates/src/act_rule.rs +++ b/crates/gpu-gates/src/act_rule.rs @@ -982,7 +982,11 @@ mod site_tests { let (q32, q128, q256) = (Some(32), Some(Q8_1_BLOCK), Some(QK_K)); for arm in [Arm::Gemv, Arm::Gemm, Arm::Expert] { assert_eq!(site_rel(GgmlType::Q3_K, arm), q(q256, q128), "Q3_K {arm:?}"); - assert_eq!(site_rel(GgmlType::IQ4_XS, arm), q(q256, q128), "IQ4_XS {arm:?}"); + assert_eq!( + site_rel(GgmlType::IQ4_XS, arm), + q(q256, q128), + "IQ4_XS {arm:?}" + ); for ty in [GgmlType::Q4_K, GgmlType::Q5_K, GgmlType::Q6_K] { assert_eq!(site_rel(ty, arm), q(q32, q128), "{ty} {arm:?}"); } @@ -1043,35 +1047,6 @@ mod site_tests { } } - /// The rounding relation follows the common site's activation forms and - /// the grouped GEMM's existing decoders. - #[test] - fn site_rel_reads_the_forms_the_launches_name() { - use gguf::card::{SiteAct, SiteEntry}; - for entry in [ - SiteEntry::Q3k, - SiteEntry::Q4k, - SiteEntry::Q5k, - SiteEntry::Q6k, - SiteEntry::Iq4xs, - ] { - assert_eq!(entry.reads(), SiteAct::Q8x128); - } - assert_eq!(SiteEntry::Q8_0.reads(), SiteAct::Q8x32); - assert_eq!(SiteEntry::F32.reads(), SiteAct::F32); - assert_eq!(SiteEntry::Q5_0.reads(), SiteAct::Q8Blocks32); - let gpu = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../gpu/src"); - let read = |file: &str| { - std::fs::read_to_string(gpu.join(file)) - .unwrap_or_else(|e| panic!("crates/gpu/src/{file}: {e}")) - }; - assert!(read("gemm/mod.rs").contains("(Q8_0, IQ4_NL, Q5_1), over [`GemmAct32`]")); - let grouped = read("gemm/grouped.rs"); - for variant in ["Iq3Xxs", "Iq4Xs"] { - assert!(grouped.contains(variant), "the grouped GEMM no longer decodes {variant}"); - } - } - /// ik's side of the table is the transcription this module already holds for the K-quants /// ([`Rule::of`]), and so is ours for Q3_K and Q4_K; Q5_K's ours differs from it by design (the /// V4.1 gemv reads f32, the bodies' site reads q8_1). diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 0e5e7696f..3fe55f491 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -1765,7 +1765,13 @@ mod tests { limit: 7.0, }; let launch = |ty, rule| swiglu_launch((ty, "down"), rule); - for ty in [SiteTy::Q3K, SiteTy::Q4K, SiteTy::Q5K, SiteTy::Q6K, SiteTy::Iq4Xs] { + for ty in [ + SiteTy::Q3K, + SiteTy::Q4K, + SiteTy::Q5K, + SiteTy::Q6K, + SiteTy::Iq4Xs, + ] { for rule in [Act::SiluMul, clamp, oai] { assert_eq!(launch(ty, rule).ok(), Some(SwigluLaunch::Q128), "{ty}"); } @@ -1842,7 +1848,12 @@ mod tests { (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - (SiteTy::Iq4Xs, GgmlType::IQ4_XS, "IQ4_XS", Some(GemmWeight::Iq4Xs)), + ( + SiteTy::Iq4Xs, + GgmlType::IQ4_XS, + "IQ4_XS", + Some(GemmWeight::Iq4Xs), + ), ]; assert_eq!(SiteTy::ALL.len(), expected.len()); for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { From efe6c7b63529bbcea43b174b247d2e696cdb9c60 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:24:03 +0900 Subject: [PATCH 47/85] Complete the mixed fixture description and remove superseded reserve regression --- crates/gpu/src/card_experts.rs | 2 ++ crates/model/src/arch/glm5next/place.rs | 48 ------------------------- docs/research/flint-attach.md | 7 ++++ 3 files changed, 9 insertions(+), 48 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 33e8afca2..eb22b78d7 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -880,6 +880,8 @@ mod tests { top_k: USED as u32, expert_ff: FF as u32, latent: None, + pre_exp_norm: false, + post_exp_norm: None, act: models::Act::SwiGlu { limit: None }, router: Router { score: Score::Sigmoid, diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index bca8663f6..99cd5a479 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -1867,54 +1867,6 @@ mod tests { } } - /// The expert tier's prompt-batch reserve follows the down forms of the file's card layers: - /// its scratch is the card leg's one formula (`kernels::leg_bytes`, which `LegBufs::for_downs` - /// allocates by) over `TIER_CHUNK` tokens in each form a layer's down reads, the staging and - /// host terms beside it moved by no form. A K-quant file, a Q6_K down and a file with no card - /// layer reserve the 128-value form alone; an NVFP4 layer or a Q8_0 down the 32-value form; layers - /// of both, both, the tiles' layers (Q4_K gate and up over a Q5_K down) counted with the rest. - /// Mutants: the reserve of the 128-value form alone; the tiles' layers left out of the forms. - #[test] - fn the_tier_reserve_follows_the_routed_downs() { - use crate::placement::kernels::{DownAct, DownForms, leg_bytes}; - use GgmlType::{IQ3_S, IQ4_XS, NVFP4, Q4_K, Q5_K, Q6_K, Q8_0}; - let q8 = DownForms::of(DownAct::Q8Act); - let b32 = DownForms::of(DownAct::Q8Blocks32); - let base = tier_batch(&inputs()).expect("valid tier batch"); - let hp = inputs().hp; - let leg = |forms| { - leg_bytes( - TIER_CHUNK, - hp.n_used as u64, - hp.n_embd as u64, - hp.expert_ff as u64, - forms, - ) - }; - assert_ne!(leg(q8), leg(b32)); - for (layers, forms) in [ - (vec![], q8), - (vec![[Q4_K, Q4_K, Q5_K]], q8), - (vec![[Q5_K, Q5_K, Q6_K]], q8), - (vec![[IQ3_S, IQ3_S, IQ4_XS]], q8), - (vec![[NVFP4; 3]], b32), - (vec![[Q4_K, Q4_K, Q8_0]], b32), - ( - vec![[IQ3_S, IQ3_S, IQ4_XS], [NVFP4; 3]], - q8.with(DownAct::Q8Blocks32), - ), - ( - vec![[Q4_K, Q4_K, Q5_K], [NVFP4; 3]], - q8.with(DownAct::Q8Blocks32), - ), - ] { - let batch = tier_batch(&routed_types(&layers)).expect("valid selected tier batch"); - assert_eq!(batch.scratch, leg(forms), "{layers:?}"); - assert_eq!(batch.staging, base.staging, "{layers:?}"); - assert_eq!(batch.host, base.host, "{layers:?}"); - } - } - /// A Q5_K gate and up with a Q6_K down takes every expert on a card /// with room for the stack; a Q6_K gate and up takes none. /// PIN(2026-10-22): the down arm admits Q6_K, so the layer is diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 6971fdff8..9ed1cc9e7 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -84,6 +84,13 @@ The wrapper fixture loads each expert part through the selected program's `route compare real step, pass, batch and tier allocations and rows against the common leg. A mismatched gate/up pair remains refused; Solar's admitted Q6_K down is not a rejection case. Tier activation selection remains the family's contract, independently of the program-owned stack admission. + +The 2026-10-26 cutover removes the obsolete always-Q8 tier-reserve regression: +selected mixed-layer and mismatched-gate/up coverage remains in +`tier_scratch_follows_selected_mixed_layers`, with allocation and budget boundaries +covered by the placement and GPU wrapper consumers. The MiniMax planner test retains +its reserve and idle-tier behavior checks rather than comparing descriptor representation +against a different constructor. Host-only projections intentionally have no card totals: fixed reserves remain owned by their machine. Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. From 3071a9f0d8ed4773692e0c040dece222b90f6bec Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:29:37 +0900 Subject: [PATCH 48/85] Drop imports made obsolete by selected reserve regression cutover --- crates/model/src/arch/glm5next/place.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index 99cd5a479..c1c6f373a 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -1480,8 +1480,7 @@ mod tests { use super::super::super::hparams::{Hc, Hparams, Indexer}; use super::super::{ DenseForms, KdaLanes, Kind, KvLayout, NextnInputs, NextnKv, PlaceError, PlanInputs, - TIER_CHUNK, card_leg_bytes, card_leg_forms, prompt_reserve_bytes, row_bytes, - tier_batch, + card_leg_bytes, card_leg_forms, prompt_reserve_bytes, row_bytes, }; use crate::placement::churn::ChurnPool; use crate::placement::workstation::{HostNeed, HostRead}; From efd97b45e073e81d0a35eadcc6cefa8a9b715c09 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:38:43 +0900 Subject: [PATCH 49/85] Reconcile final Flint selected reserve APIs and measured unsafe census --- crates/gpu/src/card_experts.rs | 1 - crates/model/src/arch/glm5next/place.rs | 1 - crates/model/tests/qwen3_bp_plan.rs | 7 +------ crates/placement/src/placement.rs | 6 ++++-- crates/placement/src/placement/workstation.rs | 10 ++++++---- tools/unsafe-ratchet.txt | 12 ++++++++++-- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index d0d9f36b5..7da1774ce 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -888,7 +888,6 @@ mod tests { Segment, }; - use crate::card_leg::tests::stack_bytes; use crate::host::leg_tier::LegTier; use crate::host::tier::{TierBlock, TierExperts, TierInput, TierIo, TierSet}; diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index b6ede12c6..7eb76c628 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -1867,7 +1867,6 @@ mod tests { } } - /// A Q5_K gate and up with a Q6_K down takes every expert on a card /// with room for the stack; a Q6_K gate and up takes none. /// PIN(2026-10-22): the down arm admits Q6_K, so the layer is diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index 5efb42a34..ff86bc0f0 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -537,12 +537,7 @@ fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place #[test] fn qwen3_30b_q8_bp_tier_experts() { let inputs = qwen3_inputs(); - let machine = tiered( - Q3_LAYERS, - A6000, - RTX_3090, - q3::tier_batch(&inputs.hp), - ); + let machine = tiered(Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp)); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) .expect("the A6000 stage plans a tier"); diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 38b23266f..d371c28cf 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -6758,7 +6758,8 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, try_tier_leg_batch_bytes_of}; let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); - let batch = try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both, false).expect("tier bytes"); + let batch = + try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both, false).expect("tier bytes"); let leg = batch.leg.expect("a tier leg descriptor"); let rule = routed_rule(Program::Qwen35Body); for (downs, first) in [ @@ -6834,7 +6835,8 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, TierLegScratch, try_tier_leg_batch_bytes_of}; let forms = DownForms::of(DownAct::Q8Act); - let batch = try_tier_leg_batch_bytes_of(256, 256, 1, (1, 1), forms, false).expect("valid batch"); + let batch = + try_tier_leg_batch_bytes_of(256, 256, 1, (1, 1), forms, false).expect("valid batch"); let good = batch.leg.expect("a leg descriptor"); let mut model = legged(&[[GgmlType::Q5_K; 3]]); model.experts_used = 1; diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index f7ebdf89b..945e0fd18 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -1746,10 +1746,12 @@ mod tests { let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); for chunk in [1, 8, 16] { for downs in [DownForms::NONE, DownForms::of(DownAct::Q8Act), both] { - let direct = super::try_tier_leg_batch_bytes_of(256, 256, 6, (chunk, 32), downs, false) - .expect("direct descriptor"); - let scaled = super::try_tier_leg_batch_bytes_of(256, 256, 6, (chunk, 32), downs, true) - .expect("scaled descriptor"); + let direct = + super::try_tier_leg_batch_bytes_of(256, 256, 6, (chunk, 32), downs, false) + .expect("direct descriptor"); + let scaled = + super::try_tier_leg_batch_bytes_of(256, 256, 6, (chunk, 32), downs, true) + .expect("scaled descriptor"); assert_eq!(scaled.staging, direct.staging); assert_eq!(scaled.host, direct.host); assert_eq!(scaled.scratch, direct.scratch + 4 * chunk * 6); diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 80ac1606e..c9bc0b731 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -626,7 +626,10 @@ gguf 7 2 0 # PIN(2026-10-26): gpu 2691 -> 2693 blocks, the six-slot F32 tier entry adds two blocks: # ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; # both blocks state their per-thread bounds and disjoint-write SAFETY invariant. -gpu 2693 346 10 +# PIN(2026-10-26): gpu 2693 -> 2748 blocks, 346 -> 350 fns: the existing Flint head128 ring +# attention, floor router and expert-input scaling retain their checked SAFETY contracts over +# main's new kernels; the six-slot entry is already included above. Counted by check-unsafe.sh. +gpu 2748 350 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. @@ -653,7 +656,12 @@ gpu 2693 346 10 # selector partials of dense.rs and experts.rs, `q5k_lane_partial`'s calls and their row stores) and the glue's Q5_K # token rows and IQ4_NL Engram rows (glue.rs: the value reads, the stream and input stores, the shared module), each # with its SAFETY line; counted by tools/check-unsafe.sh. -gpu-deepseek41 395 44 0 +# PIN(2026-10-26): gpu-deepseek41 395 -> 397 blocks: the configured residual module load +# and its checked row launch retain their SAFETY invariants over main's ISTA entries. +gpu-deepseek41 397 44 0 +# PIN(2026-10-24): gpu-flint starts at 2 blocks, 0 fns and 0 impls: step and prompt +# call the checked common attention binding with staged distinct positions, each with SAFETY. +gpu-flint 2 0 0 # PIN(2026-10-01): gpu-gates 60 -> 61 blocks, new code, not old code relaxed: gate_glm5next_serve starts # bloomery-serve with PR_SET_PDEATHSIG set between fork and exec (one pre_exec block, its SAFETY line), as the # Qwen3.8 serve gate does, so a bound that ends the gate does not leave the server holding the card. From 2b0724e3aa44e33a0f46da2dabff9720f5497d7e Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:42:26 +0900 Subject: [PATCH 50/85] Remove merge-duplicated accounting and restore fixed-form scratch callers --- crates/gpu-mimo2/src/tier.rs | 2 +- crates/gpu-minimaxm3/src/tier.rs | 2 +- crates/model/src/arch/mimo2/place.rs | 3 -- crates/placement/src/placement.rs | 34 ------------------- crates/placement/src/placement/card_budget.rs | 1 - crates/placement/src/placement/workstation.rs | 10 ++---- 6 files changed, 4 insertions(+), 48 deletions(-) diff --git a/crates/gpu-mimo2/src/tier.rs b/crates/gpu-mimo2/src/tier.rs index 37ee805ac..f0886e36c 100644 --- a/crates/gpu-mimo2/src/tier.rs +++ b/crates/gpu-mimo2/src/tier.rs @@ -34,8 +34,8 @@ use gguf::Split; use model::ModelError; use model::arch::mimo2::place::{self, PlanInputs}; use model::moe::HostLayer; +use model::placement::ModelTensors; use model::placement::workstation::TierBatchBytes; -use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::LayerSpec; diff --git a/crates/gpu-minimaxm3/src/tier.rs b/crates/gpu-minimaxm3/src/tier.rs index 60dd420d2..ad4f75122 100644 --- a/crates/gpu-minimaxm3/src/tier.rs +++ b/crates/gpu-minimaxm3/src/tier.rs @@ -20,8 +20,8 @@ use gguf::Split; use model::arch::minimaxm3::host as minimax_host; use model::arch::minimaxm3::place::{self, PlanInputs}; use model::moe::HostLayer; +use model::placement::ModelTensors; use model::placement::workstation::TierBatchBytes; -use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::{LayerSpec, Moe}; diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index d7aa20915..f1c0ef122 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -19,7 +19,6 @@ use super::{roles, spec}; use crate::arch::chat_of; use crate::arch::place_hybrid; use crate::placement::PlacementError; -use crate::placement::kernels::{DownAct, DownForms}; use crate::placement::workstation::{self, TierBatchBytes}; // The ring rule's owner is `runtime::stores`; these names stay on this path for @@ -102,7 +101,6 @@ pub fn prompt_shared_bytes(hp: &Hparams, cap: usize, card: bool) -> u64 { hp.expert_ff as u64, hp.n_used as u64, cap as u64, - DownForms::of(DownAct::Q8Act), ) as usize; leg + 2 * size_of::() * cap * n + size_of::() * cap * hp.n_used } else { @@ -214,7 +212,6 @@ mod tests { prompt_unit_bytes, tier_batch, tier_reserve_bytes, }; use crate::arch::synthetic::{V, header_shaped}; - use crate::placement::kernels::{DownAct, DownForms}; use crate::placement::{KvBytes, PlacementError, PlanLevers, workstation}; /// A file whose next-token layers carry tensors is refused at diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index d371c28cf..8e456a16b 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -3239,40 +3239,6 @@ impl Fill<'_, '_> { .routed .unwrap_or(RoutedRule::PerType(CardFormat::of_routed)); let uploads = card_uploads(self.model, self.c, self.rows, self.eligible, format)?; - let mut down_acts = if card.tier_leg.is_some() { - vec![None; self.model.layers] - } else { - Vec::new() - }; - if card.tier_leg.is_some() { - for t in &self.model.tensors { - if t.role == Role::RoutedExperts - && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) - && let Some(l) = t.layer - && let Some((_, act)) = crate::kernels::down(t.ty) - { - down_acts[l] = Some(act); - } - } - } - let scratch = |counts: &[u64]| { - card.tier_leg.map_or(Ok(0), |leg| { - let forms = counts - .iter() - .zip(&down_acts) - .filter_map(|(&n, &act)| (n > 0).then_some(act).flatten()) - .fold(crate::kernels::DownForms::NONE, |forms, act| { - forms.with(act) - }); - leg.bytes(forms) - }) - }; - let bound = card - .tier_leg - .map(|leg| leg.bytes(leg.downs)) - .transpose()? - .unwrap_or(0); - let reserves = card.selected_reserve_bytes(self.model, n_l)?; let footprint = Footprint::new(card.granule_bytes, &uploads); let mut down_acts = if card.tier_leg.is_some() { vec![None; self.model.layers] diff --git a/crates/placement/src/placement/card_budget.rs b/crates/placement/src/placement/card_budget.rs index 4d8e8695e..ce33830d7 100644 --- a/crates/placement/src/placement/card_budget.rs +++ b/crates/placement/src/placement/card_budget.rs @@ -209,7 +209,6 @@ mod tests { fn card(name: &str, ordinal: Option) -> Card { Card { name: name.to_owned(), - tier_leg: None, device: ordinal.map(|o| DeviceId { ordinal: o, uuid: [u8::try_from(o).expect("small") + 1; 16], diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 945e0fd18..1c9ea9c5c 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -540,14 +540,8 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// columns in the 128-value form ([`leg_bytes`]), in place of the tile path's /// scratch. Selected-form callers use [`tier_leg_batch_bytes_of`]. #[must_use] -pub const fn tier_chunk_scratch_bytes( - n_embd: u64, - ff: u64, - n_used: u64, - chunk: u64, - downs: DownForms, -) -> u64 { - leg_bytes(chunk, n_used, n_embd, ff, downs) +pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { + leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) } /// The host's part of the tier's prompt-batch service: per exchange set, the From adcfaf5308d531aa97c2aaa6e546598f802274d9 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:52:27 +0900 Subject: [PATCH 51/85] Complete fixed tier descriptor migration in model and frontend consumers --- crates/gpu-gates/src/bin/gate_musespark_prefill.rs | 2 +- crates/model/src/arch/minimaxm3/place.rs | 1 - crates/model/src/arch/musespark/place.rs | 1 - crates/model/tests/mimo2_meta.rs | 2 +- crates/model/tests/musespark_meta.rs | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs index 62dc44357..5adc0d835 100644 --- a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs @@ -529,7 +529,7 @@ mod gate { tier_now: Tier, ) -> Result<(Model, bool, Vec), GateError> { let inputs = PlanInputs::read(&open_split(Arch::MuseSpark, RECIPE)?)?; - let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp)?)?; + let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp))?; place::stage_placed(&mut two)?; let ctx = u64::try_from(ctx)?; // The stage card alone, the reference's machine: the budget reads its unbudgeted plan, which a plan of the two diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 63f73d714..309200f48 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -244,7 +244,6 @@ mod tests { use super::{PROMPT_ROWS, PlaceError, PlanInputs, TIER_CHUNK, stream_bytes, tier_batch}; use crate::arch::place_hybrid::ring_pass_bytes; use crate::arch::synthetic::{V, header_shaped}; - use crate::placement::kernels::{DownAct, DownForms}; use crate::placement::{PlacementError, PlanLevers, workstation}; /// The small header's global keys: the pre-tokenizer and a template the description reads. diff --git a/crates/model/src/arch/musespark/place.rs b/crates/model/src/arch/musespark/place.rs index b3d48389d..e47ddf052 100644 --- a/crates/model/src/arch/musespark/place.rs +++ b/crates/model/src/arch/musespark/place.rs @@ -628,7 +628,6 @@ mod tests { Machine { cards: vec![Card { name: "card".to_string(), - tier_leg: None, device: None, usable_bytes: usable, context_bytes: 0, diff --git a/crates/model/tests/mimo2_meta.rs b/crates/model/tests/mimo2_meta.rs index db975961b..64ecda04a 100644 --- a/crates/model/tests/mimo2_meta.rs +++ b/crates/model/tests/mimo2_meta.rs @@ -401,7 +401,7 @@ fn hw_mimo2_plan() { let tiered = workstation::plan_bp( layers, None, - model::arch::mimo2::place::tier_batch(&inputs.hp, 512).expect("tier batch bytes"), + model::arch::mimo2::place::tier_batch(&inputs.hp, 512), ); let bp = inputs .plan(&tiered, ctx, &PlanLevers::default()) diff --git a/crates/model/tests/musespark_meta.rs b/crates/model/tests/musespark_meta.rs index e765e3d56..d3cabdd9c 100644 --- a/crates/model/tests/musespark_meta.rs +++ b/crates/model/tests/musespark_meta.rs @@ -368,7 +368,7 @@ fn hw_musespark_plan() { workstation::A6000, &[workstation::RTX_3090], None, - place::tier_batch(&hp).expect("tier batch bytes"), + place::tier_batch(&hp), ) .unwrap_or_else(|e| panic!("the tiered machine: {e}")); place::stage_placed(&mut m).unwrap_or_else(|e| panic!("the tiered machine's stage: {e}")); From dd7253bf5cf183d3219b7754c01aaf8b144c7012 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:03:57 +0900 Subject: [PATCH 52/85] Reconcile quantized leg protection with current input contracts --- crates/gpu/src/card_leg.rs | 103 ++++++++++++++++++++++++++++++++- crates/gpu/src/host/handoff.rs | 6 +- crates/gpu/src/site.rs | 4 +- tools/model-tax.tsv | 2 +- 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 3724b0cab..edc2326d7 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -1147,6 +1147,7 @@ impl CardLeg { act_x, h, act_h, + protection, .. } = bufs; let scratch = Scratch { @@ -1154,6 +1155,7 @@ impl CardLeg { act_x, h, act_h, + protection, }; self.launch( gpu, @@ -1323,7 +1325,8 @@ impl CardLeg { if let Some(p) = protection { p.kernels.normalize( gpu, - (h, run.normed, &mut p.scales), + // Gate/up already activated h; None leaves the second operand unread. + (h, &*down, &mut p.scales), &selected, ff, None, @@ -2135,6 +2138,104 @@ pub(crate) mod tests { } } + /// Prequantized inputs preserve the loaded protection policy and changed-input replay. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_prequantized_leg_preserves_protection_and_replay() { + let gpu = Gpu::new().expect("a GPU"); + let stream = gpu.stream(); + let leg = CardLeg::load(&gpu).expect("the leg"); + let ty = GgmlType::Q5_K; + let weights = weights(&gpu, SHAPE, &[ty]); + let names = [ + stack_name("gate", ty), + stack_name("up", ty), + stack_name("down", ty), + ]; + let stacks = + LegStacks::of(&weights, [&names[0], &names[1], &names[2]]).expect("the stacks"); + let mut ordinary = LegBufs::new(stream, SHAPE).expect("ordinary buffers"); + let mut quantized = LegBufs::new(stream, SHAPE).expect("quantized buffers"); + let mut host = vec![0.0f32; CAP * N]; + let mut normed = DeviceBuffer::zeroed(stream, host.len()).expect("normed rows"); + let mut act = Q8Act::with_tier_cols(stream, CAP, N).expect("quantized input"); + let sel = DeviceBuffer::from_host(stream, &SEL).expect("places"); + let sentinel = vec![SENT; CAP * SPC * N]; + let mut expected = DeviceBuffer::from_host(stream, &sentinel).expect("ordinary output"); + let mut actual = DeviceBuffer::from_host(stream, &sentinel).expect("quantized output"); + for policy in [ExpertInput::Direct, ExpertInput::AmaxF16] { + ordinary + .set_expert_input(&gpu, policy) + .expect("ordinary policy"); + quantized + .set_expert_input(&gpu, policy) + .expect("quantized policy"); + for m in [CAP, CAP - 1] { + let graph = gpu + .capture(|_| { + gpu.enqueue_quantize_q8_1_cols(&normed, &mut act, m, 0)?; + leg.enqueue_rows_quantized( + &gpu, + &mut quantized, + &QuantRun { + layer: 0, + stacks: &stacks, + act: &act, + m, + sel: &sel, + n_card: N_CARD, + rule: Act::SiluMul, + }, + &mut actual, + ) + }) + .expect("capture quantized input and protected leg"); + let amplitudes = match policy { + ExpertInput::Direct => [1.0f32, 2.0], + ExpertInput::AmaxF16 => [1.0f32, 8192.0], + }; + for amplitude in amplitudes { + for (i, value) in host.iter_mut().enumerate() { + *value = (((i * 2654435761usize) % 2001) as f32 / 1000.0 - 1.0) * amplitude; + } + normed.copy_from_host(stream, &host).expect("changed input"); + expected + .copy_from_host(stream, &sentinel) + .expect("reference canaries"); + actual + .copy_from_host(stream, &sentinel) + .expect("replay canaries"); + leg.enqueue_rows( + &gpu, + &mut ordinary, + &LegRun { + layer: 0, + stacks: &stacks, + normed: &normed, + m, + sel: &sel, + n_card: N_CARD, + rule: Act::SiluMul, + }, + &mut expected, + ) + .expect("ordinary input with the same policy"); + graph.launch(stream).expect("changed-input replay"); + let want = expected.to_host_vec(stream).expect("ordinary rows"); + let got = actual.to_host_vec(stream).expect("quantized rows"); + assert!(gpu.take_fault().expect("fault word").is_none()); + let active = m * SPC * N; + let case = format!("{policy:?}, {m} tokens, amplitude {amplitude}"); + assert_hand_rows(&case, &got[..active], &want[..active]); + assert!( + got[active..].iter().all(|&value| value == SENT), + "{case}: tail" + ); + } + } + } + } + /// A gate and up of `ty` beside a Q4_K down, a combination the qwen35moe /// program's arms admit ([`accepts::leg_arms`]): the leg's rows are the /// hand launches' bit for bit (the table's gate·up entry, the quantizer diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 84f35cdc5..f7217b35d 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -1371,11 +1371,7 @@ impl HandoffKernels { ) -> Result<(), GpuError> { let (lay, tl) = (target.layout, ttarget.layout); let slots = check_tier(&lay, &tl)?; - let what = match slots { - HandoffSlots::Four => "ds41_ffn_handoff_4_tier", - HandoffSlots::Six => "ffn_handoff_6_tier", - _ => "ds41_ffn_handoff_8_tier", - }; + let what = "F32 tier handoff"; let n = lay.hidden; let grid = launch_u32(what, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; let cfg = LaunchConfig1D::new(grid, HANDOFF_THREADS, 0); diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 2b99d5fcc..b17811be7 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -1685,8 +1685,8 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, SiteTy, SwigluLaunch, WideActs, WideForms, file_site, gemm_weight, kgemv_order, - swiglu_launch, + Form, GemmWeight, Order, SiteTy, SwigluLaunch, WideActs, WideForms, file_site, gemm_weight, + kgemv_order, swiglu_launch, }; use crate::kquant::Act; use gguf::Split; diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index 2ed0b2d4f..bc1dfe604 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -131,7 +131,7 @@ crates/gpu-vision/src/encoder.rs deepseek 2 baseline D 2: media design: VideoRul crates/gpu/src/arch/hybrid.rs nemotron 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu/src/card_slots.rs deepseek 24 PIN(2026-10-21) deepseek: baseline K 20 + 4, the four-slot entry `ds41_ffn_card_acc_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it crates/gpu/src/dspark/head.rs deepseek 3 baseline K 3: rename the kernel entries by what they compute (deferred) -crates/gpu/src/host/handoff.rs deepseek 39 PIN(2026-10-25) deepseek: fixed-slot tier preparation and launch select the existing generated ABI names beside the new six-slot entry; the per-entry module methods retain their pinned names until the deferred common-entry rename +crates/gpu/src/host/handoff.rs deepseek 37 PIN(2026-10-26) deepseek: the tier launcher uses a common F32 error context instead of selecting a family-named label per slot count; existing kernel entry symbols remain ABI-stable crates/hf/src/vocab.rs qwen 4 PIN(2026-10-20) qwen: the qwen35 tokenizer pipeline name and regex; tokenizer programs are not yet described by ModelSpec crates/metal/src/bin/bloomery_serve_metal.rs qwen 3 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string crates/metal/src/bin/generate_metal.rs qwen 2 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string From b9b6e013ebfab5748688be52e9a5d398ac2fc9a3 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:04:10 +0900 Subject: [PATCH 53/85] Use common F32 tier handoff error context --- crates/gpu/src/host/handoff.rs | 6 +----- docs/research/flint-attach.md | 26 ++++++++++++++++++++++++-- tools/model-tax.tsv | 2 +- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index 84f35cdc5..f7217b35d 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -1371,11 +1371,7 @@ impl HandoffKernels { ) -> Result<(), GpuError> { let (lay, tl) = (target.layout, ttarget.layout); let slots = check_tier(&lay, &tl)?; - let what = match slots { - HandoffSlots::Four => "ds41_ffn_handoff_4_tier", - HandoffSlots::Six => "ffn_handoff_6_tier", - _ => "ds41_ffn_handoff_8_tier", - }; + let what = "F32 tier handoff"; let n = lay.hidden; let grid = launch_u32(what, "grid", n.div_ceil(HANDOFF_THREADS as usize))?; let cfg = LaunchConfig1D::new(grid, HANDOFF_THREADS, 0); diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 7cf7dd5aa..8cf68f0c2 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,30 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. This component tree additionally has a header-only reader and shared tier-handoff +work, but no Flint body, app session or serve seat. Primitive GPU evidence is not Flint model admission. + +## Six-slot handoff closure checks — 2026-10-26 + +The existing #781 branch was ordinarily merged with main +`15778e2cb2dd7af428d2dd14259e6b0ea8ca6b89`; its published history was not rewritten. +On source `9bfa55adc257f7e6aae4952aca06246eddaeef70`, the owning DS41 library ran on +Hikari's RTX 3090 (GPU0): **38 passed, 0 failed, 0 ignored**, in 28.75 seconds. +`hw_f32_tier_handoffs_conserve_consumed_routes` exercised four, six and eight routes, +including partial widths, invalid IDs, poisoned unconsumed rows and changed-input graph replay. +The following host-only changes format the guard and replace a redundant slot-dependent +error label with the common `F32 tier handoff` context. The measured common-name census +falls from 39 to 37; the ABI-stable generated kernel names and dispatch stay unchanged. +The earlier GPU run is not relabeled as a run of these later commits. + +The original remote request timed out while its child continued; a separate read-only +collector recovered the original successful log rather than replaying the run. +Original-request credential cleanup remains unconfirmed and is not implied by test success. +Raw evidence is retained outside Git under +`/home/jmtai04/flint-completion-4dadee0-20261025T205648Z/evidence/closure-20261026/`; +local receipts are under `flint-closure-20261026` in the contribution workspace. +Resource-table admission still requires actual measurements for the new entry. +Accessible cards establish only cc8.6, not physical cc12.1; do not synthesize the latter pins. ## Host reader status diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index 96a9d889c..cb3b46303 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -131,7 +131,7 @@ crates/gpu-vision/src/encoder.rs deepseek 2 baseline D 2: media design: VideoRul crates/gpu/src/arch/hybrid.rs nemotron 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu/src/card_slots.rs deepseek 24 PIN(2026-10-21) deepseek: baseline K 20 + 4, the four-slot entry `ds41_ffn_card_acc_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it crates/gpu/src/dspark/head.rs deepseek 3 baseline K 3: rename the kernel entries by what they compute (deferred) -crates/gpu/src/host/handoff.rs deepseek 38 PIN(2026-10-24) deepseek: baseline K 31 + 4 + 3, the four-slot tier entry `ds41_ffn_handoff_4_tier` (MiniMax-M3.1's bp) beside the four-slot entry `ds41_ffn_handoff_4` (MiniMax-M3 routes 4 a token): one more copy of the per-count entry, since a `#[kernel]` entry cannot be generic over the slot count; the deferred rename of the entries by what they compute absorbs it +crates/gpu/src/host/handoff.rs deepseek 37 PIN(2026-10-26) deepseek: the tier launcher uses a common F32 error context instead of selecting a family-named label per slot count; existing kernel entry symbols remain ABI-stable crates/hf/src/vocab.rs qwen 4 PIN(2026-10-20) qwen: the qwen35 tokenizer pipeline name and regex; tokenizer programs are not yet described by ModelSpec crates/metal/src/bin/bloomery_serve_metal.rs qwen 3 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string crates/metal/src/bin/generate_metal.rs qwen 2 PIN(2026-10-23) qwen: Metal arch dispatch by program: the Metal bins open the one Metal architecture's session by name until crates/metal/src/arch picks a session by the file's architecture string From 160efbe521d683b43b617ac4a077eb6a6512c1ea Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:06:27 +0900 Subject: [PATCH 54/85] Finish current-main fixed-reserve caller migration --- crates/gpu-gates/src/bin/gate_mimo2_serve.rs | 5 ++--- crates/gpu/src/host/leg_tier.rs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs index da2c6271b..2245c05d3 100644 --- a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs +++ b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs @@ -892,9 +892,8 @@ mod gate { let mut inputs = PlanInputs::describe(&split)?; inputs.room = (PLAN_ROOM, HostRead::Given); let place = unset_place(cards).on_host()?; - let batch = (!place.tier_cards().is_empty()) - .then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)) - .transpose()?; + let batch = + (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)); let machine = place.machine(None, batch)?(inputs.model.layers); let slots = NonZeroUsize::new(usize::try_from(DEFAULT_SLOTS)?).ok_or("no default slot")?; let plan = diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index b7342d744..32ec41594 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -31,7 +31,7 @@ use model::arch::models::{Ffn, LayerSpec, Moe}; use model::arch::names::routed_stacks; use model::moe::HostLayer; use model::placement::accepts::{self, LegArms, Program}; -use model::placement::kernels::{self, DownAct, DownForms}; +use model::placement::kernels::{self, DownForms}; use model::placement::{ModelTensors, Plan}; use model::r8file::R8Source; From cb77a4788116c00a02122a56f47efff97c3ba47f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:10:20 +0900 Subject: [PATCH 55/85] Iterate planted router factors without indexed lint debt --- crates/gpu-gates/src/bin/gate_flint_router.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_flint_router.rs b/crates/gpu-gates/src/bin/gate_flint_router.rs index c6052fd15..f339bf6cf 100644 --- a/crates/gpu-gates/src/bin/gate_flint_router.rs +++ b/crates/gpu-gates/src/bin/gate_flint_router.rs @@ -166,7 +166,7 @@ mod gate { }; let clean = cx.gpu.take_fault()?.is_none(); let mut expected = true; - for t in 0..TOKENS { + for (t, &factor) in factors.iter().enumerate() { let scores = &single.scores[t * EXPERTS..(t + 1) * EXPERTS]; let keys: Vec = scores .iter() @@ -180,7 +180,7 @@ mod gate { .iter() .enumerate() .map(|(e, &logit)| { - let p = sigmoid(logit * factors[t]); + let p = sigmoid(logit * factor); bias.map_or(p, |b| p + b[e]) }) .collect(); From c26025b1fb9fad1af399401d27cf709005736ba0 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:29:37 +0900 Subject: [PATCH 56/85] Select actual Q8Act tier scratch also for empty tiers --- crates/gpu/src/card_experts.rs | 23 ++++++++++++++++ crates/model/src/arch/mimo2/place.rs | 10 ------- crates/placement/src/placement.rs | 27 +++++++++++-------- crates/placement/src/placement/workstation.rs | 18 ++++++++----- docs/research/flint-attach.md | 22 +++++++++++++-- 5 files changed, 70 insertions(+), 30 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index eb22b78d7..8946499a6 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -1198,6 +1198,29 @@ mod tests { } assert!(gpu.take_fault().expect("the fault word").is_none()); } + let set = TierSet::new(0..1, EXPERTS, vec![Vec::new()]).expect("an empty tier"); + let shape = FAMILY.shape(N, FF, 2); + let idle = LegTier::new( + &gpu, + &layers(1), + &set, + &Weights::default(), + shape, + FAMILY.program, + tier_act, + ) + .expect("a configured tier need not hold an expert"); + assert_eq!( + idle.block_bytes(), + kernels::leg_bytes( + shape.cap as u64, + USED as u64, + N as u64, + FF as u64, + DownForms::NONE, + ) as usize, + "an empty tier has no down-activation allocation", + ); let triple = [IQ3_S, Q5_K, Q8_0]; let program = Program::Qwen35Body; let w = weights(&gpu, &[triple]); diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 14ad7aa30..1928f6fe2 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -266,16 +266,6 @@ mod tests { let inputs = PlanInputs::describe(&split).expect("the synthetic header describes"); let _ = std::fs::remove_file(&path); let hp = &inputs.hp; - assert_eq!( - tier_batch(hp, COLS), - workstation::tier_leg_batch_bytes( - hp.n_embd as u64, - hp.expert_ff as u64, - hp.n_used as u64, - TIER_CHUNK, - COLS, - ) - ); let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS)); let plain = workstation::plan_gate(hp.n_trunk); diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 2c09b7f87..32f9c9c8a 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -6453,13 +6453,20 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, try_tier_leg_batch_bytes_of}; let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); - let batch = try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both).expect("tier bytes"); - let leg = batch.leg.expect("a tier leg descriptor"); + let mixed = try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both).expect("tier bytes"); let rule = routed_rule(Program::Qwen35Body); - for (downs, first) in [ - ([Q5_K, Q8_0], DownAct::Q8Act), - ([Q8_0, Q5_K], DownAct::Q8Blocks32), + for (downs, first, batch) in [ + ([Q5_K, Q8_0], DownAct::Q8Act, mixed), + ([Q8_0, Q5_K], DownAct::Q8Blocks32, mixed), + ( + [Q5_K; 2], + DownAct::Q8Act, + workstation::tier_leg_batch_bytes(256, 256, 1, 2, 3), + ), ] { + let scratch = |forms| crate::kernels::leg_bytes(2, 1, 256, 256, forms); + let selected = + DownForms::of(first).with(crate::kernels::down(downs[1]).expect("second down").1); let mut model = legged(&[[Q5_K, Q5_K, downs[0]], [Q5_K, Q5_K, downs[1]]]); model.experts = 1; model.experts_used = 1; @@ -6480,15 +6487,13 @@ mod tests { .map(|t| t.file_bytes / model.experts) .sum::() }; - let one = - batch.staging + leg.bytes(DownForms::of(first)).expect("first scratch") + expert(0); - let two = - batch.staging + leg.bytes(both).expect("mixed scratch") + expert(0) + expert(1); + let one = batch.staging + scratch(DownForms::of(first)) + expert(0); + let two = batch.staging + scratch(selected) + expert(0) + expert(1); for (room, counts, forms) in [ (one - 1, [0, 0], DownForms::NONE), (one, [1, 0], DownForms::of(first)), (two - 1, [1, 0], DownForms::of(first)), - (two, [1, 1], both), + (two, [1, 1], selected), ] { let mut tier = bytes_card("tier", room, 0..0); tier.reserves @@ -6506,7 +6511,7 @@ mod tests { .expect("the joint plan"); assert_eq!(plan.n_l, [0, 0]); assert_eq!(plan.tier_n_l[0], counts, "{downs:?}, {room}"); - let selected = batch.staging + leg.bytes(forms).expect("selected scratch"); + let selected = batch.staging + scratch(forms); assert_eq!( plan.card_reserve(1, TIER_BATCH_RESERVE) .expect("selected reserve"), diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 7df753d30..976e3e82c 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -553,7 +553,10 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T } } -/// [`tier_batch_bytes`] for a fixed common-leg reserve in the 128-value form. +/// [`tier_batch_bytes`] for a candidate-selected common leg in the 128-value form. +/// +/// # Panics +/// Panics on invalid geometry or overflowing byte counts. #[must_use] pub fn tier_leg_batch_bytes( n_embd: u64, @@ -562,12 +565,13 @@ pub fn tier_leg_batch_bytes( chunk: u64, cols: u64, ) -> TierBatchBytes { - TierBatchBytes { - staging: tier_batch_staging_bytes(n_embd, n_used, cols), - scratch: tier_chunk_scratch_bytes(n_embd, ff, n_used, chunk), - host: tier_batch_host_bytes(n_embd, n_used, cols), - leg: None, - } + tier_leg_batch_bytes_of( + n_embd, + ff, + n_used, + (chunk, cols), + DownForms::of(DownAct::Q8Act), + ) } /// The common leg's buffers in `downs`, with the same checked geometry and diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 9ed1cc9e7..6dcce1886 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,8 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. This component tree has a header-only reader and selected common-leg work, +not a Flint body, app session or serve seat. Primitive evidence below is not model admission. ## Host reader status @@ -95,6 +95,24 @@ Host-only projections intentionally have no card totals: fixed reserves remain o Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. +Follow-up review found a real empty-tier mismatch: the Q8Act-only common constructor still +advertised fixed scratch while `LegTier` selected no down form for zero resident experts. +The expanded public planner regression failed before the fix: reserve24868 versus the +selected allocation20692. `tier_leg_batch_bytes` now produces the same candidate-selected +descriptor as the explicit-form constructor, preserving its infallible signature. +Qwen3/Qwen35, MiMo and Muse therefore price their actual selected union, including empty; +MiniMax, GLM5Next and Solar already use selected constructors. Separate tile/route-based +fixed-reserve paths remain unchanged. No reserve check is weakened and no unused Q8Act +buffer is allocated to hide the mismatch. The same regression passes after the repair. +An old MiMo helper-equality assertion was removed; its actual plan/reserve/refusal checks remain. +The real mixed-wrapper hardware case also checks allocation of a configured empty tier. + +Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: +GPU library418 passed plus10 held passed, verbs1 passed, and all1657 canonical entries +identical to main15778e2c (4bins/8carriers, non-JIT rows equal). +The newly repaired constructor and extended hardware case require their own source-specific +execution; the PR was returned to Draft on the finding rather than relabel the earlier pass. + A host-only public planner consumer exercised the one-byte boundaries before the first resident layer and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an From f11d5b15017f3fe8d6a5f4bb9cb30a93b212da4a Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:29:37 +0900 Subject: [PATCH 57/85] Pin measured six-slot handoff resources on cc8.6 --- docs/research/flint-attach.md | 15 +++++++++++++-- tools/ref/ptx-shapes.tsv | 4 ++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 8cf68f0c2..f69a95283 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -24,8 +24,19 @@ Original-request credential cleanup remains unconfirmed and is not implied by te Raw evidence is retained outside Git under `/home/jmtai04/flint-completion-4dadee0-20261025T205648Z/evidence/closure-20261026/`; local receipts are under `flint-closure-20261026` in the contribution workspace. -Resource-table admission still requires actual measurements for the new entry. -Accessible cards establish only cc8.6, not physical cc12.1; do not synthesize the latter pins. +The actual `generate_ds41` carrier at `b9b6e013ebfab5748688be52e9a5d398ac2fc9a3` +subsequently measured `ffn_handoff_6_tier` on sm_86 / physical cc8.6: +ptxas13.3.33 spill0, driver-JIT local0, both register counts26 (driver595.91.07, +JIT CUDA13.2, Rust1.99.1 / cuda-oxide9df9e7ce / repository patch31ab6a1d3f58). +Only that measured row is added. Its normalized entry digest is +`a176a9a5846778e40ba99fc2be637d90` (decl1, 163 lines). +This resource request also exceeded the gateway window; a read-only collector recovered +the original scan, with no remaining owned process and GPU1 active/healthy. +The original script recorded exit23 after producing the complete successful scan; +a post-timeout stdout write failure is the explanation inferred from its final curl step, +not an invented exit0 for the request. The original credential cleanup remains unconfirmed. +Accessible cards establish only cc8.6. The other three sm_86/sm_121/sm_121a + physical +cc12.1 rows remain unmeasured and absent; #781 stays Draft until that gate is dispositioned. ## Host reader status diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 07613d074..b038d2937 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -74,6 +74,10 @@ generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_4 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_4_tier 0 0 +# PIN(2026-10-26): ffn_handoff_6_tier measured from generate_ds41 at b9b6e013 with +# Rust1.99.1 / cuda-oxide9df9e7ce + repository patch31ab6a1d3f58; sm_86 ptxas13.3.33, +# RTX3090 cc8.6 driver595.91.07 JIT CUDA13.2: spill0 / jit_local0 (both register counts26). +generate_ds41 ffn_handoff_6_tier 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 generate_ds41 ds41_ffn_handoff_9 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 From ff54af24fbef5470895a3d697fd67e4bc6c9cf07 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:36:54 +0900 Subject: [PATCH 58/85] Record closure cutover and measured head decomposition --- docs/research/flint-attach.md | 79 ++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 9126e4944..8ad5531b9 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,9 +2,9 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The integration based on **`e0c82e1779a8e754c53f4442800e6c3d61271a48`** -adds an actual `gpu-flint` body, application session and shared HTTP seat, not just architecture -admission. Device execution, GPU numerical acceptance and performance remain unverified. +that snapshot. The integration adds an actual `gpu-flint` body, application session and shared +HTTP seat, not just architecture admission. Sections 10–11 retain source-specific historical +execution evidence; section 12 records the current existing-PR cutover and remaining acceptance. ## Host reader status @@ -140,9 +140,11 @@ budget regression exercises this distinction without a GPU, including OS reserve The current-main cutover keeps reserve-only draft holders, per-card budgets and each family's activation contract. `LegTier` receives its activation converter at load; -selected `Program` arms replace the redundant family arm hook. Qwen, Muse, MiMo and -MiniMax sizing callers propagate checked tier descriptors rather than bypassing -their validation. Share-budget floors use the same selected reserve calculation. +selected `Program` arms replace the redundant family arm hook. Common-leg constructors, +including the infallible Q8Act constructor used by Qwen3/Qwen35, MiMo and Muse, provide +candidate-selected descriptors. Existing metadata API signatures remain infallible where +appropriate; separate tile/route-based fixed-reserve paths are unchanged. +Share-budget floors use the same selected reserve calculation. `TierLegScratch::input_scales` names an optional physical f32 scale per chunk slot. Its checked byte formula includes `4 * chunk * n_used` when enabled, including an empty selected-down union; legacy descriptors leave it disabled. @@ -669,10 +671,10 @@ residency/NVMe, streamed HTTP and spontaneous tool selection are not established The first linear integration candidate, `4dadee0195fb652608be59ffecc033a4643ad1eb`, starts at main `43073091be21e48cda40d79fe6dae1dfcfd643d5`. The follow-up replay -starts at main `af74ea46b7f3e47692c2afae09eb64fdddf71461`; neither replay replaces -the published #799 head. Fixed-source numerical comparisons keep the original production -and first-candidate images separate from the later replay, and receipts name each image. -Section 10 remains historical evidence, not acceptance of either integration candidate. +starts at main `af74ea46b7f3e47692c2afae09eb64fdddf71461`. At that stage neither replay +replaced the published #799 head; section 12 describes the later history-preserving cutover. +Fixed-source comparisons keep the original production and candidate images separate, and +receipts name each image. Section 10 is not acceptance of either replay or the later cutover. The placement integration keeps both the cached upload `Footprint` and candidate-selected tier scratch/reserves. No compiler pin, numerical band or model admission was widened. @@ -809,6 +811,63 @@ and executed hardware case; compiled/ignored tests are not such evidence. Preser existing workloads unless their owner authorizes a separate window. Physical cc12.1 resource values must be measured, never inferred from an sm_86 build or cc8.6 run. +## 12. Existing-PR cutover and same-input head probe — 2026-10-26 + +The existing #799 branch now integrates fixed main +`15778e2cb2dd7af428d2dd14259e6b0ea8ca6b89` by ordinary merges, preserving both its +original `ebce5badb6f375c74930ff26a4eb50e7e74ce97d` ancestry and the previous D6 replay. +It also preserves the component histories of #781, #782 and #783. This is not a replacement +PR or a force-push. The cutover adopts main's Rust1.99.1 and pinned cuda-oxide9df9e7ce / +repository backend patch31ab6a1d3f58, not a contribution-specific compiler override. +The older nightly/87bc runtime arrays remain evidence of their own source/toolchain only. + +Current-main's prequantized leg input and metadata-only `Common` remain separate. +Both ordinary and prequantized entry points retain the loaded `AmaxF16` protection state. +The already-activated card leg normalizes with `rule=None`: its second device ABI operand +is unread, so it borrows the distinct existing down buffer rather than allocating or copying +a fake input. A hardware regression compares ordinary input with prequantized captured replay, +including changed amplitudes, Direct/AmaxF16, full/partial batches and skipped/tail canaries. +Compilation is not evidence that this new hardware case has executed. + +The cutover's production/test source `cb77a4788116c00a02122a56f47efff97c3ba47f` +passed native all-target Clippy and formatting, with three warning lines against ratchet three. +Its host library run passed chain23, GGUF67, placement206 and model563 (five model tests +ignored); zero tests were filtered by the historical known-base-test skip. +Common-name and unsafe ownership checks also passed after the contract reconciliation. +These checks are distinct from current-head NVIDIA/model acceptance. + +### Measured head/upstream separation, not a precision fix + +A throwaway probe used the preserved mainline CPU backend +`a37cc46104eb50900d5ab4a9ba02a0abdac039db`, four threads, and the original complete +Q8_0 output matrix (2048 × 150144). It evaluated both CPU and GPU final-normalization +vectors from the matched-boundary v2 trajectory. Its CPU-input controls reproduce the +original complete 150144-logit rows bit for bit at both calls; this validates the cross-input +comparison rather than a dequantized/reduced-matrix approximation. + +Here `HC`/`HG` are the CPU/GPU head implementations and `xC`/`xG` their saved norm inputs. +Each cell is the first named token's logit minus the second's: + +| Predicted position / token pair | HC(xC) | HC(xG) | HG(xG) | HG(xG) − HC(xG) | HC(xG) − HC(xC) | +|---|---:|---:|---:|---:|---:| +| 38 / 86671 − 55591 | +0.164879680 | −0.166242123 | −0.142640829 | +0.023601294 | −0.331121802 | +| 96 / 26007 − 29263 | +0.238318443 | +0.005065918 | −0.018128157 | −0.023194075 | −0.233252525 | + +At position96, the CPU head on the GPU norm selects26007, whereas the actual GPU head +selects29263. At position38 the CPU head on the GPU norm still selects55591. A head-only +change therefore cannot resolve the whole trajectory. The columns separate head-implementation +and upstream contributions; they do not identify the first upstream operation or attribute all +head difference to one quantization step. No production precision, tolerance, tie rule or oracle +criterion was changed, and no existing GPU arm was replayed for this CPU-only probe. + +Raw inputs, complete output rows, bit-identity controls, exact margins and reproduction source +are retained under the contribution workspace's `flint-closure-20261026/head-probe/`. +Original v2 GPU arrays remain at their earlier remote locations; they were not overwritten +or reattributed to this cutover. Keep #799 Draft pending exact-source runtime acceptance, +the Korean numerical/oracle decision, genuine physical cc12.1 resource measurements and +remaining affected common/residency/tier/NVMe coverage. Component review readiness is not +whole-model admission, merge, issue closure, ROCm support or a performance result. + [needs]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/models/src/need.rs#L505-L531 [stems]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/model/src/arch/stems.rs#L61-L125 From 082cf7ffac3bb5b5d574b489062732735985feeb Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:42:42 +0900 Subject: [PATCH 59/85] Retain tier activation fixture helper across integration --- crates/gpu/src/card_experts.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 50b5e2c0f..27a6b2f72 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -935,6 +935,12 @@ mod tests { ] } + fn tier_act(moe: &Moe) -> Result { + Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit(), + }) + } + fn weights(gpu: &Gpu, triples: &[[GgmlType; 3]]) -> Weights { let mut decls = Vec::new(); let mut data = Vec::new(); From 4d841c94aca30745ed267e0cbd9b2edd71d16540 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:03:30 +0900 Subject: [PATCH 60/85] Record repaired empty-tier consumer hardware proof --- docs/research/flint-attach.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 6dcce1886..218196059 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -110,13 +110,19 @@ The real mixed-wrapper hardware case also checks allocation of a configured empt Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: GPU library418 passed plus10 held passed, verbs1 passed, and all1657 canonical entries identical to main15778e2c (4bins/8carriers, non-JIT rows equal). -The newly repaired constructor and extended hardware case require their own source-specific -execution; the PR was returned to Draft on the finding rather than relabel the earlier pass. - -A host-only public planner consumer exercised the one-byte boundaries before the first resident layer -and before adding the other down form, plus descriptor overflow refusal. This proves planning behavior, -not device allocation or expert arithmetic. The existing CUDA leg, batch and tier gates still need an -exact-head NVIDIA run. Flint's `Program` admission and body/app/seat remain absent. +The repair at `c26025b1fb9fad1af399401d27cf709005736ba0` then executed the extended +real CUDA mixed/empty-tier consumer:1 passed,0 failed,0 ignored,427 filtered,24.18s. +Its fresh4bin/8carrier canonical comparison again found all1657entries identical to main15778e2c, +with non-JIT rows equal. Native formatting/Clippy passed (warning3=ratchet3); model557 and +placement202 passed,0 failed,5 model tests ignored,0 filtered. +Request75d4e06e-f3fb-4001-bcc4-e188c169edf5 exited0, credential cleanup unconfirmed; +GPU1 remained active/healthy and was not stopped. The PR returned to review-ready only after +this repair proof. Later documentation-only commits do not count as another hardware run. + +The host-only public planner consumer covers one-byte boundaries before the first resident layer +and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer +are not a whole-model oracle or every affected model/residency gate; canonical equality does not +waive changed host/allocation consumers. Flint's `Program` admission and body/app/seat remain absent. From 0b055bb378126f7f97de1ab636c00dcc06f9e82b Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:03:30 +0900 Subject: [PATCH 61/85] Document source-specific ordinary IQ4 hardware proof --- docs/research/flint-attach.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index f58921e84..c73e7787d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,8 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. The current tree additionally has a **header-only reader**; it still has no Flint -body, app session or serve seat. No GPU, inference, performance or weight-content validation is claimed. +that snapshot. This component tree has a header-only reader and the ordinary IQ4_XS shared site, +not a Flint body, app session or serve seat. Primitive GPU evidence below is not model admission. ## Host reader status @@ -80,14 +80,27 @@ those same sites. IQ4_XS is not relabeled as a routed expert or requantized. the ordinary `SharedExpert` layout, unequal matrix widths, one/eight-column block execution and the wide-prompt transition/captured replay. Wide scratch holds only the q8_1 input and down forms these sites read. Its nonnegative constant-weight -case derives a scalar floating-operation bound; it is not a whole-model accuracy -pin. Exact-head hardware execution is required before claiming arithmetic or -capture/replay support. +case derives a scalar floating-operation bound; it is not a whole-model accuracy pin. Program-specific admission remains unchanged: in particular the legacy Qwen projection/shared dispatch does not gain this site automatically. This common site contract adds no model admission or kernel. +On 2026-10-26, source `203f5125a3c48a5b700c6f43318cfcb9c77ff169` executed the owning +DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed,0 ignored, +28.61s, including the named IQ4 consumer above. Rust1.99.1 / pinned cuda-oxide9df9e7ce +with repository patch31ab6a1d3f58 and LLVM21.1.8 were used. +Its canonical comparison to main15778e2c covered4bins/8carriers/1657entries: all identical, +non-JIT resource rows equal; no JIT-resource comparison is claimed. +All-target native Clippy/formatting and current-head GitHub T0/lint passed. +The original execution request5a907e86-7ac6-4219-a3b6-e3cf1de84e03 exited0 but credential +cleanup remains unconfirmed; test success does not certify credential revocation. +Raw logs and canonical manifests remain in the contribution workspace's +`flint-closure-20261026` and its matching remote evidence directory. GPU1 was not stopped. +Later documentation-only commits are not a new GPU execution. Wider affected real-model, +placement and residency coverage remains explicit for maintainer review; #782 is review-ready, +not merged, and this does not establish a Flint oracle, physical cc12.1, ROCm or performance. + ## 1. Sources and observed artifact From 76f4b122d96ccfa7fdc38ae8795e85d92ab8998c Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:48:07 +0900 Subject: [PATCH 62/85] Unify selected tier accounting and checked reserve contracts --- crates/gpu-gates/src/bin/gate_mimo2_serve.rs | 5 +- .../src/bin/gate_musespark_prefill.rs | 2 +- .../gpu-gates/src/bin/shared/mimo2_twocard.rs | 2 +- .../src/bin/shared/minimaxm3_seat.rs | 4 +- .../src/bin/shared/qwen3moe_place.rs | 25 ++++- .../src/bin/shared/serve_seats/mimo2.rs | 4 +- .../src/bin/shared/serve_seats/minimaxm3.rs | 3 +- .../src/bin/shared/serve_seats/musespark.rs | 4 +- crates/gpu-mimo2/src/tier.rs | 5 +- crates/gpu-minimaxm3/src/tier.rs | 5 +- crates/gpu/src/card_experts.rs | 8 +- crates/gpu/src/card_leg.rs | 24 +++-- crates/gpu/src/host/leg_tier.rs | 8 +- crates/model/src/arch/mimo2/place.rs | 9 +- crates/model/src/arch/minimaxm3/place.rs | 40 ++------ crates/model/src/arch/musespark/place.rs | 3 +- crates/model/src/arch/qwen35moe/place.rs | 11 ++- crates/model/src/arch/qwen3moe/place.rs | 3 +- crates/model/src/arch/solar2/place.rs | 64 +++++------- crates/model/tests/mimo2_meta.rs | 2 +- crates/model/tests/musespark_meta.rs | 2 +- crates/model/tests/qwen3_bp_plan.rs | 9 +- crates/placement/src/placement.rs | 91 ++++++++++------- crates/placement/src/placement/workstation.rs | 98 ++++++++++++------- docs/research/flint-attach.md | 32 +++--- 25 files changed, 251 insertions(+), 212 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs index 2245c05d3..da2c6271b 100644 --- a/crates/gpu-gates/src/bin/gate_mimo2_serve.rs +++ b/crates/gpu-gates/src/bin/gate_mimo2_serve.rs @@ -892,8 +892,9 @@ mod gate { let mut inputs = PlanInputs::describe(&split)?; inputs.room = (PLAN_ROOM, HostRead::Given); let place = unset_place(cards).on_host()?; - let batch = - (!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)); + let batch = (!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(&inputs)) + .transpose()?; let machine = place.machine(None, batch)?(inputs.model.layers); let slots = NonZeroUsize::new(usize::try_from(DEFAULT_SLOTS)?).ok_or("no default slot")?; let plan = diff --git a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs index 6674f694b..5137e5828 100644 --- a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs @@ -530,7 +530,7 @@ mod gate { tier_now: Tier, ) -> Result<(Model, bool, Vec), GateError> { let inputs = PlanInputs::read(&open_split(Arch::MuseSpark, RECIPE)?)?; - let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp))?; + let mut two = lcpp_e2e::Arm::Bp.machine(hp.n_layer, place::tier_batch(hp)?)?; place::stage_placed(&mut two)?; let ctx = u64::try_from(ctx)?; // The stage card alone, the reference's machine: the budget reads its unbudgeted plan, which a plan of the two diff --git a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs index 3b8425314..cbf918642 100644 --- a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs +++ b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs @@ -111,7 +111,7 @@ pub fn run(path: &Path, levers: &bloomery_levers::Levers) -> Result Result q3::tier_batch(&i.hp), Inputs::Qwen35(i) => q35::leg_tier_batch(&i.spec), - }; + }?; let mut machine = workstation::plan_tiers(layers, *stage, &[*tier], None, batch)?; q3::stage_placed(&mut machine, arena); machine @@ -2099,7 +2099,7 @@ pub(super) mod tests { /// A dense qwen35 file (no router) plans through [`PlaceQ3::qwen35_on`]: its machine counts /// the whole load's ubatch arena, as the load sizes it, not a placed arena, which a file with - /// no router refuses; and the plan answers at the load's context. + /// no router refuses; an expert tier is refused by name through the same consumer. #[allow( dead_code, reason = "`generate_qwen3moe`'s test calls it; every other bin that includes this file leaves it unused" @@ -2137,6 +2137,27 @@ pub(super) mod tests { assert_eq!(workstation::counted_arena_bytes(&q.machine.cards[0]), arena); q.plan(o.ctx, &PlanLevers::default()) .unwrap_or_else(|e| panic!("the dense plan at {}: {e}", o.ctx)); + let tier = DeviceInfo { + ordinal: 1, + name: "NVIDIA RTX 3090".to_owned(), + total_bytes: 25_769_803_776, + free_bytes: 25_000_000_000, + uuid: [2; 16], + pci_bus: "0000:81:00.0".to_owned(), + held_by: None, + integrated: false, + }; + let census = [census.into_iter().next().expect("the stage card"), tier]; + let two = Place::parse("cuda0+1").expect("two card words"); + let refusal = + match PlaceQ3::qwen35_on(&split, two, &census, o, Seqs::ONE, Beside::default()) { + Err(e) => e, + Ok(_) => panic!("a dense model cannot use an expert tier"), + }; + assert!(matches!( + refusal.downcast_ref::(), + Some(PlacementError::Metadata { key, .. }) if key == "Qwen3.6 expert tier" + )); std::fs::remove_file(&path).expect("the header file"); } } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs index cf6fd96d0..1204c8373 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mimo2.rs @@ -502,7 +502,9 @@ impl Tiered for Mimo { /// The tier card's prompt-batch bytes ([`bloomery_gpu_mimo2::tier::tier_batch`]), under a /// placement with one. fn tier_batch(place: Place, inputs: &PlanInputs) -> Result, GateError> { - Ok((!place.tier_cards().is_empty()).then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs))) + Ok((!place.tier_cards().is_empty()) + .then(|| bloomery_gpu_mimo2::tier::tier_batch(inputs)) + .transpose()?) } fn flags(flag: &str, value: &str, out: &mut Flags) -> Result { diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs index 096093e07..92f704bf1 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/minimaxm3.rs @@ -506,7 +506,8 @@ impl Tiered for MiniMax { inputs: &PlanInputs, ) -> Result, GateError> { Ok((!place.tier_cards().is_empty()) - .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs))) + .then(|| bloomery_gpu_minimaxm3::tier::tier_batch(inputs)) + .transpose()?) } /// The tags the model's description names for its think span. diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs index 0d36b7074..650f28a66 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/musespark.rs @@ -462,7 +462,9 @@ impl Tiered for Muse { place: Place, inputs: &PlanInputs, ) -> Result, GateError> { - Ok((!place.tier_cards().is_empty()).then(|| place::tier_batch(&inputs.hp))) + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(&inputs.hp)) + .transpose()?) } /// The ring is kept out of the expert budget where the common rule starts diff --git a/crates/gpu-mimo2/src/tier.rs b/crates/gpu-mimo2/src/tier.rs index f0886e36c..eaded5ed9 100644 --- a/crates/gpu-mimo2/src/tier.rs +++ b/crates/gpu-mimo2/src/tier.rs @@ -34,8 +34,8 @@ use gguf::Split; use model::ModelError; use model::arch::mimo2::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; use model::placement::workstation::TierBatchBytes; +use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::LayerSpec; @@ -56,8 +56,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// The tier's prompt-batch bytes the plan reserves on the tier card and the host for the file /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(&inputs.hp, T_MAX as u64) } diff --git a/crates/gpu-minimaxm3/src/tier.rs b/crates/gpu-minimaxm3/src/tier.rs index ad4f75122..ca916c671 100644 --- a/crates/gpu-minimaxm3/src/tier.rs +++ b/crates/gpu-minimaxm3/src/tier.rs @@ -20,8 +20,8 @@ use gguf::Split; use model::arch::minimaxm3::host as minimax_host; use model::arch::minimaxm3::place::{self, PlanInputs}; use model::moe::HostLayer; -use model::placement::ModelTensors; use model::placement::workstation::TierBatchBytes; +use model::placement::{ModelTensors, PlacementError}; use model::r8file::R8Source; use models::{LayerSpec, Moe}; @@ -38,8 +38,7 @@ pub const CHUNK: usize = place::TIER_CHUNK as usize; /// The tier's prompt-batch bytes the plan reserves on the tier card and the host for the file /// `inputs` describes: [`place::tier_batch`] over blocks of a prompt unit's rows ([`T_MAX`]), the /// columns the load makes the tier's batch legs for. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs) -> Result { place::tier_batch(inputs, T_MAX as u64) } diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 8946499a6..13b6365b0 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -46,7 +46,7 @@ use model::placement::accepts::{self, LegArms, Program}; use model::placement::kernels::{self, DownAct, DownForms}; use crate::card_leg::{ - self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, + self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, STACK_ROLES, }; use crate::card_slots::{CardSlotKernels, Places}; use crate::gemm::Gemm32Kernels; @@ -57,12 +57,6 @@ use crate::ring_gemm::RingGemm; use crate::weights::Weights; use crate::{DeviceTensor, Gpu, GpuError}; -/// The selected program, rather than a union of unrelated launches, admits stacks. -const STACK_ROLES: (&str, &str) = ( - "a resident routed gate or up stack the selected card gate·up arm runs", - "a resident routed down stack the selected card down arm runs", -); - /// What a family fixes of its card experts. #[derive(Clone, Copy)] pub struct CardFamily { diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 1bb6b987c..52b642883 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -514,14 +514,12 @@ struct LegStack<'a> { } impl<'a> LegStack<'a> { - /// The resident stack `name`: a `KQuant` stack of a type `runs` admits in - /// the stack's role, or an NVFP4 stack in its planes where `planes` - /// admits NVFP4; `role` is the refusal's words. + /// The resident stack `name` of a type `runs` admits in its role, + /// including plane-backed NVFP4 when admitted; `role` names a refusal. fn of( w: &'a Weights, name: &'a str, runs: fn(GgmlType) -> bool, - planes: fn(GgmlType) -> bool, role: &'static str, ) -> Result, GpuError> { match w.get(name) { @@ -537,7 +535,7 @@ impl<'a> LegStack<'a> { w: StackW::Q5_0(w), k: *k, }), - Some(DevWeight::Nvfp4Stack { k, .. }) if planes(GgmlType::NVFP4) => Ok(LegStack { + Some(DevWeight::Nvfp4Stack { k, .. }) if runs(GgmlType::NVFP4) => Ok(LegStack { name, ty: GgmlType::NVFP4, w: StackW::Nvfp4(w.nvfp4_stack(WHAT, name)?), @@ -609,6 +607,12 @@ impl<'a> LegStack<'a> { const GATE_UP: &str = "a resident routed gate or up stack of a type the card leg runs (`kernels::card_gate_up_runs`)"; +/// The selected program admits the same resident stack roles on card and tier. +pub(crate) const STACK_ROLES: (&str, &str) = ( + "a resident routed gate or up stack the selected program's gate·up arm runs", + "a resident routed down stack the selected program's down arm runs", +); + /// A layer's three resident routed stacks, the gate and up of one type. #[derive(Clone, Copy)] pub struct LegStacks<'a> { @@ -670,9 +674,9 @@ impl<'a> LegStacks<'a> { ) -> Result, GpuError> { let [gate, up, down] = names; let (gate, up, down) = ( - LegStack::of(w, gate, arms.gate_up, arms.gate_up, roles.0)?, - LegStack::of(w, up, arms.gate_up, arms.gate_up, roles.0)?, - LegStack::of(w, down, arms.down, arms.down, roles.1)?, + LegStack::of(w, gate, arms.gate_up, roles.0)?, + LegStack::of(w, up, arms.gate_up, roles.0)?, + LegStack::of(w, down, arms.down, roles.1)?, ); if up.ty != gate.ty { return Err(GpuError::Tensor { @@ -1492,8 +1496,8 @@ impl<'a> Relu2Stacks<'a> { const DOWN: &str = "a resident routed down stack of a 32-value-block type the card leg runs (`kernels::card_sel32_runs`)"; let [up, down] = names; Ok(Relu2Stacks { - up: LegStack::of(w, up, kernels::card_sel32_runs, |_| false, UP)?, - down: LegStack::of(w, down, kernels::card_sel32_runs, |_| false, DOWN)?, + up: LegStack::of(w, up, kernels::card_sel32_runs, UP)?, + down: LegStack::of(w, down, kernels::card_sel32_runs, DOWN)?, }) } diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index 2eecc718b..c35afc97d 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -40,7 +40,7 @@ use super::run::{HostRun, HostWidths}; use super::tier::{ TierAct, TierBlock, TierCard, TierExperts, TierInput, TierIo, TierOpen, TierSet, TierShape, }; -use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks}; +use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks, STACK_ROLES}; use crate::hybrid::{Hybrid, SlotMap}; use crate::kquant::Act; use crate::model::CardBytes; @@ -53,12 +53,6 @@ const WHAT: &str = "LegTier"; /// What the stage card's side of the tier layers names in its errors. const SIDE: &str = "TierSide"; -/// The selected program's launches admit the same stacks on both cards. -const STACK_ROLES: (&str, &str) = ( - "a resident routed gate or up stack the selected tier gate·up arm runs", - "a resident routed down stack the selected tier down arm runs", -); - /// The tier the stage card's tier entries serve: they hand one tier its /// image and join one tier's rows, so a load holds one tier card /// ([`crate::host::refuse_tier_count`]). diff --git a/crates/model/src/arch/mimo2/place.rs b/crates/model/src/arch/mimo2/place.rs index 1928f6fe2..03e4345ec 100644 --- a/crates/model/src/arch/mimo2/place.rs +++ b/crates/model/src/arch/mimo2/place.rs @@ -119,8 +119,7 @@ pub const TIER_CHUNK: u64 = 16; /// plan's rule carries reads (the rule's down arm is the leg's `Q8Act` downs), and the host's rows /// and places ([`workstation::tier_leg_batch_bytes`]). The plan reserves them on the tier card and /// the host, and the load checks its allocations against them. -#[must_use] -pub fn tier_batch(hp: &Hparams, cols: u64) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams, cols: u64) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.expert_ff as u64, @@ -267,7 +266,11 @@ mod tests { let _ = std::fs::remove_file(&path); let hp = &inputs.hp; let routed = runtime::layer::hosted(&inputs.spec.layers).map_or(0, |run| run.len()); - let tiered = workstation::plan_bp(hp.n_trunk, None, tier_batch(hp, COLS)); + let tiered = workstation::plan_bp( + hp.n_trunk, + None, + tier_batch(hp, COLS).expect("valid tier geometry"), + ); let plain = workstation::plan_gate(hp.n_trunk); assert_eq!( inputs.prompt_batch_bytes(&plain).expect("a plain machine"), diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index 309200f48..e8528fb48 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -22,10 +22,10 @@ use crate::arch::names::routed_stacks; use crate::arch::place_hybrid::{ self, ring_pass_bytes, route_bytes, stream_ring_bytes, tier_places_bytes, wide_bytes, }; -use crate::placement::accepts::planes_leg_down; +use crate::placement::PlacementError; +use crate::placement::accepts::{self, Program}; use crate::placement::kernels::{self, DownAct, DownForms}; use crate::placement::workstation::{self, TierBatchBytes}; -use crate::placement::{PlacementError, Role}; pub use place_hybrid::{KvLayout, PlaceError}; @@ -134,41 +134,17 @@ pub const TIER_CHUNK: u64 = 16; /// (an NVFP4 down's 32-value blocks, a K- or i-quant down's 128-value form) and the host's rows /// and places. The plan reserves them on the tier card and the host, and the load checks its /// allocations against them. -#[must_use] -pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> TierBatchBytes { +pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result { let hp = &inputs.hp; - workstation::tier_leg_batch_bytes_of( + workstation::try_tier_leg_batch_bytes_of( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, (TIER_CHUNK, cols), - leg_downs(inputs), + accepts::leg_down_forms(Program::MiniMaxM3Body, &inputs.model), ) } -/// The forms the card leg's routed downs of `inputs` read: each routed layer's down stack whose -/// type the leg runs ([`planes_leg_down`]), by its kernel table row ([`kernels::down`]); the -/// 128-value form on a file with none. -fn leg_downs(inputs: &PlanInputs) -> DownForms { - let downs = (0..inputs.model.layers) - .map(|l| routed_stacks(l)[2].clone()) - .filter_map(|name| { - inputs - .model - .tensors - .iter() - .find(|t| t.role == Role::RoutedExperts && t.name == name) - }) - .filter(|t| planes_leg_down(t.ty)) - .filter_map(|t| kernels::down(t.ty)) - .fold(DownForms::NONE, |forms, (_, act)| forms.with(act)); - if downs == DownForms::NONE { - DownForms::of(DownAct::Q8Act) - } else { - downs - } -} - /// The slots of each half of the expert stream's ring a streaming plan keeps room for /// ([`stream_bytes`]) [derived: at the ring pass's route GEMMs the stream rule's crossover is about /// 18 columns of a walk, and a walk of the default group (two units of [`PROMPT_ROWS`] rows, four @@ -425,7 +401,11 @@ mod tests { fn a_tier_card_is_planned_with_the_leg_batch() { const COLS: u64 = 3 * TIER_CHUNK; let inputs = described("mm3-place-tier"); - let tiered = workstation::plan_bp(LAYERS, None, tier_batch(&inputs, COLS)); + let tiered = workstation::plan_bp( + LAYERS, + None, + tier_batch(&inputs, COLS).expect("valid tier geometry"), + ); let plain = workstation::plan_gate(LAYERS); let places = 3 * 4 * 256 * 2; assert_eq!(inputs.tier_reserve, Some(places)); diff --git a/crates/model/src/arch/musespark/place.rs b/crates/model/src/arch/musespark/place.rs index e47ddf052..f0900aced 100644 --- a/crates/model/src/arch/musespark/place.rs +++ b/crates/model/src/arch/musespark/place.rs @@ -76,8 +76,7 @@ pub const fn pass_rows_max(n_used: usize) -> usize { /// placed pass's rows ([`pass_rows_max`]): the experts run in the latent, so /// the tier's rows and its staging are the latent's width, through the card /// leg in chunks of [`TIER_CHUNK`]. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { let ff = hp.expert_width(); workstation::tier_leg_batch_bytes( hp.latent as u64, diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index 2f60c4201..7433ccbc3 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1498,11 +1498,12 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// card's staging, one chunk of the leg's scratch and the host's rows, for /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. -#[must_use] -pub fn leg_tier_batch(spec: &ModelSpec) -> TierBatchBytes { - let (expert_ff, n_used) = spec - .routed() - .map_or((0, 0), |m| (u64::from(m.expert_ff), u64::from(m.top_k))); +pub fn leg_tier_batch(spec: &ModelSpec) -> Result { + let moe = spec.routed().ok_or_else(|| PlacementError::Metadata { + key: "Qwen3.6 expert tier".into(), + detail: "a dense model has no routed experts for an expert tier".into(), + })?; + let (expert_ff, n_used) = (u64::from(moe.expert_ff), u64::from(moe.top_k)); workstation::tier_leg_batch_bytes( u64::from(spec.hidden), expert_ff, diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 3c71a02d2..8afc5c3ed 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -45,8 +45,7 @@ pub const TIER_CHUNK: u64 = 8; /// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. -#[must_use] -pub fn tier_batch(hp: &Hparams) -> TierBatchBytes { +pub fn tier_batch(hp: &Hparams) -> Result { workstation::tier_leg_batch_bytes( hp.n_embd as u64, hp.experts.ff as u64, diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 8aaf7f92a..8c011861e 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -176,7 +176,7 @@ mod tests { use super::super::hparams::tests::{LAYERS, keys, tensors}; use super::{PROMPT_ROWS, PlaceError, PlanInputs, prompt_reserve_bytes, tier_batch}; - use crate::arch::synthetic::{V, header_shaped}; + use crate::arch::synthetic::{Ty, V, header_shaped, header_typed}; use crate::placement::{Device, PlacementError, PlanLevers, Violation, workstation}; /// The small header's file, described; each test names its own file (the tests run in @@ -319,9 +319,6 @@ mod tests { /// at the allocation). #[test] fn a_host_only_mixture_keeps_the_exact_prompt_reserve_boundary() { - use crate::placement::{ExpertPart, Role}; - use gguf::GgmlType::{Q5_K, Q8_0}; - let mut kv = keys(); for (key, value) in &mut kv { if matches!(*key, "embedding_length" | "expert_feed_forward_length") { @@ -367,47 +364,28 @@ mod tests { ("tokenizer.ggml.pre", V::Str("solar-open")), ("tokenizer.chat_template", V::Str("{{ messages }}")), ]; - let path = header_shaped("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); + let ts: Vec<_> = ts + .into_iter() + .map(|(name, dims)| { + let ty = if name == "blk.0.ffn_down_exps.weight" { + Ty::Q8_0 + } else if matches!( + name.rsplit('.').nth(1), + Some("ffn_gate_exps" | "ffn_up_exps" | "ffn_down_exps") + ) { + Ty::Q4K + } else { + Ty::F32 + }; + (name, dims, ty) + }) + .collect(); + let path = header_typed("so2-place-mixed-reserve", "solar-open2", &kv, &global, &ts); let split = Split::open(&path).expect("the mixed fixture opens"); - let mut inputs = PlanInputs::describe(&split).expect("valid routed geometry"); + let inputs = PlanInputs::describe(&split).expect("valid routed geometry"); std::fs::remove_file(path).expect("remove the header fixture"); - for t in inputs - .model - .tensors - .iter_mut() - .filter(|t| t.role == Role::RoutedExperts) - { - t.ty = Q5_K; - t.file_bytes = t.dims.iter().product::() / t.ty.blck_size().unwrap() - * t.ty.type_size().unwrap(); - } - let admitted = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) - .expect("the admitted prompt reserve"); - let admitted_tier = tier_batch(&inputs).expect("the admitted tier reserve"); - let down = inputs - .model - .tensors - .iter_mut() - .find(|t| { - t.role == Role::RoutedExperts - && t.layer == Some(0) - && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) - }) - .expect("layer zero's down"); - down.ty = Q8_0; - down.file_bytes = down.dims.iter().product::() / down.ty.blck_size().unwrap() - * down.ty.type_size().unwrap(); let reserve = prompt_reserve_bytes(&inputs.hp, &inputs.spec, &inputs.model, PROMPT_ROWS) .expect("prompt reserve"); - assert_eq!( - reserve, admitted, - "the host-only Q8_0 down adds no card form" - ); - assert_eq!( - tier_batch(&inputs).expect("mixed tier reserve"), - admitted_tier - ); - inputs.prompt_reserve = reserve; let ample = workstation::plan_gate(LAYERS); let placed = inputs .plan(&le, 64, &PlanLevers::default()) @@ -457,5 +435,9 @@ mod tests { .plan(&machine, 64, &levers) .expect("the card holds the terms and the reserve"); assert!(plan.violations().is_empty()); + assert!( + plan.n_l.iter().all(|&n| n == 0), + "the exact dense-and-prompt budget leaves no expert room" + ); } } diff --git a/crates/model/tests/mimo2_meta.rs b/crates/model/tests/mimo2_meta.rs index 64ecda04a..a48c53896 100644 --- a/crates/model/tests/mimo2_meta.rs +++ b/crates/model/tests/mimo2_meta.rs @@ -401,7 +401,7 @@ fn hw_mimo2_plan() { let tiered = workstation::plan_bp( layers, None, - model::arch::mimo2::place::tier_batch(&inputs.hp, 512), + model::arch::mimo2::place::tier_batch(&inputs.hp, 512).expect("valid tier geometry"), ); let bp = inputs .plan(&tiered, ctx, &PlanLevers::default()) diff --git a/crates/model/tests/musespark_meta.rs b/crates/model/tests/musespark_meta.rs index d3cabdd9c..079fc4c19 100644 --- a/crates/model/tests/musespark_meta.rs +++ b/crates/model/tests/musespark_meta.rs @@ -368,7 +368,7 @@ fn hw_musespark_plan() { workstation::A6000, &[workstation::RTX_3090], None, - place::tier_batch(&hp), + place::tier_batch(&hp).expect("valid tier geometry"), ) .unwrap_or_else(|e| panic!("the tiered machine: {e}")); place::stage_placed(&mut m).unwrap_or_else(|e| panic!("the tiered machine's stage: {e}")); diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index ff86bc0f0..64175fc36 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -522,7 +522,7 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); - let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.spec)); + let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.spec)?); let plan = inputs.plan_rule( &machine, CTX, @@ -537,7 +537,12 @@ fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place #[test] fn qwen3_30b_q8_bp_tier_experts() { let inputs = qwen3_inputs(); - let machine = tiered(Q3_LAYERS, A6000, RTX_3090, q3::tier_batch(&inputs.hp)); + let machine = tiered( + Q3_LAYERS, + A6000, + RTX_3090, + q3::tier_batch(&inputs.hp).expect("valid tier geometry"), + ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) .expect("the A6000 stage plans a tier"); diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 32f9c9c8a..2d57f3239 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -287,20 +287,14 @@ impl Card { let Some(leg) = self.tier_leg else { return Ok(None); }; - let forms = model - .tensors - .iter() - .filter(|t| { - t.role == Role::RoutedExperts - && t.layer - .is_some_and(|l| counts.get(l).is_some_and(|&n| n > 0)) - && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) - }) - .filter_map(|t| crate::kernels::down(t.ty)) - .fold(crate::kernels::DownForms::NONE, |forms, (_, act)| { - forms.with(act) - }); - Ok(Some((leg.bytes(leg.downs)?, leg.bytes(forms)?))) + Ok(Some(( + leg.bytes(leg.downs)?, + selected_leg_scratch( + leg, + counts, + routed_down_acts(model).map(|(l, act)| (l, crate::kernels::DownForms::of(act))), + )?, + ))) } fn selected_reserve_bytes( @@ -372,6 +366,41 @@ impl Card { } } +/// Routed down forms with their layers, before a placement selects any experts. +fn routed_down_acts( + model: &ModelTensors, +) -> impl Iterator + '_ { + model.tensors.iter().filter_map(|t| { + if t.role != Role::RoutedExperts || ExpertPart::of_name(&t.name) != Some(ExpertPart::Down) { + return None; + } + Some((t.layer?, crate::kernels::down(t.ty)?.1)) + }) +} + +/// Price the actual selected union, from tensors or cached per-layer forms. +/// Empty counts and layers with no down keep the descriptor's empty-form scratch. +fn selected_leg_scratch( + leg: workstation::TierLegScratch, + counts: &[u64], + downs: impl IntoIterator, +) -> Result { + use crate::kernels::{DownAct, DownForms}; + + let forms = downs + .into_iter() + .filter(|(l, _)| counts.get(*l).is_some_and(|&n| n > 0)) + .fold(DownForms::NONE, |mut forms, (_, down)| { + for act in [DownAct::Q8Act, DownAct::Q8Blocks32] { + if down.holds(act) { + forms = forms.with(act); + } + } + forms + }); + leg.bytes(forms) +} + /// The host tier: its usable bytes and what is set aside before any tensor. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Host { @@ -3109,32 +3138,23 @@ impl Fill<'_, '_> { .routed .unwrap_or(RoutedRule::PerType(CardFormat::of_routed)); let uploads = card_uploads(self.model, self.c, self.rows, self.eligible, format)?; - let mut down_acts = if card.tier_leg.is_some() { - vec![None; self.model.layers] + // Only tier legs prepare the per-layer forms. Candidates reuse these + // rather than scanning tensors, including when a layer has several downs. + let mut down_forms = if card.tier_leg.is_some() { + vec![crate::kernels::DownForms::NONE; self.model.layers] } else { Vec::new() }; if card.tier_leg.is_some() { - for t in &self.model.tensors { - if t.role == Role::RoutedExperts - && ExpertPart::of_name(&t.name) == Some(ExpertPart::Down) - && let Some(l) = t.layer - && let Some((_, act)) = crate::kernels::down(t.ty) - { - down_acts[l] = Some(act); + for (l, act) in routed_down_acts(self.model) { + if let Some(forms) = down_forms.get_mut(l) { + *forms = forms.with(act); } } } let scratch = |counts: &[u64]| { card.tier_leg.map_or(Ok(0), |leg| { - let forms = counts - .iter() - .zip(&down_acts) - .filter_map(|(&n, &act)| (n > 0).then_some(act).flatten()) - .fold(crate::kernels::DownForms::NONE, |forms, act| { - forms.with(act) - }); - leg.bytes(forms) + selected_leg_scratch(leg, counts, down_forms.iter().copied().enumerate()) }) }; let bound = card @@ -3142,7 +3162,12 @@ impl Fill<'_, '_> { .map(|leg| leg.bytes(leg.downs)) .transpose()? .unwrap_or(0); - let reserves = card.selected_reserve_bytes(self.model, n_l)?; + let selected = scratch(n_l)?; + let reserves = card + .reserve_bytes() + .and_then(|fixed| fixed.checked_sub(bound)) + .and_then(|fixed| fixed.checked_add(selected)) + .ok_or_else(|| card.bytes_past_u64())?; let footprint = Footprint::new(card.granule_bytes, &uploads); let budget = i128::from(capped(card, self.card_budget)) - i128::from(self.kv) @@ -6461,7 +6486,7 @@ mod tests { ( [Q5_K; 2], DownAct::Q8Act, - workstation::tier_leg_batch_bytes(256, 256, 1, 2, 3), + workstation::tier_leg_batch_bytes(256, 256, 1, 2, 3).expect("valid tier bytes"), ), ] { let scratch = |forms| crate::kernels::leg_bytes(2, 1, 256, 256, forms); diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 976e3e82c..5ab150537 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -476,8 +476,38 @@ const TILE_COLS: u64 = 8; /// activations, the tier places, their q8_1 form, the down outputs by slot. #[must_use] pub const fn tier_batch_staging_bytes(n_embd: u64, n_used: u64, cols: u64) -> u64 { - let slots = cols * n_used; - 4 * cols * n_embd + 4 * slots + q8_act_bytes(cols, n_embd) + 4 * slots * n_embd + match try_tier_batch_staging_bytes(n_embd, n_used, cols) { + Some(bytes) => bytes, + None => panic!("tier_batch_staging_bytes: staging bytes overflow u64"), + } +} + +const fn try_tier_batch_staging_bytes(n_embd: u64, n_used: u64, cols: u64) -> Option { + let Some(slots) = cols.checked_mul(n_used) else { + return None; + }; + let Some(places) = 4u64.checked_mul(slots) else { + return None; + }; + let Some(down) = places.checked_mul(n_embd) else { + return None; + }; + let Some(rows) = 4u64.checked_mul(cols) else { + return None; + }; + let Some(rows) = rows.checked_mul(n_embd) else { + return None; + }; + let Some(bytes) = rows.checked_add(places) else { + return None; + }; + let Some(q8) = try_q8_act_bytes(cols, n_embd) else { + return None; + }; + let Some(bytes) = bytes.checked_add(q8) else { + return None; + }; + bytes.checked_add(down) } /// The tile path's scratch for blocks of up to `slots` slots of rows of @@ -527,7 +557,7 @@ pub const fn walk_a_tile_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: /// token, through experts of `ff`, cut in chunks of `chunk` tokens: one run of /// the card leg's buffers (`bloomery_gpu::card_leg::LegBufs`) with its slots' /// columns in the 128-value form ([`leg_bytes`]), in place of the tile path's -/// scratch. Selected-form callers use [`tier_leg_batch_bytes_of`]. +/// scratch. Selected-form callers use [`try_tier_leg_batch_bytes_of`]. #[must_use] pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: u64) -> u64 { leg_bytes(chunk, n_used, n_embd, ff, DownForms::of(DownAct::Q8Act)) @@ -537,8 +567,26 @@ pub const fn tier_chunk_scratch_bytes(n_embd: u64, ff: u64, n_used: u64, chunk: /// rows the tier hands back and the places it reads. #[must_use] pub const fn tier_batch_host_bytes(n_embd: u64, n_used: u64, cols: u64) -> u64 { - let slots = cols * n_used; - 2 * (4 * slots * n_embd + 4 * slots) + match try_tier_batch_host_bytes(n_embd, n_used, cols) { + Some(bytes) => bytes, + None => panic!("tier_batch_host_bytes: host bytes overflow u64"), + } +} + +const fn try_tier_batch_host_bytes(n_embd: u64, n_used: u64, cols: u64) -> Option { + let Some(slots) = cols.checked_mul(n_used) else { + return None; + }; + let Some(places) = 4u64.checked_mul(slots) else { + return None; + }; + let Some(down) = places.checked_mul(n_embd) else { + return None; + }; + let Some(rows_and_places) = down.checked_add(places) else { + return None; + }; + 2u64.checked_mul(rows_and_places) } /// The expert tier's prompt-batch bytes for blocks of up to `cols` tokens of @@ -555,17 +603,15 @@ pub const fn tier_batch_bytes(n_embd: u64, ff: u64, n_used: u64, cols: u64) -> T /// [`tier_batch_bytes`] for a candidate-selected common leg in the 128-value form. /// -/// # Panics -/// Panics on invalid geometry or overflowing byte counts. -#[must_use] +/// Refuses invalid geometry or overflowing byte counts before placement. pub fn tier_leg_batch_bytes( n_embd: u64, ff: u64, n_used: u64, chunk: u64, cols: u64, -) -> TierBatchBytes { - tier_leg_batch_bytes_of( +) -> Result { + try_tier_leg_batch_bytes_of( n_embd, ff, n_used, @@ -574,23 +620,6 @@ pub fn tier_leg_batch_bytes( ) } -/// The common leg's buffers in `downs`, with the same checked geometry and -/// candidate-dependent reserve as [`try_tier_leg_batch_bytes_of`]. -/// -/// # Panics -/// Panics on invalid geometry or overflowing byte counts. -#[must_use] -pub fn tier_leg_batch_bytes_of( - n_embd: u64, - ff: u64, - n_used: u64, - shape: (u64, u64), - downs: DownForms, -) -> TierBatchBytes { - try_tier_leg_batch_bytes_of(n_embd, ff, n_used, shape, downs) - .expect("valid common leg batch geometry and byte counts") -} - /// The common leg's staging, host rows and selected-form scratch, refusing /// invalid geometry or overflowing byte counts before placement. pub fn try_tier_leg_batch_bytes_of( @@ -612,17 +641,10 @@ pub fn try_tier_leg_batch_bytes_of( if cols == 0 { return None; } - let slots = cols.checked_mul(n_used)?; - let places = 4u64.checked_mul(slots)?; - let down = places.checked_mul(n_embd)?; - let staging = 4u64 - .checked_mul(cols)? - .checked_mul(n_embd)? - .checked_add(places)? - .checked_add(try_q8_act_bytes(cols, n_embd)?)? - .checked_add(down)?; - let host = 2u64.checked_mul(down.checked_add(places)?)?; - Some((staging, host)) + Some(( + try_tier_batch_staging_bytes(n_embd, n_used, cols)?, + try_tier_batch_host_bytes(n_embd, n_used, cols)?, + )) }; let (staging, host) = counted().ok_or_else(|| PlacementError::Metadata { key: "tier leg batch".into(), diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 218196059..013052334 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -95,16 +95,25 @@ Host-only projections intentionally have no card totals: fixed reserves remain o Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. -Follow-up review found a real empty-tier mismatch: the Q8Act-only common constructor still -advertised fixed scratch while `LegTier` selected no down form for zero resident experts. -The expanded public planner regression failed before the fix: reserve24868 versus the -selected allocation20692. `tier_leg_batch_bytes` now produces the same candidate-selected -descriptor as the explicit-form constructor, preserving its infallible signature. -Qwen3/Qwen35, MiMo and Muse therefore price their actual selected union, including empty; -MiniMax, GLM5Next and Solar already use selected constructors. Separate tile/route-based -fixed-reserve paths remain unchanged. No reserve check is weakened and no unused Q8Act -buffer is allocated to hide the mismatch. The same regression passes after the repair. -An old MiMo helper-equality assertion was removed; its actual plan/reserve/refusal checks remain. +`tier_leg_batch_bytes` and the explicit-form `try_tier_leg_batch_bytes_of` return +`Result` and construct candidate-selected descriptors. +An empty tier selects no down form, matching `LegTier`'s actual allocation instead of +advertising unused Q8Act scratch. Qwen3/Qwen35, MiMo and Muse use this checked path. +MiniMax obtains its forms from `accepts::leg_down_forms` under its selected program, +as GLM5Next and Solar do; no fallback format can admit a mismatched triple. +The checked constructors and non-leg paths share staging and host byte formulas. +Malformed geometry and overflow are refused rather than panicking; a dense Qwen3.6 +request for an expert tier is a named metadata refusal, not zero expert geometry. +Separate tile/route-based fixed-reserve paths remain unchanged. + +The planner's selected scratch calculation has one union/byte rule. Fill prepares +per-layer forms once for dynamic tier-leg cards, including multiple forms in one +layer, and reuses them with its cached footprint instead of rescanning tensors for +each candidate. Fixed-reserve cards need no selected-form cache. + +The Solar budget regression starts from a typed mixed GGUF description and exercises +the planner's one-byte refusal/success boundary without overwriting its described +prompt reserve. The exact dense-and-prompt budget leaves no room for expert weights. The real mixed-wrapper hardware case also checks allocation of a configured empty tier. Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: @@ -115,9 +124,6 @@ real CUDA mixed/empty-tier consumer:1 passed,0 failed,0 ignored,427 filtered,24. Its fresh4bin/8carrier canonical comparison again found all1657entries identical to main15778e2c, with non-JIT rows equal. Native formatting/Clippy passed (warning3=ratchet3); model557 and placement202 passed,0 failed,5 model tests ignored,0 filtered. -Request75d4e06e-f3fb-4001-bcc4-e188c169edf5 exited0, credential cleanup unconfirmed; -GPU1 remained active/healthy and was not stopped. The PR returned to review-ready only after -this repair proof. Later documentation-only commits do not count as another hardware run. The host-only public planner consumer covers one-byte boundaries before the first resident layer and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer From 5bee29875664d05cfd6cd9ddbdf49ffc50c7a684 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:52:05 +0900 Subject: [PATCH 63/85] Bound Solar reserve regression from a host-only reference plan --- crates/model/src/arch/solar2/place.rs | 51 +++++++++++---------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 8c011861e..4de9ecd33 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -177,7 +177,7 @@ mod tests { use super::super::hparams::tests::{LAYERS, keys, tensors}; use super::{PROMPT_ROWS, PlaceError, PlanInputs, prompt_reserve_bytes, tier_batch}; use crate::arch::synthetic::{Ty, V, header_shaped, header_typed}; - use crate::placement::{Device, PlacementError, PlanLevers, Violation, workstation}; + use crate::placement::{Device, PlacementError, PlanLevers, workstation}; /// The small header's file, described; each test names its own file (the tests run in /// threads of one process). @@ -400,36 +400,25 @@ mod tests { ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); - machine.cards[0].usable_bytes = machine.cards[0].margin_bytes; - let short = loop { - match inputs.plan(&machine, 64, &levers) { - Err(PlaceError::Broken(v)) => { - let over = v - .iter() - .find_map(|v| match v { - Violation::CardOver { total, limit, .. } => Some(total - limit), - _ => None, - }) - .unwrap_or_else(|| panic!("{v:?}, no card over its limit")); - machine.cards[0].usable_bytes += over; - } - Err(PlaceError::PromptOver { - reserve: got, - left, - short, - .. - }) => { - assert_eq!((got, short), (reserve, reserve - left)); - break short; - } - other => panic!("{other:?}, not the reserve's refusal"), - } - }; - machine.cards[0].usable_bytes += short - 1; - assert!(matches!( - inputs.plan(&machine, 64, &levers), - Err(PlaceError::PromptOver { short: 1, .. }) - )); + let host_only = crate::placement::plan_host_routed( + &inputs.model, + &machine, + 64, + &inputs.seq_terms(), + &levers, + ) + .expect("the host-only reference plans"); + let dense = crate::arch::place_hybrid::card_terms(&host_only.cards[0]); + machine.cards[0].usable_bytes = machine.cards[0].margin_bytes + dense + reserve - 1; + match inputs.plan(&machine, 64, &levers) { + Err(PlaceError::PromptOver { + reserve: got, + left, + short, + .. + }) => assert_eq!((got, left, short), (reserve, reserve - 1, 1)), + other => panic!("{other:?}, not the reserve's one-byte refusal"), + } machine.cards[0].usable_bytes += 1; let plan = inputs .plan(&machine, 64, &levers) From 5c1c67a7d990c404d7dc700b0c9ee580f5da653a Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:52:05 +0900 Subject: [PATCH 64/85] Preserve IQ4 pin provenance and meaningful refusal diagnostics --- crates/gpu-deepseek41/src/block.rs | 10 +++++++++- crates/gpu/src/site.rs | 13 +++++++++++-- crates/placement/src/kernels.rs | 10 ++++++++-- docs/research/flint-attach.md | 17 ++++++++--------- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/crates/gpu-deepseek41/src/block.rs b/crates/gpu-deepseek41/src/block.rs index d35f5b674..750092b49 100644 --- a/crates/gpu-deepseek41/src/block.rs +++ b/crates/gpu-deepseek41/src/block.rs @@ -711,7 +711,15 @@ mod tests { [Mxfp8, Q8_0, Mxfp8], [Q4K, Mxfp8, Q4K], ] { - assert!(matches!(plan(types), Err(GpuError::Shape { .. }))); + let Err(GpuError::Shape { what, detail }) = plan(types) else { + panic!( + "{types:?}: incompatible activation forms were not refused as a shape error" + ); + }; + assert_eq!(what, WHAT); + for ty in types { + assert!(detail.contains(&ty.to_string()), "{types:?}: {detail}"); + } } } diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 3fe55f491..b8612ccaf 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -1093,7 +1093,8 @@ pub fn gemm( k.gemm.enqueue_gemm( stream, GemmArgs { - ty: gemm_weight(ty).ok_or(GpuError::state(WHAT, "a K-quant's GEMM type"))?, + ty: gemm_weight(ty) + .ok_or(GpuError::state(WHAT, "the selected site's GEMM type"))?, w: match src { SiteWeight::Words { w, .. } => w, _ => kq(w, WHAT, name)?, @@ -1747,7 +1748,15 @@ mod tests { ("shared", (2, 256), &[SiteTy::Q4K][..]), ("unsupported", (2, 256), SiteTy::ALL), ] { - assert!(file_site(&file, "test", name, shape, allowed).is_err()); + let Err(chain::ChainError::Shape { what, detail }) = + file_site(&file, "test", name, shape, allowed) + else { + panic!( + "{name}: invalid shape or launch admission was not refused as a shape error" + ); + }; + assert_eq!(what, "test"); + assert!(detail.contains(name), "{name}: {detail}"); } } diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index a54404d49..28be171a7 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -634,7 +634,13 @@ mod tests { ); } - // PIN(2026-10-27): IQ4_XS adds the existing selected GEMV as an ordinary site. + // PIN(2026-10-16): the dense pairs gain NVFP4 (`nvfp4_gemv` through + // `site::kgemv`); every other pair is unchanged. + // PIN(2026-10-19): and bloomery's own MXFP8 (`mxfp8_gemv` through + // `site::gemv`); every other pair is unchanged. + // PIN(2026-10-20): and bloomery's own FP8_B128 (`fp8_b128_gemv` through + // `site::gemv`); BF16 also uses `bf16_gemv` and `bf16_tile_gemm` over its words. + // PIN(2026-10-26): IQ4_XS adds the existing selected GEMV as an ordinary site (UTC). #[test] fn dense_admits_exactly_the_pinned_pairs() { assert_eq!( @@ -743,7 +749,7 @@ mod tests { // columns); every other pair is unchanged. // PIN(2026-10-25): IQ2_XS, IQ1_S, IQ2_S and IQ1_M load as the file's // blocks, as IQ2_XXS does; every other pair is unchanged. - // PIN(2026-10-27): ordinary IQ4_XS matrices retain their file blocks. + // PIN(2026-10-26): ordinary IQ4_XS matrices retain their file blocks (UTC). #[test] fn the_card_layouts_are_the_pinned_pairs() { assert_eq!( diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index c73e7787d..23c2e88c4 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -80,7 +80,9 @@ those same sites. IQ4_XS is not relabeled as a routed expert or requantized. the ordinary `SharedExpert` layout, unequal matrix widths, one/eight-column block execution and the wide-prompt transition/captured replay. Wide scratch holds only the q8_1 input and down forms these sites read. Its nonnegative constant-weight -case derives a scalar floating-operation bound; it is not a whole-model accuracy pin. +case derives a scalar floating-operation bound; uniform codes do not detect nibble +decode ordering. That remains a separate IQ4 kernel-gate obligation, not a +whole-model accuracy pin or a claim made by this wrapper fixture. Program-specific admission remains unchanged: in particular the legacy Qwen projection/shared dispatch does not gain this site automatically. This common @@ -92,14 +94,11 @@ DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed, with repository patch31ab6a1d3f58 and LLVM21.1.8 were used. Its canonical comparison to main15778e2c covered4bins/8carriers/1657entries: all identical, non-JIT resource rows equal; no JIT-resource comparison is claimed. -All-target native Clippy/formatting and current-head GitHub T0/lint passed. -The original execution request5a907e86-7ac6-4219-a3b6-e3cf1de84e03 exited0 but credential -cleanup remains unconfirmed; test success does not certify credential revocation. -Raw logs and canonical manifests remain in the contribution workspace's -`flint-closure-20261026` and its matching remote evidence directory. GPU1 was not stopped. -Later documentation-only commits are not a new GPU execution. Wider affected real-model, -placement and residency coverage remains explicit for maintainer review; #782 is review-ready, -not merged, and this does not establish a Flint oracle, physical cc12.1, ROCm or performance. +All-target native Clippy/formatting and source203f5125 GitHub T0/lint passed. +These are source-specific historical results, not another hardware execution of +later diagnostic, test or documentation changes. Wider affected real-model, +placement and residency coverage remains explicit; this does not establish a +Flint oracle, physical cc12.1, ROCm or performance. From a3206fe49a3e6cfb6c88e8160a9be51509a38875 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:55:40 +0900 Subject: [PATCH 65/85] Use one resident sequence for the Solar reference budget --- crates/model/src/arch/solar2/place.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 4de9ecd33..6f0af8616 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -404,7 +404,7 @@ mod tests { &inputs.model, &machine, 64, - &inputs.seq_terms(), + &inputs.seq_terms().slots_of(1), &levers, ) .expect("the host-only reference plans"); From 6ecd144b3fda017388440e050258dfd2ffbc9b7f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 07:57:52 +0900 Subject: [PATCH 66/85] Exclude reusable granule padding from the no-expert-room boundary --- crates/model/src/arch/solar2/place.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 6f0af8616..d6051fae9 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -400,6 +400,9 @@ mod tests { ); let levers = PlanLevers::default(); let mut machine = workstation::plan_gate(LAYERS); + // Rounded dense padding may hold experts for no extra card bytes. + // Byte granules isolate the reserve's exact no-expert-room boundary. + machine.cards[0].granule_bytes = 1; let host_only = crate::placement::plan_host_routed( &inputs.model, &machine, From f3b2a21199126a46b87979cb677eb7447a9f27f5 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:02:27 +0900 Subject: [PATCH 67/85] Use the nonzero byte-granule constant in the reserve fixture --- crates/model/src/arch/solar2/place.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index d6051fae9..849e38fba 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -402,7 +402,7 @@ mod tests { let mut machine = workstation::plan_gate(LAYERS); // Rounded dense padding may hold experts for no extra card bytes. // Byte granules isolate the reserve's exact no-expert-room boundary. - machine.cards[0].granule_bytes = 1; + machine.cards[0].granule_bytes = std::num::NonZeroU64::MIN; let host_only = crate::placement::plan_host_routed( &inputs.model, &machine, From 7e37ef9a6ebbd766032ee521a0ee16e25f163851 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:04:45 +0900 Subject: [PATCH 68/85] Document the exact-granule reserve boundary --- docs/research/flint-attach.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 013052334..4bf20e2ab 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -113,7 +113,8 @@ each candidate. Fixed-reserve cards need no selected-form cache. The Solar budget regression starts from a typed mixed GGUF description and exercises the planner's one-byte refusal/success boundary without overwriting its described -prompt reserve. The exact dense-and-prompt budget leaves no room for expert weights. +prompt reserve. With byte granules (no reusable dense padding), the exact dense-and-prompt +budget leaves no room for expert weights. The real mixed-wrapper hardware case also checks allocation of a configured empty tier. Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: From 12391442fd2004c9140bc777a58c5d6c44c8e4e1 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:12:14 +0900 Subject: [PATCH 69/85] Keep handoff measurements durable and move operational receipts out of contract docs --- docs/research/flint-attach.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index f69a95283..841f93e6d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -18,25 +18,15 @@ error label with the common `F32 tier handoff` context. The measured common-name falls from 39 to 37; the ABI-stable generated kernel names and dispatch stay unchanged. The earlier GPU run is not relabeled as a run of these later commits. -The original remote request timed out while its child continued; a separate read-only -collector recovered the original successful log rather than replaying the run. -Original-request credential cleanup remains unconfirmed and is not implied by test success. -Raw evidence is retained outside Git under -`/home/jmtai04/flint-completion-4dadee0-20261025T205648Z/evidence/closure-20261026/`; -local receipts are under `flint-closure-20261026` in the contribution workspace. The actual `generate_ds41` carrier at `b9b6e013ebfab5748688be52e9a5d398ac2fc9a3` subsequently measured `ffn_handoff_6_tier` on sm_86 / physical cc8.6: ptxas13.3.33 spill0, driver-JIT local0, both register counts26 (driver595.91.07, JIT CUDA13.2, Rust1.99.1 / cuda-oxide9df9e7ce / repository patch31ab6a1d3f58). Only that measured row is added. Its normalized entry digest is `a176a9a5846778e40ba99fc2be637d90` (decl1, 163 lines). -This resource request also exceeded the gateway window; a read-only collector recovered -the original scan, with no remaining owned process and GPU1 active/healthy. -The original script recorded exit23 after producing the complete successful scan; -a post-timeout stdout write failure is the explanation inferred from its final curl step, -not an invented exit0 for the request. The original credential cleanup remains unconfirmed. -Accessible cards establish only cc8.6. The other three sm_86/sm_121/sm_121a + physical -cc12.1 rows remain unmeasured and absent; #781 stays Draft until that gate is dispositioned. +Measurements establish only physical cc8.6. The other three +sm_86/sm_121/sm_121a + physical cc12.1 rows remain unmeasured and absent; +their genuine driver-JIT resource evidence is still required. ## Host reader status From 420131b145b9b38e3cfe5238a2c52d76926f4340 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:13:29 +0900 Subject: [PATCH 70/85] Integrate checked tier contracts with Flint scale ownership --- crates/model/src/arch/minimaxm3/place.rs | 1 + crates/placement/src/placement/workstation.rs | 1 - docs/research/flint-attach.md | 57 +++++++++---------- 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index e8528fb48..c794a365e 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -142,6 +142,7 @@ pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result Date: Tue, 27 Oct 2026 09:26:29 +0900 Subject: [PATCH 71/85] Format merged file-site imports --- crates/gpu/src/site.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 939790aad..036ba51a6 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2023,8 +2023,8 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, GemmWeight, Order, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, - gemm_weight, kgemv_order, swiglu_launch, + Form, GemmWeight, Order, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, + file_site, gemm_weight, kgemv_order, swiglu_launch, }; use crate::kquant::Act; use gguf::Split; From 49b11b36d4a1de2e140010c5a2d2c9afc3e2671a Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:29:32 +0900 Subject: [PATCH 72/85] Reconcile projected-site IQ4 admission with current main --- crates/model/src/arch/coverage.rs | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index 24630733c..3f6d64975 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -3731,6 +3731,8 @@ mod tests { // PIN(2026-10-26): the K-quants (Q3_K to Q6_K) joined the dense sites' set: both walks project a site by name // (`site::ByName`), the q8_1 quantization and the K-quant gemv by chunks on the ubatch walk. The shared expert's // down keeps the f32-row types: K = 640 is not whole super-blocks. + // PIN(2026-10-26): IQ4_XS uses the same q8_1 input through the selected site's + // `_sel` launch. It is admitted for projected sites, not a K = 640 shared down. #[test] fn the_qwen38_dense_pins_read_the_projected_sites() { let q8_1 = [ @@ -3738,6 +3740,7 @@ mod tests { GgmlType::Q4_K, GgmlType::Q5_K, GgmlType::Q6_K, + GgmlType::IQ4_XS, ]; let f32_rows = [ GgmlType::Q8_0, @@ -3751,24 +3754,15 @@ mod tests { GgmlType::Q2_K, GgmlType::IQ3_S, GgmlType::IQ4_NL, - GgmlType::IQ4_XS, GgmlType::Q4_0, ]; - for (name, role, what) in [ - ("attn_q.weight", Role::Attention, "attention matrices"), - ("ssm_out.weight", Role::Attention, "attention matrices"), - ( - "indexer.q_proj.weight", - Role::Attention, - "attention matrices", - ), - ( - "indexer.k_proj.weight", - Role::Attention, - "attention matrices", - ), - ("ffn_gate_shexp.weight", Role::SharedExpert, "shared expert"), - ("ffn_up_shexp.weight", Role::SharedExpert, "shared expert"), + for (name, role) in [ + ("attn_q.weight", Role::Attention), + ("ssm_out.weight", Role::Attention), + ("indexer.q_proj.weight", Role::Attention), + ("indexer.k_proj.weight", Role::Attention), + ("ffn_gate_shexp.weight", Role::SharedExpert), + ("ffn_up_shexp.weight", Role::SharedExpert), ] { for ty in q8_1.into_iter().chain(f32_rows) { let got = items(Program::Qwen38Body, &matrix(name, role, ty)); @@ -3777,7 +3771,6 @@ mod tests { for ty in refused { let got = items(Program::Qwen38Body, &matrix(name, role, ty)); assert_eq!(got.len(), 1, "{name} {ty}: {got:?}"); - assert!(got[0].starts_with(&format!("{ty} {what}")), "{got:?}"); } } // The shared expert's down is K = 640: a K-quant's rows are whole @@ -3791,10 +3784,6 @@ mod tests { for ty in q8_1 { let got = items(Program::Qwen38Body, &matrix(down, Role::SharedExpert, ty)); assert_eq!(got.len(), 1, "{down} {ty}: {got:?}"); - assert!( - got[0].starts_with(&format!("{ty} shared expert down (K = 640")), - "{got:?}" - ); } for ty in refused { let got = items(Program::Qwen38Body, &matrix(down, Role::SharedExpert, ty)); From cf035b8f68e96361a8eb56691a12b72be410032f Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:31:42 +0900 Subject: [PATCH 73/85] Document exact-source mixed-tier review consumers --- docs/research/flint-attach.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 4bf20e2ab..aa2c04739 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -126,6 +126,18 @@ Its fresh4bin/8carrier canonical comparison again found all1657entries identical with non-JIT rows equal. Native formatting/Clippy passed (warning3=ratchet3); model557 and placement202 passed,0 failed,5 model tests ignored,0 filtered. +The reviewed contract source `7e37ef9a6ebbd766032ee521a0ee16e25f163851` +executed the real CUDA mixed/empty-tier consumer on RTX3090 cc8.6:1 passed, +0 failed/ignored,427 filtered,23.98s. Its metadata-only +`a_dense_header_plans_through_qwen35_on` frontend consumer also passed1 case, +including the named dense-header expert-tier refusal; that is not a second GPU case. +The review code passed native formatting/all-target Clippy (warning3=ratchet3), +chain23, GGUF67, model557 and placement202 (five model cases ignored,0 filtered). +An exact-source7e37ef9a canonical comparison against main15778e2c found all1657 +entries identical across4bins/8carriers and non-JIT rows equal; it did not compare +driver-JIT resource rows. The older full-library counts above remain historical, +not a new full-library run of the review repairs. + The host-only public planner consumer covers one-byte boundaries before the first resident layer and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer are not a whole-model oracle or every affected model/residency gate; canonical equality does not From 7a7cfdc783cc37281c49f7e01126b9125cca7b5b Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:37:33 +0900 Subject: [PATCH 74/85] Exercise IQ4 projected admission through typed site fixture --- crates/gpu/src/arch/qwen3moe/plan38.rs | 4 ++-- crates/gpu/src/site.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/gpu/src/arch/qwen3moe/plan38.rs b/crates/gpu/src/arch/qwen3moe/plan38.rs index 3340b2497..46714324b 100644 --- a/crates/gpu/src/arch/qwen3moe/plan38.rs +++ b/crates/gpu/src/arch/qwen3moe/plan38.rs @@ -484,7 +484,7 @@ fn gqa_fits(g: &Gqa) -> bool { /// Every layer's plan from its kind, `ple` the PLE site's layer, each weight /// checked against the geometry: type and shape. A dense projection, the /// indexer's two among them, is of a type the common site projects by name -/// (`site::PROJECTED`: the f32 rows' families and the K-quants), each +/// (`site::PROJECTED`: the f32 rows' families, the K-quants and IQ4_XS), each /// tensor's own: the walks pick its launch by the resident form. The joined /// β·α stack is of an f32 rows' family (`site::BY_NAME`). An HC matrix is one /// `HcMatrix::of` reads. The gains, the conv and the injects are F32 (the @@ -648,7 +648,7 @@ const SUPER_BLOCK: usize = 256; /// The resident weight `name`, `rows` rows of `k` values, of a type a /// by-name projection launches (`site::PROJECTED`: every dense family but -/// Q5_0, the K-quants and NVFP4 over the q8_1 rows of their input); another +/// Q5_0, the K-quants, IQ4_XS and NVFP4 over the q8_1 rows of their input); another /// type or shape is refused by name. fn projected_shape(w: &Weights, name: &str, rows: usize, k: usize) -> Result<(), GpuError> { let ty = dense::resident_site(w, WHAT, name, (rows, k), &dense::PROJECTED)?; diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index a1beebdef..b2a2923b4 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2101,7 +2101,7 @@ mod tests { ); } assert_eq!( - file_site(&file, "test", "shared", (2, 256), SiteTy::ALL).ok(), + file_site(&file, "test", "shared", (2, 256), &super::PROJECTED).ok(), Some(SiteTy::Iq4Xs) ); for (name, shape, allowed) in [ From b0fc05e5349330927ceb33ce6f30e584f25ecf4d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:45:53 +0900 Subject: [PATCH 75/85] Keep projected-file admission coverage without metadata-copy guard --- crates/gpu/src/site.rs | 68 ------------------------------------------ 1 file changed, 68 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index b2a2923b4..f2449c320 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2027,12 +2027,10 @@ mod tests { Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, gemm_weight, kgemv_order, swiglu_launch, }; - use crate::gemm::GemmWeight; use crate::kquant::Act; use gguf::Split; use gguf::quant::GgmlType; use gguf::write::{Layout, TensorDecl}; - use model::placement::kernels; /// A by-name projection's extra launches follow the type: none for a site /// that reads the f32 rows or the q8 blocks of 32, the quantization for a @@ -2186,72 +2184,6 @@ mod tests { } } - /// Every site family is the common table's dense row, with its form and wide launch. - #[test] - fn every_family_is_the_tables_dense_row_of_one_type() { - let own = [ - GgmlType::F8E4M3_ROWS, - GgmlType::MXFP8, - GgmlType::NVFP4_ROWS, - GgmlType::FP8_B128, - ]; - for ty in gguf::QROWS.iter().map(|r| r.ty).chain(own) { - let family = SiteTy::of_ggml(ty); - if ty == GgmlType::F16 { - assert_eq!(family, Some(SiteTy::F32)); - assert_eq!(kernels::dense(ty), None); - continue; - } - assert_eq!(family.is_some(), kernels::dense(ty).is_some(), "{ty}"); - if let Some(s) = family { - assert_eq!(s.ggml(), ty, "{s}"); - } - } - let expected = [ - (SiteTy::Q3K, GgmlType::Q3_K, "Q3_K", Some(GemmWeight::Q3K)), - (SiteTy::Q4K, GgmlType::Q4_K, "Q4_K", Some(GemmWeight::Q4K)), - (SiteTy::Q5K, GgmlType::Q5_K, "Q5_K", Some(GemmWeight::Q5K)), - (SiteTy::Q6K, GgmlType::Q6_K, "Q6_K", Some(GemmWeight::Q6K)), - (SiteTy::Q5_0, GgmlType::Q5_0, "Q5_0", None), - (SiteTy::Q8_0, GgmlType::Q8_0, "Q8_0", None), - (SiteTy::F32, GgmlType::F32, "F32", None), - (SiteTy::Nvfp4, GgmlType::NVFP4, "NVFP4", None), - (SiteTy::Mxfp8, GgmlType::MXFP8, "MXFP8", None), - (SiteTy::Fp8B128, GgmlType::FP8_B128, "FP8_B128", None), - (SiteTy::Bf16, GgmlType::BF16, "BF16", None), - ( - SiteTy::Iq4Xs, - GgmlType::IQ4_XS, - "IQ4_XS", - Some(GemmWeight::Iq4Xs), - ), - ]; - assert_eq!(SiteTy::ALL.len(), expected.len()); - for (&s, (want, ty, label, grouped)) in SiteTy::ALL.iter().zip(expected) { - assert_eq!(s, want); - assert_eq!(s.ggml(), ty); - assert_eq!(s.to_string(), label); - assert_eq!(gemm_weight(s), grouped); - assert_eq!(kgemv_order(s).is_some(), s.reads() == Form::Q8x128); - let by_name = model::placement::accepts::dense_by_name(ty); - assert_eq!(super::BY_NAME.contains(&s), by_name.is_some(), "{s}"); - if let Some(entry) = by_name { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - let projected = model::placement::accepts::dense_projected(ty); - assert_eq!(super::PROJECTED.contains(&s), projected.is_some(), "{s}"); - if let Some(entry) = projected { - assert_eq!(SiteTy::of_entry(entry), Some(s), "{s}"); - } - } - assert_eq!(SiteTy::Q5_0.reads(), Form::Q8Blocks32); - assert_eq!(kgemv_order(SiteTy::Q5_0), None); - assert_eq!(SiteTy::Nvfp4.reads(), Form::Q8x128); - assert_eq!(kgemv_order(SiteTy::Nvfp4), Some(Order::TokenMajor)); - assert_eq!(SiteTy::Mxfp8.reads(), Form::F32); - assert_eq!(kgemv_order(SiteTy::Iq4Xs), Some(Order::TokenMajor)); - } - #[test] fn a_bf16_site_reads_the_f32_rows_and_is_a_gemv() { assert_eq!(SiteTy::of_ggml(GgmlType::BF16), Some(SiteTy::Bf16)); From 6a46aa1ad6282e818f0f1c1d488bc5b6c68d854d Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:05:49 +0900 Subject: [PATCH 76/85] Document projected-file execution and remove obsolete test import --- crates/gpu/src/site.rs | 2 +- docs/research/flint-attach.md | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index f2449c320..29c2287ae 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2024,7 +2024,7 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, Order, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, + Form, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, gemm_weight, kgemv_order, swiglu_launch, }; use crate::kquant::Act; diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 23c2e88c4..9eb58065d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -84,9 +84,18 @@ case derives a scalar floating-operation bound; uniform codes do not detect nibb decode ordering. That remains a separate IQ4 kernel-gate obligation, not a whole-model accuracy pin or a claim made by this wrapper fixture. -Program-specific admission remains unchanged: in particular the legacy Qwen -projection/shared dispatch does not gain this site automatically. This common -site contract adds no model admission or kernel. +The pinned current-main merge (`cb8d92ebddd1ce1a325a15450c06a01f57ecf88c`) also +reconciles its existing Qwen3.8 projected-site consumer: `site::PROJECTED` includes +IQ4_XS, and selected dispatch uses the supplied stream. Projected attention/shared +sites are admitted; the K=640 shared down and unsupported types/shapes remain refused. +This reuses the existing program and kernel, not a new model implementation. + +At source `b0fc05e5349330927ceb33ce6f30e584f25ecf4d`, the typed GGUF +`iq4_sites_keep_shape_and_launch_admission_separate` consumer passed1 case, +0 failed/ignored,430 filtered. Its successful arm uses `PROJECTED`, not merely +`ALL`; this checks actual file admission rather than copying a metadata registry. +The case is metadata-only and creates no CUDA context; oxide owns its linking. +It is not fresh NVIDIA execution of the current-main stream/launch paths. On 2026-10-26, source `203f5125a3c48a5b700c6f43318cfcb9c77ff169` executed the owning DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed,0 ignored, From 3bd0c0411a475aaa6e7e3340bbe592a5a80f1d0a Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:14:52 +0900 Subject: [PATCH 77/85] Format the cleaned site test imports --- crates/gpu/src/site.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/gpu/src/site.rs b/crates/gpu/src/site.rs index 29c2287ae..3f6a673ae 100644 --- a/crates/gpu/src/site.rs +++ b/crates/gpu/src/site.rs @@ -2024,8 +2024,8 @@ fn swiglu( #[cfg(test)] mod tests { use super::{ - Form, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, - gemm_weight, kgemv_order, swiglu_launch, + Form, SiteTy, SwigluLaunch, WideActs, WideForms, by_name_extra, file_site, gemm_weight, + kgemv_order, swiglu_launch, }; use crate::kquant::Act; use gguf::Split; From fd2a64b37f759063190233185bdaa1adfe846774 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:16:41 +0900 Subject: [PATCH 78/85] Document current-main dense-header consumer execution --- docs/research/flint-attach.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 0af54f40f..b05e17b4b 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -140,9 +140,16 @@ not a new full-library run of the review repairs. The component subsequently merges pinned upstream `cb8d92ebddd1ce1a325a15450c06a01f57ecf88c`. That reconciliation retains upstream's GLM mixer-form consumers alongside this component's -checked tier-byte API; the explicit-form API has no input-scale flag. The execution results -above belong to their named historical sources, not the merged source. No new runtime or -canonical-equivalence result is claimed by this merge. +checked tier-byte API; the explicit-form API has no input-scale flag. The GPU results +above belong to their named historical sources, not the merged source. + +At merged source `c38d603ca1d33c6c2bede63aba102199715fb9ef`, native formatting and +all-target Clippy passed, as did chain23, GGUF67, model559 and placement202 +(five model cases ignored, zero failed/filtered). The actual oxide-linked +`a_dense_header_plans_through_qwen35_on` frontend consumer passed1 case, +0 failed/ignored,6 filtered, including the named expert-tier refusal. +This is metadata-only host execution, not fresh NVIDIA execution of the merged source +or a whole-model numerical result. The host-only public planner consumer covers one-byte boundaries before the first resident layer and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer From 923da640f121f6a89444a7bbcbcdf2a8bd1918d9 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:20:56 +0900 Subject: [PATCH 79/85] Include FlInt in the current-main llama serve build recipe --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 41369f513..6bb4f62bf 100644 --- a/justfile +++ b/justfile @@ -1541,7 +1541,7 @@ gate-gpu-llama-dense: # through a scripted engine behind the same template, and answers /completion at temperature 0 with the ids generate_qwen3moe --last-step prints for the # same ids (run once the server has stopped, the file does not fit a card twice). One card; the gate lock. gate-gpu-llama-serve: - BLOOMERY_MODEL=nanbeige43 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_llama_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/llama-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_llama_serve --dir $D' + BLOOMERY_MODEL=nanbeige43 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_llama_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/llama-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_llama_serve --dir $D' # Body35's image rows on Clef-Flash at bartowski's Q8_0 (9.5 GB, either card), against the clefvis sets (set B's tower rows # as the spans' rows; set C's three prompts, their CPU twins and the C'' runs): the unit rows and the text rows a media call From a35ddd0eaa7abd164aa2e6b2fea44472a5608cef Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:42:02 +0900 Subject: [PATCH 80/85] Document the shared serve recipe feature contract --- docs/research/flint-attach.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 06207ecde..91dceea2d 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -940,6 +940,11 @@ q8_1 launch accounting. The Qwen3.8 admission fixture accepts its projected site retaining the K=640 shared-down refusal. IQ4 activation bands were not relaxed, and the removed source-text wiring guard was not restored. +Every recipe that builds the shared `bloomery-serve` binary carries its required-feature +union, including `flint`; this also applies to newly merged consumers such as +`gate-gpu-llama-serve`. Recipe validation and actual model-fence opening must agree +before a batch can schedule that consumer. This configuration check is not a GPU serve run. + ### Measured head/upstream separation, not a precision fix A throwaway probe used the preserved mainline CPU backend From cd034285a0354eba35d82c82b50905a365b8f113 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:56:08 +0900 Subject: [PATCH 81/85] Document upstream split scratch compatibility and host proof --- docs/research/flint-attach.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 9eb58065d..df975eb14 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -97,6 +97,16 @@ At source `b0fc05e5349330927ceb33ce6f30e584f25ecf4d`, the typed GGUF The case is metadata-only and creates no CUDA context; oxide owns its linking. It is not fresh NVIDIA execution of the current-main stream/launch paths. +Pinned upstream `95a483fc242d00dec7987efe4a05e24b5f651c1f` subsequently adds the +split gate/up path that materializes f32 SwiGLU for a down that reads f32. +The obsolete split-to-f32 refusal is removed, not re-pinned; mismatched gate/up +forms still refuse by semantic Shape/caller/type data. IQ4_XS follows the shared +q8x128 predicate in both row dispatch and `writes_h`, so the GLM prompt consumer +chooses the buffer that is actually written instead of allocating/reading a fake f32 row. +At source `5abe1ae66b651446938e334331c06683cef793a4`, the actual block metadata +consumers passed5 cases,0 failed/ignored,34 filtered, including both IQ4 down forms. +These host cases are not execution evidence for the remaining GPU cases. + On 2026-10-26, source `203f5125a3c48a5b700c6f43318cfcb9c77ff169` executed the owning DS41 library on Hikari GPU0 (RTX3090 cc8.6, driver595.91.07):38 passed,0 failed,0 ignored, 28.61s, including the named IQ4 consumer above. Rust1.99.1 / pinned cuda-oxide9df9e7ce From 462213ca4afaaa7757644567eecd5976302a7833 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:53:07 +0900 Subject: [PATCH 82/85] Fix Qwen tier descriptor down-form bounds --- crates/gpu-gates/src/bin/shared/qwen3moe_place.rs | 4 ++-- crates/model/src/arch/qwen35moe/place.rs | 11 ++++++----- crates/model/src/arch/qwen3moe/place.rs | 11 ++++++----- crates/model/tests/qwen3_bp_plan.rs | 4 ++-- docs/research/flint-attach.md | 8 ++++++-- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs b/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs index bb38199a7..74d7c093a 100644 --- a/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs +++ b/crates/gpu-gates/src/bin/shared/qwen3moe_place.rs @@ -271,8 +271,8 @@ fn laid( [stage] => workstation::placed_with_arena(*stage, layers, arena), [stage, tier] => { let batch = match inputs { - Inputs::Qwen3(i) => q3::tier_batch(&i.hp), - Inputs::Qwen35(i) => q35::leg_tier_batch(&i.spec), + Inputs::Qwen3(i) => q3::tier_batch(i), + Inputs::Qwen35(i) => q35::leg_tier_batch(i), }?; let mut machine = workstation::plan_tiers(layers, *stage, &[*tier], None, batch)?; q3::stage_placed(&mut machine, arena); diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index 7433ccbc3..c85d9bbb0 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -52,7 +52,7 @@ use super::hparams::Kind; use super::{mtp, names}; use crate::arch::{architecture_is, coverage}; use crate::placement::accepts::{ - Program, leg_card_format, program_of, qwen38_down, qwen38_gate_up, routed_rule, + Program, leg_card_format, leg_down_forms, program_of, qwen38_down, qwen38_gate_up, routed_rule, }; use crate::placement::ctx::{CtxLimit, PastLimit, check_slot_ctx}; use crate::placement::ctx_ask::PlanRefusal; @@ -1498,18 +1498,19 @@ pub const LEG_TIER_CHUNK: u64 = 8; /// card's staging, one chunk of the leg's scratch and the host's rows, for /// blocks of the host union's columns. [`tier_batch`] is the qwen4exp plan's /// figure. -pub fn leg_tier_batch(spec: &ModelSpec) -> Result { +pub fn leg_tier_batch(inputs: &PlanInputs) -> Result { + let spec = &inputs.spec; let moe = spec.routed().ok_or_else(|| PlacementError::Metadata { key: "Qwen3.6 expert tier".into(), detail: "a dense model has no routed experts for an expert tier".into(), })?; let (expert_ff, n_used) = (u64::from(moe.expert_ff), u64::from(moe.top_k)); - workstation::tier_leg_batch_bytes( + workstation::try_tier_leg_batch_bytes_of( u64::from(spec.hidden), expert_ff, n_used, - LEG_TIER_CHUNK, - crate::moe::UNION_MAX_COLS as u64, + (LEG_TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), + leg_down_forms(Program::Qwen35Body, &inputs.model), ) } diff --git a/crates/model/src/arch/qwen3moe/place.rs b/crates/model/src/arch/qwen3moe/place.rs index 8afc5c3ed..7d2aa664a 100644 --- a/crates/model/src/arch/qwen3moe/place.rs +++ b/crates/model/src/arch/qwen3moe/place.rs @@ -19,7 +19,7 @@ use gguf::Split; use super::hparams::Hparams; use super::roles; -use crate::placement::accepts::{Program, routed_rule}; +use crate::placement::accepts::{Program, leg_down_forms, routed_rule}; use crate::placement::ctx_ask::PlanRefusal; use crate::placement::workstation::{self, TierBatchBytes}; use crate::placement::{ @@ -45,13 +45,14 @@ pub const TIER_CHUNK: u64 = 8; /// The expert tier's prompt-batch bytes ([`TierBatchBytes`]) for blocks of /// the host union's columns ([`crate::moe::UNION_MAX_COLS`]): the tier card's /// staging, one chunk of the card leg's scratch and the host's rows. -pub fn tier_batch(hp: &Hparams) -> Result { - workstation::tier_leg_batch_bytes( +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let hp = &inputs.hp; + workstation::try_tier_leg_batch_bytes_of( hp.n_embd as u64, hp.experts.ff as u64, hp.experts.n_used as u64, - TIER_CHUNK, - crate::moe::UNION_MAX_COLS as u64, + (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), + leg_down_forms(PROGRAM, &inputs.model), ) } diff --git a/crates/model/tests/qwen3_bp_plan.rs b/crates/model/tests/qwen3_bp_plan.rs index 64175fc36..da3cf6dc0 100644 --- a/crates/model/tests/qwen3_bp_plan.rs +++ b/crates/model/tests/qwen3_bp_plan.rs @@ -522,7 +522,7 @@ fn qwen36_inputs() -> q35::PlanInputs { fn qwen36_tier(stage: CardSpec, tier: CardSpec) -> Result<(u64, u64, u64), Place35Error> { let inputs = qwen36_inputs(); - let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs.spec)?); + let machine = tiered(Q36_LAYERS, stage, tier, q35::leg_tier_batch(&inputs)?); let plan = inputs.plan_rule( &machine, CTX, @@ -541,7 +541,7 @@ fn qwen3_30b_q8_bp_tier_experts() { Q3_LAYERS, A6000, RTX_3090, - q3::tier_batch(&inputs.hp).expect("valid tier geometry"), + q3::tier_batch(&inputs).expect("valid tier geometry"), ); let plan = inputs .plan(&machine, CTX, &PlanLevers::default()) diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index b05e17b4b..c8697b9f6 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -99,8 +99,12 @@ budget regression exercises this distinction without a GPU, including OS reserve `Result` and construct candidate-selected descriptors. An empty tier selects no down form, matching `LegTier`'s actual allocation instead of advertising unused Q8Act scratch. Qwen3/Qwen35, MiMo and Muse use this checked path. -MiniMax obtains its forms from `accepts::leg_down_forms` under its selected program, -as GLM5Next and Solar do; no fallback format can admit a mismatched triple. +Qwen3/Qwen35 and MiniMax derive their descriptor bounds from the described model's +admitted forms through `accepts::leg_down_forms`, as GLM5Next and Solar do. +The Qwen public wrappers take `PlanInputs`, not geometry alone: a supported +Q8_0, Q5_1 or IQ4_NL candidate must not fail the whole plan just because its +32-value form remains unselected at the tier's budget boundary. +No fallback format can admit a mismatched triple. The checked constructors and non-leg paths share staging and host byte formulas. Malformed geometry and overflow are refused rather than panicking; a dense Qwen3.6 request for an expert tier is a named metadata refusal, not zero expert geometry. From 695018db2b5a7714190afc7b7bb16f08bd33e441 Mon Sep 17 00:00:00 2001 From: Burntgogi <224273819+Burntgogi@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:54:55 +0900 Subject: [PATCH 83/85] Keep Qwen tier input scales disabled in integrated descriptor --- crates/model/src/arch/qwen35moe/place.rs | 1 + crates/model/src/arch/qwen3moe/place.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index c85d9bbb0..9b8ccbff2 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1511,6 +1511,7 @@ pub fn leg_tier_batch(inputs: &PlanInputs) -> Result Result hp.experts.n_used as u64, (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), leg_down_forms(PROGRAM, &inputs.model), + false, ) } From 800700db14acd5d0ccf825547f06b40ca62be091 Mon Sep 17 00:00:00 2001 From: midagedev Date: Tue, 27 Oct 2026 16:08:28 +0900 Subject: [PATCH 84/85] placement: a tier card with no leg descriptor keeps its reserve; the overflow test names its panic Co-Authored-By: Claude Opus 5.5 --- crates/placement/src/kernels.rs | 2 +- crates/placement/src/placement.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index 8a8443f86..5aefc70ec 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -451,7 +451,7 @@ mod tests { } #[test] - #[should_panic] + #[should_panic(expected = "leg_bytes: scratch bytes overflow u64")] fn infallible_leg_sizing_refuses_overflow() { let _ = leg_bytes(1 << 61, 8, 256, 256, DownForms::of(DownAct::Q8Act)); } diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index 2d57f3239..f470d2617 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -4473,7 +4473,7 @@ impl Plan<'_> { if rows.next().is_some() { return Ok(None); } - if name != workstation::TIER_BATCH_RESERVE { + if name != workstation::TIER_BATCH_RESERVE || card.tier_leg.is_none() { return Ok(Some(*bytes)); } let counts = match c.checked_sub(self.machine.cards.len()) { From 49f096d9d685fc6aa36dff8e8aff911ad88a3691 Mon Sep 17 00:00:00 2001 From: midagedev Date: Tue, 27 Oct 2026 18:07:57 +0900 Subject: [PATCH 85/85] card_experts: restore the leg buffers' card_downs test; the documented bloomery-serve builds name flint Co-Authored-By: Claude Opus 5.5 --- crates/gpu/src/card_experts.rs | 12 ++++++++++++ docs/BUILD.md | 8 ++++---- docs/contrib/dgx-spark.md | 2 +- docs/serving.md | 2 +- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index e0cb1359b..a3fd228dd 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -1035,6 +1035,18 @@ mod tests { /// quant: both one-form loads and mixed loads conserve bytes and execute /// step, pass, batch and tier chunks as the same common leg. Host slots /// stay untouched on the tier, including a chunk's short final run. + #[test] + fn the_leg_buffers_hold_the_card_downs_forms() { + let q8 = DownForms::of(DownAct::Q8Act); + let b32 = DownForms::of(DownAct::Q8Blocks32); + assert_eq!(card_downs(DownForms::NONE), q8); + assert_eq!(card_downs(q8), q8); + assert_eq!(card_downs(b32), b32); + assert!(!card_downs(b32).holds(DownAct::Q8Act)); + let both = q8.with(DownAct::Q8Blocks32); + assert_eq!(card_downs(both), both); + } + #[test] #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] fn hw_mixed_card_and_tier_wrappers_conserve_forms_and_rows() { diff --git a/docs/BUILD.md b/docs/BUILD.md index 9a4621005..d875ba3ef 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -85,7 +85,7 @@ This builds the V4.1 CLI, the GPU kernels and the host expert tier into one bina |---|---|---| | `generate_ds41`, `bloomery-chat` | `cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features deepseek41 --release --bin ` | DeepSeek-V4.1-Flash | | `bloomery-serve-ds41`, `bloomery-serve-qwen38` | the same, `--features deepseek41,gemma-embedding2,hf` (`gemma-embedding2` attaches the optional embedding side seat) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next | -| `bloomery-serve` | `… --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve` (`--model ds41\|qwen38\|glm\|qwen3\|mimo2\|solar2\|minimaxm3\|nemotron\|raon1\|decide`; the ds41 and qwen38 seats are the two binaries above; `vision` adds V4.1's `--mmproj`; `checkpoint` opens a Qwen3.8 safetensors checkpoint folder as stored) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next, GLM-5.3-Flash, Qwen3.6-35B-A3B, Qwen3-30B-A3B, MiMo-V2.6-Flash, Solar-Open2-250B, MiniMax-M3, Raon-1-mini-preview, Nemotron 3 Nano 4B, Nemotron 3.5 Bolt 74B-A9B, Nemotron-3.5-Lightning 30B-A3B, Clef-Flash, lev | +| `bloomery-serve` | `… --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve` (`--model ds41\|qwen38\|glm\|qwen3\|mimo2\|solar2\|minimaxm3\|nemotron\|raon1\|decide`; the ds41 and qwen38 seats are the two binaries above; `vision` adds V4.1's `--mmproj`; `checkpoint` opens a Qwen3.8 safetensors checkpoint folder as stored) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next, GLM-5.3-Flash, Qwen3.6-35B-A3B, Qwen3-30B-A3B, MiMo-V2.6-Flash, Solar-Open2-250B, MiniMax-M3, Raon-1-mini-preview, Nemotron 3 Nano 4B, Nemotron 3.5 Bolt 74B-A9B, Nemotron-3.5-Lightning 30B-A3B, Clef-Flash, lev | | `generate_qwen3moe` | `… --features gpu --release --bin generate_qwen3moe` | Qwen3.8-Flash-Next, Qwen3.6-35B-A3B, Qwen3-30B-A3B | | `generate_glm5next` | `… --features glm5next-hf --release --bin generate_glm5next` | GLM-5.3-Flash | | `r8conv` | `cargo build --release -p bloomery-model --bin r8conv` | the V4.1 sidecar | @@ -256,10 +256,10 @@ target/release/bloomery-tokenize -m "$M" --decode -p "$(sed -n 's/^tokens //p' g Flags: `-n N` (default 16), `--ctx C` (default 2048, at most 16,384), `--place a|gate` (default `gate`), `--prefill batch|steps` (default `batch`), `--mode graph|eager`, `--plan` (print the plan and exit before the load), `--logits`, `--time [--warm W]`. -The server is `bloomery-serve --model glm` (build it with `--features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf --bin bloomery-serve`). With `--place` unset it plans `bp` when a second card is visible and keeps it when the plan puts at least 526 experts on that card (the break-even, derived in `docs/cards/glmbp-ab.card`), else it runs `a`; a `place unset` line on stderr names the pick, the tier's experts and the break-even. On the development machine's A6000 + 3090 the `bp` plan holds 1,501 tier experts, and one request (MTP draft on, residency `mid-p0-s1`) decodes 13.0 % ± 2.8 faster than under `a` after a 512-token prompt and prefills it 11.1 % ± 0.5 faster (rig-log 10-06#glmbp-ab); two requests run 35.26 tok/s in total after a 512-token prompt (rig-log 10-06#num3090). At `--place a` and `--place bp` (the 3090 as an expert tier) it runs adaptive residency and the MTP draft by default (`BLOOMERY_RESIDENCY=off` and `BLOOMERY_DRAFT=off` turn them off), and `--plan` prints the plan and those choices and exits before any card is opened: +The server is `bloomery-serve --model glm` (build it with `--features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf --bin bloomery-serve`). With `--place` unset it plans `bp` when a second card is visible and keeps it when the plan puts at least 526 experts on that card (the break-even, derived in `docs/cards/glmbp-ab.card`), else it runs `a`; a `place unset` line on stderr names the pick, the tier's experts and the break-even. On the development machine's A6000 + 3090 the `bp` plan holds 1,501 tier experts, and one request (MTP draft on, residency `mid-p0-s1`) decodes 13.0 % ± 2.8 faster than under `a` after a 512-token prompt and prefills it 11.1 % ± 0.5 faster (rig-log 10-06#glmbp-ab); two requests run 35.26 tok/s in total after a 512-token prompt (rig-log 10-06#num3090). At `--place a` and `--place bp` (the 3090 as an expert tier) it runs adaptive residency and the MTP draft by default (`BLOOMERY_RESIDENCY=off` and `BLOOMERY_DRAFT=off` turn them off), and `--plan` prints the plan and those choices and exits before any card is opened: ```sh -cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve +cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf --release --bin bloomery-serve BLOOMERY_REF_MODEL="$M" target/release/bloomery-serve --model glm --place a --ctx 4096 --host 127.0.0.1 --port 8080 ``` @@ -310,7 +310,7 @@ The same binary opens the Qwen3-30B-A3B-Instruct-2507 `Q4_K_M` file (`unsloth/Qw [Clef-Flash](https://huggingface.co/Cloudflare/clef-flash) is Cloudflare's decision model: a Qwen3.5 backbone and a small joint schema head that scores every allowed option of every question in one prompt pass. bartowski's current GGUF upload is llama.cpp's `clef` layout, which carries the head inside the file; an older `qwen35`-layout file takes the release's own head file. ```sh -cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve +cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf --release --bin bloomery-serve # the backbone on one card (see Picking the card), the head (in the file) on the host target/release/bloomery-serve --hf bartowski/Cloudflare_clef-flash-GGUF:Q8_0 --port 8091 diff --git a/docs/contrib/dgx-spark.md b/docs/contrib/dgx-spark.md index 6f3ce6f4f..6fb6aa40e 100644 --- a/docs/contrib/dgx-spark.md +++ b/docs/contrib/dgx-spark.md @@ -56,7 +56,7 @@ the tested toolkit, answer 7) with LLVM 21.1.8's ARM64 release `llc` named by `C for example: ```sh -cargo oxide build --arch sm_121a -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve +cargo oxide build --arch sm_121a -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve ``` Recipes select `sm_121a` on a GB10 when `BLOOMERY_ARCH` is unset. This enables the existing native FP4 routed prompt diff --git a/docs/serving.md b/docs/serving.md index 4e5238d03..753f23dca 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -699,7 +699,7 @@ In progress: serving on N cards for every model; Clef from the IQ and Q2_K files ## EmbeddingGemma 2 images A side seat serves the `gemma-embedding2` encoder beside a generative model. Build `bloomery-serve` with -`--features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,vision` through `cargo oxide`, and add these flags +`--features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,vision` through `cargo oxide`, and add these flags to the chat model's command: ```text