diff --git a/Cargo.lock b/Cargo.lock index 3fd1ac1d2..1a2f81b08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,6 +83,7 @@ dependencies = [ "bloomery-gguf", "bloomery-gpu", "bloomery-gpu-deepseek41", + "bloomery-gpu-flint", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-minimaxm3", @@ -206,6 +207,20 @@ dependencies = [ "cuda-host", ] +[[package]] +name = "bloomery-gpu-flint" +version = "0.1.0" +dependencies = [ + "bloomery-gguf", + "bloomery-gpu", + "bloomery-gpu-deepseek41", + "bloomery-levers", + "bloomery-model", + "bloomery-models", + "bloomery-runtime", + "cuda-core", +] + [[package]] name = "bloomery-gpu-gates" version = "0.3.9" @@ -217,6 +232,7 @@ dependencies = [ "bloomery-gguf", "bloomery-gpu", "bloomery-gpu-deepseek41", + "bloomery-gpu-flint", "bloomery-gpu-gemma-embedding2", "bloomery-gpu-gemma4a", "bloomery-gpu-glm5next", diff --git a/Cargo.toml b/Cargo.toml index f04237316..39373b849 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/qdot", "crates/gguf", "crates/models", "crates/model", "crates/model-hf", "crates/placement", "crates/threads", "crates/levers", "crates/verbs", "crates/gpu", "crates/gpu-deepseek41", "crates/gpu-glm5next", "crates/gpu-mimo2", "crates/gpu-solar2", "crates/gpu-minimaxm3", "crates/gpu-gemma-embedding2", "crates/gpu-gates", "crates/engram", "crates/engram-lab", "crates/sampler", "crates/tokenizer", "crates/serve", "crates/jinja", "crates/vision", "crates/gpu-vision", "crates/gpu-gemma4a", "crates/refset", "crates/runtime", "crates/app", "crates/decision", "crates/embed", "crates/hf", "crates/safetensors", "crates/os", "crates/device", "crates/metal", "crates/record", "crates/chain"] +members = ["crates/qdot", "crates/gguf", "crates/models", "crates/model", "crates/model-hf", "crates/placement", "crates/threads", "crates/levers", "crates/verbs", "crates/gpu", "crates/gpu-deepseek41", "crates/gpu-glm5next", "crates/gpu-mimo2", "crates/gpu-solar2", "crates/gpu-minimaxm3", "crates/gpu-flint", "crates/gpu-gemma-embedding2", "crates/gpu-gates", "crates/engram", "crates/engram-lab", "crates/sampler", "crates/tokenizer", "crates/serve", "crates/jinja", "crates/vision", "crates/gpu-vision", "crates/gpu-gemma4a", "crates/refset", "crates/runtime", "crates/app", "crates/decision", "crates/embed", "crates/hf", "crates/safetensors", "crates/os", "crates/device", "crates/metal", "crates/record", "crates/chain"] # 의도적으로 컴파일되지 않는 재현기다(MUL-7). 워크스페이스에 넣으면 --workspace가 영구히 빨갛다. exclude = ["crates/oxide-ice-unroll", "crates/oxide-ice-unroll/variants/unroll_smoke", "fuzz", "labs/hip-native"] diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index b828c35a2..38f3d0dd0 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -29,6 +29,7 @@ bloomery-gpu-mimo2 = { path = "../gpu-mimo2", optional = true } bloomery-gpu-minimaxm3 = { path = "../gpu-minimaxm3", optional = true } # The Solar-Open2 session (`arch::solar2`). bloomery-gpu-solar2 = { path = "../gpu-solar2", optional = true } +bloomery-gpu-flint = { path = "../gpu-flint", optional = true } # The levers the sessions' own configurations carry (the plan's card budget, the host set's load # settings). The crate has no dependency of its own, so every session reads it. bloomery-levers = { path = "../levers" } @@ -41,3 +42,4 @@ glm5next = ["dep:bloomery-gpu-glm5next"] mimo2 = ["dep:bloomery-gpu-mimo2"] minimaxm3 = ["dep:bloomery-gpu-minimaxm3"] solar2 = ["dep:bloomery-gpu-solar2"] +flint = ["dep:bloomery-gpu-flint"] diff --git a/crates/app/src/arch/flint.rs b/crates/app/src/arch/flint.rs new file mode 100644 index 000000000..5587fe585 --- /dev/null +++ b/crates/app/src/arch/flint.rs @@ -0,0 +1,76 @@ +//! Flint's session over its actual attention-ring body and common hybrid planner. + +use std::num::NonZeroUsize; + +use bloomery_gpu::head::Head; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError, GpuModel}; +use bloomery_gpu_flint::Body; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::PlanInputs; +use model::placement::Plan; + +use super::{Batch, ResidentSequences}; + +const WHAT: &str = "flint session"; + +/// A Flint session, with resident slots backed by the body's own stores. +pub type FlintSession = crate::Session; + +impl Batch for Body { + type Inputs = PlanInputs; + type Sequences = ResidentSequences; + + const WHAT: &'static str = self::WHAT; + + fn read(file: &Split) -> Result { + PlanInputs::read(file).map_err(|e| GpuError::plan(WHAT, e)) + } + + fn configure(inputs: &mut PlanInputs, cfg: &super::HybridCfg) -> Result<(), GpuError> { + model::arch::flint::place::configure_group(inputs, cfg.group) + .map_err(|e| GpuError::plan(WHAT, e)) + } + + fn open_placed( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + host: HostCfg, + residency: Residency, + slots: NonZeroUsize, + ) -> Result, GpuError> { + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) + } + + fn batch( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, + ids: &[u32], + at: u32, + ) -> Result { + bloomery_gpu_flint::prefill::prefill(gpu, w, body, head, ids, at) + } + + fn group(m: &mut GpuModel, g: usize) -> Result { + bloomery_gpu_flint::set_prefill_group(m, g) + } + + fn keepable(m: &GpuModel, n: u32) -> u32 { + m.body(WHAT) + .map_or(0, |b| b.keepable_at(m.pos(), n.min(m.pos()))) + } + + fn cut(m: &mut GpuModel, n: u32) -> Result<(), GpuError> { + if n == 0 { + return m.reset(); + } + let pos = m.pos(); + m.body_parts(WHAT)?.2.stand_for_cut(pos); + m.rollback(n) + } +} diff --git a/crates/app/src/arch/minimaxm3/mod.rs b/crates/app/src/arch/minimaxm3/mod.rs index ba4cadc72..765a0ce3d 100644 --- a/crates/app/src/arch/minimaxm3/mod.rs +++ b/crates/app/src/arch/minimaxm3/mod.rs @@ -9,8 +9,8 @@ //! every layer's key and value rows at those positions, every sparse layer's //! index keys, and the last position's logits. Every store is a plane of //! positions a later position never reads before it writes, so a cut keeps -//! every held position. This session holds one sequence; the serve seat parks -//! more ([`bloomery_gpu_minimaxm3`]'s `Slots`). +//! every held position. The plan counts the configured resident sequences; +//! [`crate::Session::add_slots`] parks them through the body's actual `Slots`. //! //! The session's shape is the common owner's ([`super::Batch`]): what the //! placement plans, how the prompt runs and how a cut is kept are shared with @@ -37,6 +37,7 @@ const WHAT: &str = "minimaxm3 session"; impl Batch for Body { type Inputs = PlanInputs; + type Sequences = super::ResidentSequences; const WHAT: &'static str = self::WHAT; @@ -52,8 +53,9 @@ impl Batch for Body { inputs: &PlanInputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError> { - Body::open_placed_slots(file, plan, inputs, 0, host, residency, NonZeroUsize::MIN) + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) } fn batch( diff --git a/crates/app/src/arch/mod.rs b/crates/app/src/arch/mod.rs index 04b2b3bd2..bff1125c4 100644 --- a/crates/app/src/arch/mod.rs +++ b/crates/app/src/arch/mod.rs @@ -8,10 +8,12 @@ //! and the cut rule — and [`Batch`] holds the rest: the [`Open`] the placement //! plans by, the [`Prompt`] the batch runs and the [`Keep`] the cut follows. +use std::num::NonZeroUsize; + use bloomery_gpu::head::Head; use bloomery_gpu::host::served::prompt_pass; use bloomery_gpu::host::swap::Residency; -use bloomery_gpu::model::{ChainBody, Rollback}; +use bloomery_gpu::model::{ChainBody, Rollback, Slots}; use bloomery_gpu::weights::Weights; use bloomery_gpu::{Gpu, GpuError, GpuModel}; use bloomery_levers::HostCfg; @@ -24,6 +26,8 @@ use crate::{Keep, Open, Prompt}; #[cfg(feature = "deepseek41")] pub mod deepseek41; +#[cfg(feature = "flint")] +pub mod flint; #[cfg(feature = "glm5next")] pub mod glm5next; #[cfg(feature = "mimo2")] @@ -56,6 +60,8 @@ pub struct HybridCfg { pub residency: Residency, /// The batches a prompt group runs layer by layer. pub group: usize, + /// The resident sequences charged by the plan before any allocation. + pub slots: NonZeroUsize, } /// The plan inputs of a hybrid body: the family's own hyperparameters under @@ -71,6 +77,7 @@ pub trait HybridInputs { machine: &'a Machine, ctx: u64, levers: &PlanLevers, + slots: NonZeroUsize, what: &'static str, ) -> Result, GpuError>; } @@ -85,9 +92,11 @@ impl HybridInputs for PlanInputs { machine: &'a Machine, ctx: u64, levers: &PlanLevers, + slots: NonZeroUsize, what: &'static str, ) -> Result, GpuError> { - PlanInputs::plan(self, machine, ctx, levers).map_err(|e| GpuError::plan(what, e)) + self.plan_with_slots(machine, ctx, levers, slots) + .map_err(|e| GpuError::plan(what, e)) } } @@ -98,6 +107,8 @@ impl HybridInputs for PlanInputs { pub trait Batch: ChainBody + Rollback + Sized { /// The plan inputs the body is loaded by. type Inputs: HybridInputs; + /// The sequence capability the body actually implements. + type Sequences: SequenceSlots; /// The session's name in its refusals. const WHAT: &'static str; @@ -105,6 +116,11 @@ pub trait Batch: ChainBody + Rollback + Sized { /// The plan inputs of `file`. fn read(file: &Split) -> Result; + /// Price configuration-dependent buffers before placement; unchanged for a fixed schedule. + fn configure(_inputs: &mut Self::Inputs, _cfg: &HybridCfg) -> Result<(), GpuError> { + Ok(()) + } + /// The body resident by `plan`, which `inputs` made: the layers the plan /// puts on the card, the host tier over the routed ones. fn open_placed( @@ -113,6 +129,7 @@ pub trait Batch: ChainBody + Rollback + Sized { inputs: &Self::Inputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError>; /// The prompt batch of `ids` from `at`: the head's row after the last. @@ -154,6 +171,10 @@ impl Open for B { B::read(file) } + fn configure_inputs(inputs: &mut B::Inputs, cfg: &HybridCfg) -> Result<(), GpuError> { + B::configure(inputs, cfg) + } + fn layer_count(inputs: &B::Inputs) -> usize { inputs.layers() } @@ -169,8 +190,9 @@ impl Open for B { ctx: usize, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, B::WHAT)?; let ctx = one_card_ctx(B::WHAT, machine, ctx)?; - inputs.plan_at(machine, ctx, &cfg.place, B::WHAT) + inputs.plan_at(machine, ctx, &cfg.place, cfg.slots, B::WHAT) } fn open( @@ -179,7 +201,8 @@ impl Open for B { plan: &Plan<'_>, cfg: &HybridCfg, ) -> Result, GpuError> { - B::open_placed(file, plan, inputs, cfg.host, cfg.residency) + >::check(cfg.slots, B::WHAT)?; + B::open_placed(file, plan, inputs, cfg.host, cfg.residency, cfg.slots) } /// The configuration's group; the units it needs are made here. @@ -222,6 +245,81 @@ impl Keep for B { } } +/// The sequence operations a common hybrid seat may offer. +pub trait SequenceSlots { + /// Whether several resident sequences are backed by body-owned stores. + const RESIDENT: bool; + /// Refuse an unsupported count before planning or allocation. + fn check(slots: NonZeroUsize, what: &'static str) -> Result<(), GpuError>; + /// Park the planned sequences through the session's actual slot operations. + fn add(s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError>; + /// The number of resident sequences the session currently holds. + fn count(s: &crate::Session) -> usize; + /// Select the actual sequence every following call acts on. + fn select(s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError>; +} + +/// A body that holds one sequence; other slots are refused, not simulated. +pub struct SingleSequence; + +impl SequenceSlots for SingleSequence { + const RESIDENT: bool = false; + + fn check(slots: NonZeroUsize, what: &'static str) -> Result<(), GpuError> { + if slots.get() != 1 { + return Err(GpuError::Shape { + what, + detail: format!( + "the body holds one sequence, not {} resident slots", + slots.get() + ), + }); + } + Ok(()) + } + + fn add(_s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError> { + >::check(slots, "single-sequence session").map_err(Into::into) + } + + fn count(_s: &crate::Session) -> usize { + 1 + } + + fn select(_s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError> { + if slot == 0 { + Ok(()) + } else { + Err(crate::SessionError::Refused(format!( + "select_slot {slot}: the body holds one sequence" + ))) + } + } +} + +/// Resident sequences backed by the body's actual parked stores. +pub struct ResidentSequences; + +impl> SequenceSlots for ResidentSequences { + const RESIDENT: bool = true; + + fn check(_slots: NonZeroUsize, _what: &'static str) -> Result<(), GpuError> { + Ok(()) + } + + fn add(s: &mut crate::Session, slots: NonZeroUsize) -> Result<(), crate::SessionError> { + s.add_slots(slots.get()) + } + + fn count(s: &crate::Session) -> usize { + s.slots() + } + + fn select(s: &mut crate::Session, slot: usize) -> Result<(), crate::SessionError> { + s.select_slot(slot) + } +} + /// `ctx` positions as a plan takes them, for a body whose chain runs on one /// card: refused by name (`what` the body's session) when `machine` puts the /// layers on any other number of cards or `ctx` passes u64. diff --git a/crates/app/src/arch/musespark/mod.rs b/crates/app/src/arch/musespark/mod.rs index 91194dd31..a66a0ca01 100644 --- a/crates/app/src/arch/musespark/mod.rs +++ b/crates/app/src/arch/musespark/mod.rs @@ -47,8 +47,9 @@ impl Open for Body { ctx: usize, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, WHAT)?; let ctx = one_card_ctx(WHAT, machine, ctx)?; - inputs.plan_at(machine, ctx, &cfg.place, WHAT) + inputs.plan_at(machine, ctx, &cfg.place, cfg.slots, WHAT) } /// The model resident by `plan`, on the plan's one card, under the @@ -60,6 +61,7 @@ impl Open for Body { plan: &Plan<'_>, cfg: &HybridCfg, ) -> Result, GpuError> { + >::check(cfg.slots, WHAT)?; GpuModel::::open_placed_with(file, plan, 0, cfg.host, cfg.residency) } diff --git a/crates/app/src/arch/solar2/mod.rs b/crates/app/src/arch/solar2/mod.rs index 482533bfe..56eac5094 100644 --- a/crates/app/src/arch/solar2/mod.rs +++ b/crates/app/src/arch/solar2/mod.rs @@ -40,6 +40,7 @@ const WHAT: &str = "solar2 session"; impl Batch for Body { type Inputs = PlanInputs; + type Sequences = super::SingleSequence; const WHAT: &'static str = self::WHAT; @@ -53,8 +54,9 @@ impl Batch for Body { inputs: &PlanInputs, host: HostCfg, residency: Residency, + slots: NonZeroUsize, ) -> Result, GpuError> { - Body::open_placed_slots(file, plan, inputs, 0, host, residency, NonZeroUsize::MIN) + Body::open_placed_slots(file, plan, inputs, 0, host, residency, slots) } fn batch( diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 5eee1b184..df3396b79 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -161,6 +161,12 @@ pub trait Open: ChainBody + Sized { /// Read `file`'s headers. fn inputs(file: &Split) -> Result; + /// Apply load settings that change planned buffers, before the plan borrows its inputs. + /// Bodies with no configuration-dependent buffers keep the header-derived inputs. + fn configure_inputs(_inputs: &mut Self::Inputs, _cfg: &Self::Cfg) -> Result<(), GpuError> { + Ok(()) + } + /// The layer count a placement's machine is laid out for. fn layer_count(inputs: &Self::Inputs) -> usize; @@ -303,7 +309,8 @@ impl Loaded { log: &mut impl OpenLog, edit: impl FnOnce(&B::Inputs, &mut Plan<'_>) -> Result<(), SessionError>, ) -> Result>, SessionError> { - let inputs = B::inputs(&file)?; + let mut inputs = B::inputs(&file)?; + B::configure_inputs(&mut inputs, &args.cfg)?; let machine = (args.machine)(B::layer_count(&inputs)); let mut plan = B::plan(&inputs, &machine, args.ctx, &args.cfg)?; edit(&inputs, &mut plan)?; diff --git a/crates/gpu-deepseek41/src/experts_mxfp4.rs b/crates/gpu-deepseek41/src/experts_mxfp4.rs index 6c2888207..51f15c799 100644 --- a/crates/gpu-deepseek41/src/experts_mxfp4.rs +++ b/crates/gpu-deepseek41/src/experts_mxfp4.rs @@ -182,7 +182,9 @@ mod dflash_kernels { // N_EXPERT = n_expert <= bias.len() by the launch contract; score and // sel_v are block-shared arrays of N_EXPERT entries; the barrier below // publishes them. - unsafe { publish_n::(probs.as_mut_ptr(), bias, e0, score, sel_v, tid, fault) }; + unsafe { + publish_n::(probs.as_mut_ptr(), Some(bias), e0, score, sel_v, tid, fault) + }; thread::sync_threads(); if tid < 32 { diff --git a/crates/gpu-deepseek41/src/prompt_walk.rs b/crates/gpu-deepseek41/src/prompt_walk.rs index 905cba000..5b182c96f 100644 --- a/crates/gpu-deepseek41/src/prompt_walk.rs +++ b/crates/gpu-deepseek41/src/prompt_walk.rs @@ -214,7 +214,14 @@ impl BatchBufs { })?; let stream = gpu.stream(); let sets = group_sets(group); - Ok(Self { + let expected = model::placement::kernels::ffn::PromptWalk { + hidden: embd as u64, + sets: sets as u64, + } + .bytes(t_max as u64) + .and_then(|bytes| usize::try_from(bytes).ok()) + .ok_or_else(|| GpuError::shape("prompt batch buffers", "buffer geometry overflows"))?; + let buffers = Self { group, t_max, units: (0..sets) @@ -225,7 +232,14 @@ impl BatchBufs { stream: PromptStream::default(), facts: None, timing: None, - }) + }; + if buffers.bytes() != expected { + return Err(GpuError::shape( + "prompt batch buffers", + format!("allocated {} bytes, planned {expected}", buffers.bytes()), + )); + } + Ok(buffers) } /// What the next prompt calls move (`BLOOMERY_XSTREAM`), behind the guards every family shares diff --git a/crates/gpu-deepseek41/src/shared_routed.rs b/crates/gpu-deepseek41/src/shared_routed.rs index 29bfb4742..f88b84f25 100644 --- a/crates/gpu-deepseek41/src/shared_routed.rs +++ b/crates/gpu-deepseek41/src/shared_routed.rs @@ -4,20 +4,17 @@ //! [`shadow`], [`back`]) or a unit's rows (the batch: [`Routed::front_rows`] and its three). Both feeds leave the same //! bits but the shared expert's rows of a unit of [`GATED_FROM`] rows or more, which its GEMMs write. //! -//! Every layer is a routed layer with one shared expert, so the block has no dense arm. The router, the handoff, the -//! go, the card experts' call and the wait and add around the host leg are the common routed block -//! ([`bloomery_gpu::routed_block`]); this file holds the shared expert and the base the card sum is added to. Its -//! parts around the host leg: -//! - the front: `ffn_norm` RMS of the rows of the residual after the mixer `x1`, and the router over them: the -//! scores, the selection bias (the choice only), the top `R::N_USED`, the weights from the unbiased scores -//! normalized and scaled. The step goes on to the handoff into the host-mapped image and the go; four launches. -//! The batch downloads the rows, weights and ids to the host ([`Routed::download`]). -//! - the shadow: while the host computes, the shared expert, its output summed into `x1` (`x1s`), then the card -//! experts the slot map puts on the card with `x1s` as the base their sum is added to; none of the latter on a -//! layer with no card experts. The base is `x1 + shared` before the card sum is added, in the step and the batch -//! alike. -//! - the back: the wait, then the add of the host's routed sum and the shadow's `card + x1s` (`x1s` alone on a -//! layer with no card experts). One launch after the wait. +//! Dense leading layers run the common dense block; routed layers have one ungated shared expert. The router, +//! handoff, go, card experts and host wait are the common routed block ([`bloomery_gpu::routed_block`]). +//! [`FfnConfig`] resolves the residual association and expert-input range protection once at load: +//! - the front normalizes the attention residual, scores and selects through `R`'s normalization and bias +//! policy, then hands the normed row and picks to the host; +//! - the shadow computes the shared expert and card experts while the host runs. The default saves +//! `residual + shared` as the base. [`ResidualOrder::RoutedSharedResidual`] keeps the residual separate; +//! - the back joins the host and card routed sums. The default adds the host to the card-plus-base rows; +//! the other association computes `(routed + shared) + residual`, after the complete routed sum is ready. +//! +//! The step and prompt batch use the same loaded association, including tier-card joins. //! //! A load whose slot map names an expert tier card ([`TierSide`]) runs a layer the tier holds experts of as a tier //! layer: the front's handoff also writes the tier's image and each slot's tier place and the go is the host's and the @@ -28,15 +25,15 @@ //! [`Routed::join_rows`] (the host leg's [`bloomery_gpu::host::BatchLeg::join_tiered`] hands the tier's rows to the //! join, then [`Routed::back_rows`]). //! -//! What a family fixes is its router instance, its [`CardFamily`] (stack names, counts) and what its description -//! says: the routed experts' and the shared expert's activation ([`rule_of`]: a SwiGLU clamp or SwiGLU-OAI), the -//! weights' scale and widths, which every layer agrees on ([`describe`]). The shared expert's down is Q6_K or Q4_K -//! by layer, so its block plan is read per layer. The shared expert's Q8_0 pair kernel runs the clamp alone, so a -//! SwiGLU-OAI shared expert on Q8_0 weights is refused by name at the first launch. +//! What a family fixes is its router instance, its [`CardFamily`] (stack names, counts), the loaded arithmetic +//! facts, and its description: the routed and shared activations ([`rule_of`]), scales and widths +//! ([`describe`]). Shared and dense block plans are read per layer and support mixed sites, including IQ4_XS. +//! A Q8_0 shared pair runs the clamp alone; a SwiGLU-OAI shared expert on it is refused before launch. use std::sync::Arc; use bloomery_gpu::card_experts::{CardBatch, CardExperts, CardFamily}; +use bloomery_gpu::card_leg::ExpertInput; use bloomery_gpu::gemm::{Gemm32Kernels, GemmInput, GemmKernels, GemmRoute}; use bloomery_gpu::host::leg_tier::TierSide; use bloomery_gpu::host::run::HostRun; @@ -58,9 +55,13 @@ use crate::chain::ffn::FfnBatchKernels; use crate::experts::ExpertKernels; use crate::span::{span, span_mut}; use crate::tier_join::{self, TierBatch, TierSlots}; -use cuda_core::{CudaStream, DeviceBuffer}; +use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; +use cuda_device::float::add_rn_f32; +use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread}; +use cuda_host::cuda_module; use gguf::Split; use model::arch::names; +use model::placement::kernels::ffn; use models::{Act as ActRule, Ffn, LayerSpec, Score}; /// What the refusals of this file name. @@ -70,7 +71,125 @@ const WHAT: &str = "shared routed block"; /// GEMM a site): below it `enqueue_block_cols` runs the step's gemvs on each /// column (at most 8 columns), so a unit of up to 8 rows writes the step's /// bits and from it a launch count of 6 replaces 4 a column. -pub const GATED_FROM: usize = 9; +pub const GATED_FROM: usize = ffn::GATED_FROM as usize; + +/// The association of a routed block's shared output and attention residual. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ResidualOrder { + /// The shared output joins the residual before the routed sum. + #[default] + ResidualSharedRouted, + /// The complete routed sum joins the shared output before the residual. + RoutedSharedResidual, +} + +/// Arithmetic facts resolved once when the routed block is loaded. +#[derive(Clone, Copy, Debug)] +pub struct FfnConfig { + pub residual: ResidualOrder, + pub expert_input: ExpertInput, +} + +impl Default for FfnConfig { + fn default() -> Self { + Self { + residual: ResidualOrder::ResidualSharedRouted, + expert_input: ExpertInput::Direct, + } + } +} + +#[cuda_module] +mod residual_kernels { + use super::*; + + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + host.len() >= n, + card.len() >= n, + shared.len() >= n, + residual.len() >= n, + dest.len() >= n + ) + )] + pub fn routed_shared_residual( + host: &[f32], + card: &[f32], + shared: &[f32], + residual: &[f32], + has_card: u32, + n: u32, + mut dest: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= n as usize { + return; + } + // SAFETY: the launch bounds every input and output by n; i is guarded. + unsafe { + let mut routed = *host.get_unchecked(i); + if has_card != 0 { + routed = add_rn_f32(routed, *card.get_unchecked(i)); + } + let ffn = add_rn_f32(routed, *shared.get_unchecked(i)); + *dest.get_unchecked_mut(i) = add_rn_f32(ffn, *residual.get_unchecked(i)); + } + } +} + +struct ResidualKernels { + module: residual_kernels::LoadedModule, +} + +impl ResidualKernels { + fn load(ctx: &Arc) -> Result { + // SAFETY: this package owns the embedded bundle; enqueue checks its contract. + let module = unsafe { bloomery_gpu::shared_module!(residual_kernels, ctx)? }; + Ok(Self { module }) + } + + fn enqueue( + &self, + stream: &CudaStream, + (host, card): (&DeviceBuffer, Option<&DeviceBuffer>), + (shared, residual): (&DeviceBuffer, &DeviceBuffer), + n: usize, + dest: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + if n == 0 + || host.len() < n + || card.is_some_and(|c| c.len() < n) + || shared.len() < n + || residual.len() < n + || dest.len() < n + { + return Err(GpuError::shape( + WHAT, + "routed residual join outside its buffers", + )); + } + let n = bloomery_gpu::launch_u32(WHAT, "residual values", n)?; + let prep = self + .module + .prepare_routed_shared_residual(LaunchConfig1D::new(n.div_ceil(256), 256, 0))?; + self.module.routed_shared_residual( + stream, + &prep, + host, + card.unwrap_or(host), + shared, + residual, + u32::from(card.is_some()), + n, + dest, + )?; + Ok(()) + } +} /// What the load takes of the model besides the description and the file. #[derive(Clone, Copy, Debug)] @@ -146,16 +265,15 @@ impl DenseFfn { } } -/// Layer `l`'s routed block for the common one: its norm, router and bias -/// tensors (every layer has a bias) and its row in `map`. -fn route_layer(map: &SlotMap, l: usize) -> Result { +/// Layer `l`'s norm, router, optional selection bias and row in `map`. +fn route_layer(map: &SlotMap, l: usize, bias: bool) -> Result { let row_off = map .row_offset(l) .ok_or_else(|| GpuError::shape(WHAT, format!("layer {l}: no row in the slot map")))?; Ok(RouteLayer { norm: names::ffn_norm(l), router: names::ffn_gate_inp(l), - bias: Some(names::exp_probs_b(l)), + bias: bias.then(|| names::exp_probs_b(l)), row_off, }) } @@ -169,6 +287,8 @@ struct Kernels { experts: ExpertKernels, /// The K-quant block's kernels: the shared expert is K-quant in this file. block: Option, + /// Loaded only for the routed-plus-shared, then residual association. + residual: Option, } /// The step's buffers: the router's outputs and each slot's place, and the @@ -191,6 +311,31 @@ struct SharedWide { u: DeviceBuffer, } +fn wide_layout( + (gate_up, down): (&[SiteTy], &[SiteTy]), + (embd, ff): (usize, usize), +) -> ffn::WideBlock { + let forms = |types: &[SiteTy]| { + types.iter().fold(ffn::WideForms::default(), |forms, ty| { + forms.with(ty.reads()) + }) + }; + ffn::WideBlock { + hidden: embd as u64, + ff: ff as u64, + input: forms(gate_up).without_f32(), + down: forms(down), + } +} + +fn device_forms(forms: ffn::WideForms) -> site::WideForms { + site::WideForms { + q128: forms.quant.q128, + q32: forms.quant.q32, + f32: forms.f32, + } +} + impl SharedWide { /// Buffers for up to `cap` rows over a block of width `ff` on `embd` values, whose gate and up read /// the types `gate_up` and whose down reads the types `down`. Load-time only. @@ -200,11 +345,12 @@ impl SharedWide { (cap, embd, ff): (usize, usize, usize), ) -> Result { let stream = gpu.stream(); + let layout = wide_layout((gate_up, down), (embd, ff)); Ok(SharedWide { gemm: GemmKernels::load(gpu.context())?, route: GemmRoute::new(stream, cap, 1)?, - x: WideActs::new(stream, WideActs::wants_of(gate_up).without_f32(), cap, embd)?, - h: WideActs::new(stream, WideActs::wants_of(down), cap, ff)?, + x: WideActs::new(stream, device_forms(layout.input), cap, embd)?, + h: WideActs::new(stream, device_forms(layout.down), cap, ff)?, g: DeviceBuffer::zeroed(stream, cap * ff)?, u: DeviceBuffer::zeroed(stream, cap * ff)?, }) @@ -325,7 +471,21 @@ impl BatchBufs { let z = |n: usize| DeviceBuffer::::zeroed(stream, n); let (shared_gate_up, shared_down) = block_types(per.iter().flatten().map(|l| &l.plan)); let (dense_gate_up, dense_down) = block_types(dense.iter().flatten().map(|l| &l.plan)); - Ok(BatchBufs { + let layout = ffn::RoutedBatch { + hidden: embd as u64, + experts: R::N_EXPERT as u64, + used: R::N_USED as u64, + shared: wide_layout((&shared_gate_up, &shared_down), (embd, ff)), + dense: (dense_ff > 0) + .then(|| wide_layout((&dense_gate_up, &dense_down), (embd, dense_ff))), + card: (card.any() || card.any_tier()).then(|| card.batch_layout(cap)), + tiered, + }; + let expected = layout + .bytes(cap as u64) + .and_then(|bytes| usize::try_from(bytes).ok()) + .ok_or_else(|| GpuError::shape(WHAT, "routed batch scratch byte overflow"))?; + let buffers = BatchBufs { cap, normed: z(cap * embd)?, probs: z(cap * R::N_EXPERT)?, @@ -348,7 +508,14 @@ impl BatchBufs { tier: tiered .then(|| TierBatch::new(stream, cap, R::N_USED)) .transpose()?, - }) + }; + if buffers.bytes() != expected { + return Err(GpuError::shape( + WHAT, + "routed batch allocation differs from its common scratch descriptor", + )); + } + Ok(buffers) } fn bytes(&self) -> usize { @@ -448,8 +615,8 @@ struct Described { } /// The description of `layers` as the compiled router, handoff and shared -/// block run it: `R`'s experts and picks, sigmoid scores with a bias and -/// normalized weights, a SwiGLU or SwiGLU-OAI shared expert without a gate. +/// block run it: `R`'s experts, picks, normalization and bias policy; sigmoid scores and a +/// SwiGLU or SwiGLU-OAI shared expert without a gate. /// Anything else is refused by name with its layer. fn describe(layers: &[LayerSpec]) -> Result { let (mut scale, mut expert_ff, mut shared_ff) = (None, None, None); @@ -469,8 +636,8 @@ fn describe(layers: &[LayerSpec]) -> Result || m.top_k as usize != R::N_USED || m.latent.is_some() || r.score != Score::Sigmoid - || !r.bias - || r.norm != models::RouterNorm::SelectedSum + || (!r.bias && !R::OPTIONAL_BIAS) + || r.norm != R::NORM || r.hash || shared.sigmoid_gate { @@ -478,9 +645,11 @@ fn describe(layers: &[LayerSpec]) -> Result WHAT, format!( "layer {l}: {m:?} is not the {}-expert top-{} sigmoid router \ - with a bias, normalized weights and an ungated shared expert", + with {:?} normalization, optional bias={} and an ungated shared expert", R::N_EXPERT, - R::N_USED + R::N_USED, + R::NORM, + R::OPTIONAL_BIAS ), )); } @@ -530,6 +699,28 @@ impl Routed { (layers, map, slots): (&[LayerSpec], &SlotMap, &Arc>), d: RoutedDims, card: CardFamily, + ) -> Result, GpuError> { + Self::new_configured( + gpu, + file, + w, + (layers, map, slots), + d, + card, + FfnConfig::default(), + ) + } + + /// Load the same common block with explicit arithmetic facts. The default keeps the legacy + /// association and expert input unchanged; the selected policies apply to every routed layer. + pub fn new_configured( + gpu: &Gpu, + file: &Split, + w: &Weights, + (layers, map, slots): (&[LayerSpec], &SlotMap, &Arc>), + d: RoutedDims, + card: CardFamily, + config: FfnConfig, ) -> Result, GpuError> { if (card.n_expert, card.n_used) != (R::N_EXPERT, R::N_USED) { return Err(GpuError::shape( @@ -564,7 +755,11 @@ impl Routed { }) .collect::, GpuError>>()?; let routes = (0..layers.len()) - .map(|l| routed(l).then(|| route_layer(map, l)).transpose()) + .map(|l| { + moe_of(layers, l) + .map(|m| route_layer(map, l, m.router.bias)) + .transpose() + }) .collect::, _>>()?; let route_dims = RouteDims { embd: d.embd, @@ -577,6 +772,9 @@ impl Routed { let k = Kernels { g32: Gemm32Kernels::load(ctx)?, experts: ExpertKernels::load(ctx)?, + residual: (config.residual == ResidualOrder::RoutedSharedResidual) + .then(|| ResidualKernels::load(ctx)) + .transpose()?, block: per .iter() .flatten() @@ -606,10 +804,12 @@ impl Routed { }) .transpose()?; let dims = (d.embd, desc.expert_ff); - let card = match tier { + let mut card = match tier { Some(_) => CardExperts::tiered(gpu, w, card, layers, map, dims, 1)?, None => CardExperts::new(gpu, w, card, layers, map, dims, 1)?, }; + card.set_expert_input(gpu, config.expert_input)?; + card.set_add_base(config.residual == ResidualOrder::ResidualSharedRouted); let dense_ff = dense.iter().flatten().map(|l| l.ff).max().unwrap_or(0); let batch = (d.batch_rows > 1) .then(|| { @@ -703,9 +903,8 @@ impl Routed { self.step.route.ids() } - /// Layer `l`'s selection bias, the values the router adds to the scores - /// before it ranks them: the ranking a reference engine's picks are read - /// at. + /// Layer `l`'s selection bias for reference-pick inspection. A layer with no bias is refused by + /// name; execution handles it through the router's optional-bias policy. pub fn route_bias<'a>( &'a self, w: &'a Weights, @@ -766,10 +965,8 @@ fn shared_launch<'a>( (kernels, blk) } -/// Enqueue layer `l`'s shared expert and card experts of the step in its host -/// leg's shadow, after its [`front`]: `x1s` is `x1` plus the shared expert's -/// output, and the card sum plus `x1s` goes to the card experts' `pre` where -/// the layer has any. +/// Enqueue layer `l`'s shared and card experts after [`front`], while the host runs. `x1s` holds the +/// legacy residual-plus-shared base, or the saved residual when the loaded association joins it last. pub fn shadow( gpu: &Gpu, w: &Weights, @@ -786,8 +983,13 @@ pub fn shadow( y: &mut r.step.sh_y, }; enqueue_block_act(gpu, w, &kernels, &mut r.block, (&blk, r.rules.1), io)?; - gpu.elem() - .enqueue_add(stream, x1, &r.step.sh_y, r.d.embd, &mut r.step.x1s)?; + if r.k.residual.is_some() { + span_mut(WHAT, &mut r.step.x1s, 0, r.d.embd)? + .copy_from_device_async(&*span(WHAT, x1, 0, r.d.embd)?, stream)?; + } else { + gpu.elem() + .enqueue_add(stream, x1, &r.step.sh_y, r.d.embd, &mut r.step.x1s)?; + } let rule = r.rules.0; r.route.shadow( (gpu, w), @@ -812,6 +1014,19 @@ pub fn back( ) -> Result<(), GpuError> { let n_tier = r.card.tier_k(l); if n_tier == 0 { + if let Some(join) = &r.k.residual { + hybrid.boundary().enqueue_back_of(gpu.stream(), 0, l)?; + return join.enqueue( + gpu.stream(), + ( + hybrid.boundary().hsum_of(0)?, + r.card.has(l).then(|| r.card.acc()), + ), + (&r.step.sh_y, &r.step.x1s), + r.d.embd, + dest, + ); + } return r.route.back(gpu, hybrid, &r.card, l, (&r.step.x1s, dest)); } const W: &str = "shared routed back (tier layer)"; @@ -827,6 +1042,22 @@ pub fn back( n_tier, n_used: R::N_USED, }; + if let Some(residual) = &r.k.residual { + hybrid.enqueue_tier_back(gpu.stream(), l, 0)?; + let mut join = r.card.step_join(r.step.route.sel()); + tier_join::enqueue_acc( + (gpu.stream(), kernels), + (&mut join, &slots), + (hybrid.tier_rows(0)?, 1), + )?; + return residual.enqueue( + gpu.stream(), + (hybrid.boundary().hsum_of(0)?, Some(join.acc)), + (&r.step.sh_y, &r.step.x1s), + r.d.embd, + dest, + ); + } tier_join::enqueue_step_join( gpu, kernels, @@ -961,10 +1192,8 @@ impl Routed { self.card_rows(gw, lm, (&[], None)) } - /// Layer `l`'s shared expert over the `m` rows [`Routed::front_rows`] - /// wrote: the shared expert's rows (per column below [`GATED_FROM`] rows, - /// on `site::gated` from it) and `x1s = x1 + shared`, the base - /// [`Routed::card_rows`] adds the card sum to. + /// Layer `l`'s shared expert over the `m` normed rows, per column below [`GATED_FROM`] and on + /// `site::gated` from it. Keep the residual-plus-shared base or the separate residual, as loaded. pub fn shared_rows( &mut self, (gpu, w): (&Gpu, &Weights), @@ -1007,8 +1236,14 @@ impl Routed { (&b.normed, &mut b.sh), )?; } - gpu.elem() - .enqueue_add(gpu.stream(), x1, &b.sh, m * n, &mut b.x1s) + if k.residual.is_some() { + span_mut(WHAT, &mut b.x1s, 0, m * n)? + .copy_from_device_async(&*span(WHAT, x1, 0, m * n)?, gpu.stream())?; + Ok(()) + } else { + gpu.elem() + .enqueue_add(gpu.stream(), x1, &b.sh, m * n, &mut b.x1s) + } } /// Layer `l`'s card experts, where the slot map puts any, over the `m` @@ -1097,6 +1332,13 @@ impl Routed { }); }; let slots = tb.slots(n, (n_card, n_tier, R::N_USED)); + if self.k.residual.is_some() { + return tier_join::enqueue_acc( + (gpu.stream(), kernels), + (&mut cb.join(), &slots), + (trows, m), + ); + } tier_join::enqueue_join( gpu, (gpu.stream(), kernels), @@ -1106,10 +1348,8 @@ impl Routed { ) } - /// `x = hsum +` the shadow's rows (the card sum plus `x1s`, or `x1s` on a - /// layer with no card experts) over the unit's `m` rows, after - /// [`Routed::shadow_rows`] and the host sums' upload; on a tier layer after - /// [`Routed::join_rows`], which wrote the sum. + /// Join the uploaded host sums and shadow results over `m` rows with the loaded association. + /// On a tier layer, [`Routed::join_rows`] first forms the raw routed sum over both cards. pub fn back_rows( &self, gpu: &Gpu, @@ -1118,6 +1358,20 @@ impl Routed { ) -> Result<(), GpuError> { let b = self.batch.as_ref().ok_or_else(no_batch)?; b.check(l, m)?; + if let Some(join) = &self.k.residual { + let card = b + .card + .as_ref() + .filter(|_| self.card.sums(l)) + .map(CardBatch::acc); + return join.enqueue( + gpu.stream(), + (hsum, card), + (&b.sh, &b.x1s), + m * self.d.embd, + x, + ); + } let shadowed = match b.card.as_ref().filter(|_| self.card.sums(l)) { Some(cb) => cb.pre(), None => &b.x1s, diff --git a/crates/gpu-deepseek41/src/tier_join.rs b/crates/gpu-deepseek41/src/tier_join.rs index e02e500ba..ddaff2392 100644 --- a/crates/gpu-deepseek41/src/tier_join.rs +++ b/crates/gpu-deepseek41/src/tier_join.rs @@ -53,6 +53,18 @@ pub fn enqueue_join( (join, slots): (&mut TierJoin<'_>, &TierSlots<'_>), (trows, m): (&DeviceBuffer, usize), base: &DeviceBuffer, +) -> Result<(), GpuError> { + enqueue_acc((stream, kernels), (&mut *join, slots), (trows, m))?; + gpu.elem() + .enqueue_add(stream, join.acc, base, m * slots.n, join.pre) +} + +/// Sum both cards' routed slots without adding a base. A caller whose shared output and residual +/// join after the complete routed sum uses this result directly. +pub fn enqueue_acc( + (stream, kernels): (&CudaStream, &FfnBatchKernels), + (join, slots): (&mut TierJoin<'_>, &TierSlots<'_>), + (trows, m): (&DeviceBuffer, usize), ) -> Result<(), GpuError> { let a = CardAccTier { down: join.down, @@ -66,9 +78,7 @@ pub fn enqueue_join( n_tier: slots.n_tier, n_used: slots.n_used, }; - kernels.enqueue_card_acc_tier(stream, &a, join.acc)?; - gpu.elem() - .enqueue_add(stream, join.acc, base, m * slots.n, join.pre) + kernels.enqueue_card_acc_tier(stream, &a, join.acc) } /// Layer `l`'s back of the step on a tier layer at the page's row `row`: the wait for the host and diff --git a/crates/gpu-flint/Cargo.toml b/crates/gpu-flint/Cargo.toml new file mode 100644 index 000000000..2e1678700 --- /dev/null +++ b/crates/gpu-flint/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "bloomery-gpu-flint" +version = "0.1.0" +edition = "2024" +publish = false +license = "MIT" + +[lints] +workspace = true + +[dependencies] +cuda-core.workspace = true +bloomery-gpu = { path = "../gpu" } +bloomery-gpu-deepseek41 = { path = "../gpu-deepseek41" } +bloomery-gguf = { path = "../gguf" } +bloomery-levers = { path = "../levers" } +bloomery-model = { path = "../model" } +bloomery-models = { path = "../models" } +bloomery-runtime = { path = "../runtime" } diff --git a/crates/gpu-flint/examples/trace_flint.rs b/crates/gpu-flint/examples/trace_flint.rs new file mode 100644 index 000000000..b592c2cda --- /dev/null +++ b/crates/gpu-flint/examples/trace_flint.rs @@ -0,0 +1,473 @@ +//! Forced-ID Flint trajectory capture. Diagnostic artifacts, not an accuracy gate or refset. + +use std::ffi::OsString; +use std::fs::{self, File, OpenOptions}; +use std::io::{BufWriter, Write}; +use std::num::NonZeroUsize; +use std::path::{Path, PathBuf}; + +use bloomery_gpu::host::served::prompt_pass; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::model::StepMode; +use bloomery_gpu_flint::{Body, FlintModel, prefill, set_prefill_group, set_taps}; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::{self, PlanInputs}; +use model::placement::{Plan, PlanLevers, workstation}; + +type Error = Box; +type Result = std::result::Result; + +const WHAT: &str = "trace_flint"; +const WIDTH: usize = 2048; +const LAYERS: usize = 49; +const VOCAB: usize = 150_144; +const PROMPT: usize = 22; +const TOTAL: usize = 96; +const CALLS: usize = TOTAL - PROMPT + 1; +const CTX: usize = 1024; +const SLOTS: usize = 2; +const USAGE: &str = "Usage: trace_flint -m MODEL --ids FILE --prompt 22 --out NEWDIR --ctx 1024 --slots 2 --mode eager|graph [--taps]\n\ +Captures flint-trajectory-v2: 21-ID prefix prefill, then the last prompt ID and 74 forced one-ID steps.\n\ +FILE must contain exactly 96 decimal token IDs; no tokenization, sampling or EOG stopping.\n\ +--ctx is per-slot (total context 2048); exactly one CUDA device must be visible.\n\ +--taps captures each call's last-position layer outputs and result_norm.\n\ +Without --taps those tensors are explicitly absent; full logits are always captured.\n\ +The output directory must not exist. MANIFEST.tsv is published only after all files finish.\n\ +-h, --help print this usage without opening CUDA. --levers prints the existing lever table."; + +struct Args { + model: PathBuf, + ids: PathBuf, + out: PathBuf, + mode: StepMode, + taps: bool, + plan_levers: PlanLevers, +} + +fn assign(slot: &mut Option, value: T, name: &str) -> Result<()> { + if slot.replace(value).is_some() { + return Err(format!("{name}: duplicate option").into()); + } + Ok(()) +} + +fn parse() -> Result> { + let mut args = std::env::args_os().skip(1); + let (mut model, mut ids, mut out, mut mode) = (None, None, None, None); + let (mut prompt, mut ctx, mut slots) = (None, None, None); + let mut taps = false; + while let Some(arg) = args.next() { + let option = arg.to_str().ok_or("option is not UTF-8")?; + if matches!(option, "-h" | "--help") { + println!("{USAGE}"); + return Ok(None); + } + if option == "--levers" { + continue; + } + if option == "--taps" { + if taps { + return Err("--taps: duplicate option".into()); + } + taps = true; + continue; + } + if !matches!( + option, + "-m" | "--ids" | "--out" | "--mode" | "--prompt" | "--ctx" | "--slots" + ) { + return Err(format!("unknown option {option:?}\n{USAGE}").into()); + } + let value: OsString = args + .next() + .ok_or_else(|| format!("{option}: missing value"))?; + match option { + "-m" => assign(&mut model, PathBuf::from(value), option)?, + "--ids" => assign(&mut ids, PathBuf::from(value), option)?, + "--out" => assign(&mut out, PathBuf::from(value), option)?, + "--mode" => { + let value = match value.to_str() { + Some("eager") => StepMode::Eager, + Some("graph") => StepMode::Graph, + _ => return Err("--mode: expected eager or graph".into()), + }; + assign(&mut mode, value, option)?; + } + "--prompt" | "--ctx" | "--slots" => { + let value = value.to_str().ok_or("numeric option is not UTF-8")?; + if value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { + return Err(format!("{option}: expected an unsigned decimal integer").into()); + } + let value: usize = value.parse()?; + let slot = match option { + "--prompt" => &mut prompt, + "--ctx" => &mut ctx, + _ => &mut slots, + }; + assign(slot, value, option)?; + } + _ => unreachable!(), + } + } + // This exits through the common parser before any CUDA call. + let levers = bloomery_levers::at_main(&[bloomery_levers::CARD_BUDGET])?; + if prompt != Some(PROMPT) || ctx != Some(CTX) || slots != Some(SLOTS) { + return Err( + format!("required: --prompt {PROMPT} --ctx {CTX} --slots {SLOTS}\n{USAGE}").into(), + ); + } + Ok(Some(Args { + model: model.ok_or("required: -m MODEL")?, + ids: ids.ok_or("required: --ids FILE")?, + out: out.ok_or("required: --out NEWDIR")?, + mode: mode.ok_or("required: --mode eager|graph")?, + taps, + plan_levers: PlanLevers::from_levers(&levers)?, + })) +} + +fn tsv_path(path: &Path) -> Result<&str> { + let text = path.to_str().ok_or("artifact path is not UTF-8")?; + if text.contains(['\t', '\n', '\r']) { + return Err("artifact path contains a TSV delimiter".into()); + } + Ok(text) +} + +fn read_ids(path: &Path) -> Result> { + let text = fs::read_to_string(path)?; + let ids = text + .split_whitespace() + .map(|word| { + if !word.bytes().all(|b| b.is_ascii_digit()) { + return Err(format!("invalid decimal token ID {word:?}").into()); + } + let id: u32 = word.parse()?; + if id as usize >= VOCAB { + return Err(format!("token ID {id} is outside vocabulary {VOCAB}").into()); + } + Ok(id) + }) + .collect::>>()?; + if ids.len() != TOTAL { + return Err(format!("expected exactly {TOTAL} IDs, found {}", ids.len()).into()); + } + Ok(ids) +} + +fn create(path: &Path) -> Result> { + Ok(BufWriter::new( + OpenOptions::new().write(true).create_new(true).open(path)?, + )) +} + +fn finish(mut file: BufWriter) -> Result<()> { + file.flush()?; + file.get_ref().sync_all()?; + Ok(()) +} + +fn floats(file: &mut BufWriter, name: &str, values: &[f32], len: usize) -> Result<()> { + if values.len() != len { + return Err(format!("{name}: expected {len} floats, found {}", values.len()).into()); + } + if let Some(index) = values.iter().position(|x| !x.is_finite()) { + return Err(format!("{name}: non-finite float at {index}").into()); + } + for value in values { + file.write_all(&value.to_le_bytes())?; + } + Ok(()) +} + +fn placement(out: &Path, plan: &Plan<'_>, m: &FlintModel) -> Result<()> { + let mut rows = create(&out.join("placement.tsv"))?; + writeln!( + rows, + "tensor\tshard\tlayer\trole\tfile_type\tdims\tstage\tdevice\tformat\texpert_ids\tresident_bytes\tread_bytes" + )?; + for row in &plan.rows { + let tensor = &plan.model.tensors[row.tensor]; + for segment in &row.segments { + writeln!( + rows, + "{}\t{}\t{:?}\t{:?}\t{:?}\t{:?}\t{}\t{:?}\t{:?}\t{}\t{}\t{}", + tensor.name, + tensor.shard, + tensor.layer, + tensor.role, + tensor.ty, + tensor.dims, + row.stage, + segment.device, + segment.format, + segment + .experts + .as_ref() + .map_or_else(|| "absent".into(), ToString::to_string), + segment.resident_bytes, + row.read_bytes + )?; + } + } + finish(rows)?; + let body = m.body(WHAT)?; + let map = body.hybrid().slots(); + let mut slots = create(&out.join("expert-slots.tsv"))?; + writeln!(slots, "layer\texpert_id\tactual_slot")?; + for layer in map.layers() { + for id in 0..map.n_expert() { + let id = u32::try_from(id)?; + writeln!( + slots, + "{layer}\t{id}\t{:?}", + map.slot(layer, id).ok_or("missing loaded expert slot")? + )?; + } + } + finish(slots)?; + let mut details = create(&out.join("placement.txt"))?; + writeln!( + details, + "machine={:#?}\ncard_totals={:#?}\nhost_totals={:#?}\ncard_budget={:?}\nbudget_from_pool={}\nn_l={:?}\ntier_n_l={:?}\nnvme_bytes={}\nloaded_ctx={}\nloaded_store_bytes_per_slot={}\nloaded_scratch_bytes={}\nloaded_prompt_bytes={}", + plan.machine, + plan.cards, + plan.host, + plan.card_budget, + plan.budget_from_pool, + plan.n_l, + plan.tier_n_l, + plan.nvme_bytes, + body.ctx(), + body.store_bytes(), + body.scratch_bytes(), + body.prompt_bytes() + )?; + finish(details) +} + +struct Capture { + steps: BufWriter, + graph: BufWriter, + counters: BufWriter, + logits: BufWriter, + taps: Option<(BufWriter, BufWriter)>, + row: Vec, + graph_nodes: Option, +} + +impl Capture { + fn new(out: &Path, taps: bool) -> Result { + let mut steps = create(&out.join("steps.tsv"))?; + writeln!(steps, "call\tprocessed_pos\tpredict_pos\tinput_id\targmax")?; + let mut graph = create(&out.join("graph.tsv"))?; + writeln!(graph, "call\tmode\thas_capture\tdecode_nodes")?; + let mut counters = create(&out.join("counters.tsv"))?; + writeln!(counters, "call\thost_stats\ttier_stats")?; + Ok(Self { + steps, + graph, + counters, + logits: create(&out.join("logits.f32"))?, + taps: if taps { + Some(( + create(&out.join("layers.f32"))?, + create(&out.join("norm.f32"))?, + )) + } else { + None + }, + row: vec![0.0; VOCAB], + graph_nodes: None, + }) + } + + fn call(&mut self, m: &FlintModel, call: usize, input: u32, argmax: u32) -> Result<()> { + let processed = PROMPT - 1 + call; + if m.pos() as usize != processed + 1 || m.n_vocab()? != VOCAB || argmax as usize >= VOCAB { + return Err(format!("call {call}: invalid position, vocabulary or argmax").into()); + } + // Read the last call's taps before another prompt/step can overwrite them. + if let Some((layers, norm)) = &mut self.taps { + let values = m.body(WHAT)?.taps(m.gpu())?; + floats(layers, "layers", &values, LAYERS * WIDTH)?; + floats(norm, "result_norm", &m.head_normed()?, WIDTH)?; + } + m.logits_into(&mut self.row)?; + floats(&mut self.logits, "logits", &self.row, VOCAB)?; + let best = self.row.iter().copied().fold(f32::NEG_INFINITY, f32::max); + if self.row[argmax as usize] != best { + return Err(format!( + "call {call}: device argmax {argmax} does not maximize saved logits" + ) + .into()); + } + let captured = m.has_capture(); + if m.mode() == StepMode::Graph && !captured { + return Err(format!("call {call}: graph mode did not capture a decode chain").into()); + } + if m.mode() == StepMode::Eager && captured { + return Err(format!("call {call}: eager mode unexpectedly holds a capture").into()); + } + let nodes = if captured { + if let Some(nodes) = self.graph_nodes { + nodes + } else { + let nodes = m.step_graph_nodes()?.len(); + self.graph_nodes = Some(nodes); + nodes + } + } else { + 0 + }; + if captured && nodes == 0 { + return Err("captured decode graph is empty".into()); + } + writeln!( + self.steps, + "{call}\t{processed}\t{}\t{input}\t{argmax}", + processed + 1 + )?; + writeln!(self.graph, "{call}\t{:?}\t{captured}\t{nodes}", m.mode())?; + let hybrid = m.body(WHAT)?.hybrid(); + writeln!( + self.counters, + "{call}\t{:?}\t{:?}", + hybrid.stats(), + hybrid.tier_stats() + )?; + Ok(()) + } + + fn finish(self) -> Result<()> { + finish(self.steps)?; + finish(self.graph)?; + finish(self.counters)?; + finish(self.logits)?; + if let Some((layers, norm)) = self.taps { + finish(layers)?; + finish(norm)?; + } + Ok(()) + } +} + +fn run(args: Args) -> Result<()> { + let model = fs::canonicalize(&args.model)?; + let ids_path = fs::canonicalize(&args.ids)?; + tsv_path(&model)?; + tsv_path(&ids_path)?; + let ids = read_ids(&ids_path)?; + let file = Split::open(&model)?; + let mut inputs = PlanInputs::read(&file)?; + if inputs.spec.layers.len() != LAYERS + || inputs.spec.hidden as usize != WIDTH + || inputs.spec.vocab as usize != VOCAB + { + return Err("expected the full 49-layer, width-2048, vocab-150144 Flint model".into()); + } + place::configure_group(&mut inputs, 1)?; + // Atomic directory creation rejects existing directories, files and symlinks before CUDA is touched. + fs::create_dir(&args.out)?; + let out = fs::canonicalize(&args.out)?; + let mut tokens = create(&out.join("tokens.ids"))?; + for id in &ids { + writeln!(tokens, "{id}")?; + } + finish(tokens)?; + let census = bloomery_gpu::census()?; + let [device] = census.as_slice() else { + return Err(format!( + "exactly one CUDA device must be visible, found {}", + census.len() + ) + .into()); + }; + let mut hardware = create(&out.join("device.txt"))?; + writeln!(hardware, "{device:#?}")?; + finish(hardware)?; + let machine = workstation::plan_on(workstation::spec_of_device(device), LAYERS); + let slots = NonZeroUsize::new(SLOTS).ok_or("zero resident slots")?; + let plan = inputs.plan_with_slots(&machine, CTX as u64, &args.plan_levers, slots)?; + let host = HostCfg { + populate: false, + lock: false, + card_dontneed: false, + r8: false, + route_trace: false, + }; + let mut m = Body::open_placed_slots(file, &plan, &inputs, 0, host, Residency::Off, slots)?; + m.add_slots(SLOTS)?; + if m.body(WHAT)?.ctx() != CTX { + return Err("loaded per-slot context is not 1024".into()); + } + set_prefill_group(&mut m, 1)?; + if args.taps { + set_taps(&mut m, true)?; + } + // Arming taps selects eager and invalidates captures; the requested arm is selected afterwards. + m.set_mode(args.mode); + placement(&out, &plan, &m)?; + let mut capture = Capture::new(&out, args.taps)?; + // The HTTP Slot::close_call prefills all but the last prompt ID, then calls next. + prompt_pass(&mut m, |m| { + m.run_rows(PROMPT - 1, WHAT, |gpu, weights, body, head, at| { + prefill(gpu, weights, body, head, &ids[..PROMPT - 1], at) + }) + })? + .ok_or("batched prompt prefix produced no final argmax")?; + for (call, &id) in ids[PROMPT - 1..].iter().enumerate() { + let argmax = m.step(&[id])?; + capture.call(&m, call, id, argmax)?; + } + let mut nodes = create(&out.join("graph-nodes.tsv"))?; + writeln!(nodes, "node\tdescriptor")?; + if args.mode == StepMode::Graph { + for (index, node) in m.step_graph_nodes()?.iter().enumerate() { + writeln!(nodes, "{index}\t{node:?}")?; + } + } + finish(nodes)?; + capture.finish()?; + let mut manifest = create(&out.join("MANIFEST.tsv.partial"))?; + let mode = match args.mode { + StepMode::Eager => "eager", + StepMode::Graph => "graph", + }; + let instrumentation = if args.taps { + "layer-head-taps" + } else { + "no-taps" + }; + let layers_file = if args.taps { + "layers.f32" + } else { + "absent:not-requested" + }; + let norm_file = if args.taps { + "norm.f32" + } else { + "absent:not-requested" + }; + writeln!( + manifest, + "protocol\tflint-trajectory-v2\nmodel\t{}\nids_source\t{}\nengine\tbloomery-gpu-flint\nmode\t{mode}\ninstrumentation\t{instrumentation}\nctx_per_slot\t{CTX}\nctx_total\t{}\nslots\t{SLOTS}\nactive_slot\t0\nwidth\t{WIDTH}\nlayers\t{LAYERS}\nvocab\t{VOCAB}\nprompt\t{PROMPT}\ntotal_ids\t{TOTAL}\ncalls\t{CALLS}\nlogits\tlogits.f32\nlayers_file\t{layers_file}\nnorm_file\t{norm_file}\nlogits_layout\tf32le[call][150144]\nlayers_layout\tf32le[call][49][2048]\nnorm_layout\tf32le[call][2048]\nresidency\toff\nhost_populate\tfalse\nhost_lock\tfalse\ncard_dontneed\tfalse\nr8\tfalse\nroute_trace\tfalse\nprefill_group\t1\nprompt_submission\tprefix-prefill-then-next\nprefix_prefill_ids\t21\nfirst_observed_call\tone-id-next\nprompt_mode\teager-batched-prefix\ndecode_graph_captured\t{}\nplacement\tplacement.tsv\nloaded_expert_slots\texpert-slots.tsv\nplacement_counters\tplacement.txt\ndevice\tdevice.txt\ngraph_state\tgraph.tsv\ngraph_nodes\tgraph-nodes.tsv\nruntime_counters\tcounters.tsv\nprovenance\texternal-receipt-required\ncomplete\t1", + tsv_path(&model)?, + tsv_path(&ids_path)?, + CTX * SLOTS, + m.has_capture() + )?; + finish(manifest)?; + fs::rename(out.join("MANIFEST.tsv.partial"), out.join("MANIFEST.tsv"))?; + File::open(&out)?.sync_all()?; + Ok(()) +} + +fn main() { + let result = parse().and_then(|args| args.map_or(Ok(()), run)); + if let Err(error) = result { + eprintln!("{WHAT}: {error}"); + std::process::exit(1); + } +} diff --git a/crates/gpu-flint/src/body.rs b/crates/gpu-flint/src/body.rs new file mode 100644 index 000000000..27b7654db --- /dev/null +++ b/crates/gpu-flint/src/body.rs @@ -0,0 +1,583 @@ +//! Checked load, sequence ownership and common model seam for Flint. + +use std::num::NonZeroUsize; +use std::ops::Range; +use std::sync::Arc; + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch as AttnScratch}; +use bloomery_gpu::card_experts::CardFamily; +use bloomery_gpu::card_leg::ExpertInput; +use bloomery_gpu::head::Head; +use bloomery_gpu::host::leg_tier::{Hang, LegOpen}; +use bloomery_gpu::host::refuse_tier_count; +use bloomery_gpu::host::run::{HostRun, HostWidths}; +use bloomery_gpu::host::served::{ResidencyParts, TierBody}; +use bloomery_gpu::host::shell::TierShell; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::host::swap_source::{ + DEADLINE, LIVE_DELAY, ResidencyGlue, ResidencySpec, hosted_stacks, +}; +use bloomery_gpu::host::tier::TierOpen; +use bloomery_gpu::hybrid::{Chain, HostResidency, Hybrid, SlotMap}; +use bloomery_gpu::model::{ + CardBytes, ChainBody, HostServed, Rollback, Slots, StepMode, hosted_hybrid_tiered, + refuse_reserve_past, refuse_scratch_past, stage_run, +}; +use bloomery_gpu::rope_table::{RopeRows, RopeSpec, RopeTable}; +use bloomery_gpu::route_body::sigmoid_floor_256_6::RouterKernels; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{DeviceTensor, Gpu, GpuError, GpuModel}; +use bloomery_gpu_deepseek41::prompt_walk::{self, BatchBufs}; +use bloomery_gpu_deepseek41::shared_routed::{FfnConfig, ResidualOrder, Routed, RoutedDims}; +use bloomery_levers::{ChurnHosting, HostCfg}; +use cuda_core::{CudaStream, DeviceBuffer}; +use gguf::Split; +use model::arch::Arch; +use model::arch::flint::{host, place::PlanInputs}; +use model::arch::names; +use model::embed::EmbedTable; +use model::placement::Plan; +use model::placement::accepts::Program; +use models::{Ffn, Mixer, RopeMode}; +use runtime::layer::{Layer, MixerKind, ResidualKind, hosted}; + +pub(crate) const WHAT: &str = "flint Body"; +pub type FlintModel = GpuModel; +pub(crate) type FfnBody = Routed; + +/// The per-replay position. The embedding row is staged before refresh. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StepInput { + pub pos: u32, +} + +pub(crate) struct StepRows { + pub x: DeviceBuffer, + pub x1: DeviceBuffer, + pub xn: DeviceBuffer, + pub delta: DeviceBuffer, + pub pos: DeviceBuffer, + pub n_keys: DeviceBuffer, +} + +impl StepRows { + fn new(stream: &CudaStream, hidden: usize) -> Result { + Ok(Self { + x: DeviceBuffer::zeroed(stream, hidden)?, + x1: DeviceBuffer::zeroed(stream, hidden)?, + xn: DeviceBuffer::zeroed(stream, hidden)?, + delta: DeviceBuffer::zeroed(stream, hidden)?, + pos: DeviceBuffer::zeroed(stream, 1)?, + n_keys: DeviceBuffer::zeroed(stream, 1)?, + }) + } + + fn bytes(&self) -> usize { + [&self.x, &self.x1, &self.xn, &self.delta] + .into_iter() + .map(DeviceBuffer::num_bytes) + .sum::() + + self.pos.num_bytes() + + self.n_keys.num_bytes() + } +} + +/// Weights, kernels and scratch belong to the load; only caches and their high-water mark belong to a sequence. +pub struct Body { + pub(crate) shell: TierShell, + pub(crate) cfg: Vec, + pub(crate) bindings: Vec, + pub(crate) stores: Vec, + pub(crate) k: Kernels, + pub(crate) attention: AttnScratch, + pub(crate) batch_attention: AttnScratch, + pub(crate) s: StepRows, + pub(crate) rope: RopeRows, + pub(crate) ffn: FfnBody, + pub(crate) units: BatchBufs, + pub(crate) embed: EmbedTable, + row: Vec, + pub(crate) slots: Arc>, + pub(crate) at: u32, + pub(crate) ctx: usize, + pub(crate) hidden: usize, + eps: f32, + slots_planned: usize, + slots_made: usize, + prompt_reserve: u64, +} + +impl Body { + /// A single-sequence placed load with fixed expert residency. + pub fn open_placed( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + card: usize, + host: HostCfg, + ) -> Result { + Self::open_placed_slots( + file, + plan, + inputs, + card, + host, + Residency::Off, + NonZeroUsize::MIN, + ) + } + + /// Load exactly the number of resident sequences counted by the placement. + #[allow( + clippy::too_many_arguments, + reason = "the placed load's file, plan, card and host policy, residency and sequence count" + )] + pub fn open_placed_slots( + file: Split, + plan: &Plan<'_>, + inputs: &PlanInputs, + card: usize, + host: HostCfg, + residency: Residency, + slots: NonZeroUsize, + ) -> Result { + let tiers = TierOpen::of_machine(plan.machine)?; + refuse_tier_count(WHAT, tiers.len())?; + let (layers, _) = stage_run(WHAT, plan, card, inputs.spec.layers.len())?; + let one = inputs.kv.bytes(layers, plan.ctx_max); + refuse_store_count(plan, card, one, slots)?; + let run = hosted(&inputs.spec.layers).map_err(|e| shape(e.to_string()))?; + let residency = ResidencySpec { + lever: residency, + pool: ChurnHosting::Hosted, + delay: LIVE_DELAY, + deadline: DEADLINE, + top_k: 6, + max_rows: 1, + stacks: Arc::new(hosted_stacks(&inputs.model, WHAT, (run.len(), true))?), + }; + GpuModel::load_placed_with( + file, + plan, + card, + host, + residency, + |_, _, _, _| Ok(()), + |gpu, file, w, set, glue| { + Self::load( + gpu, + file, + w, + (plan, inputs, card), + host, + (set, glue), + (&tiers, slots), + ) + }, + ) + } + + #[allow( + clippy::too_many_arguments, + reason = "the checked placed load's disjoint common loader parts" + )] + fn load( + gpu: &Gpu, + file: &Arc, + w: &Weights, + (plan, inputs, card): (&Plan<'_>, &PlanInputs, usize), + host_cfg: HostCfg, + (set, glue): (HostResidency, ResidencyGlue), + (tiers, seqs): (&[TierOpen], NonZeroUsize), + ) -> Result { + gpu.context().bind_to_thread()?; + let spec = &inputs.spec; + let (layers, ctx) = stage_run(WHAT, plan, card, spec.layers.len())?; + let hidden = spec.hidden as usize; + let cfg: Vec<_> = spec.layers.iter().map(Layer::of).collect(); + for (l, kind) in cfg.iter().enumerate() { + if kind.mixer != MixerKind::Gqa || kind.residual != ResidualKind::Plain { + return Err(shape(format!( + "layer {l}: {kind:?}, not plain-residual GQA" + ))); + } + match &spec.layers[l].ffn { + Ffn::Moe(m) if m.experts != 256 || m.top_k != 6 || m.expert_ff != 512 => { + return Err(shape(format!( + "layer {l}: expert geometry {m:?}, not the compiled floor256top6 block" + ))); + } + _ => {} + } + } + let run = hosted(&spec.layers).map_err(|e| shape(e.to_string()))?; + let tier_cards: Vec<_> = tiers.iter().map(|t| t.card).collect(); + let map = SlotMap::of_plan_tiers(plan, card, &tier_cards, run.clone(), 256)?; + let bindings = spec + .layers + .iter() + .enumerate() + .map(|(l, s)| Binding::load(file, w, l, s, hidden, spec.rms_eps, ctx)) + .collect::, _>>()?; + let stores = bindings + .iter() + .map(|b| Cache::new(gpu, b)) + .collect::, _>>()?; + let held: usize = stores.iter().map(Cache::bytes).sum(); + let planned = inputs.kv.bytes(layers, plan.ctx_max); + if held as u64 != planned { + return Err(shape(format!( + "stores hold {held} bytes; the plan counts {planned}" + ))); + } + for (l, b) in bindings.iter().enumerate() { + let counted = inputs.kv.rows_at(l, plan.ctx_max); + if b.capacity() as u64 != counted { + return Err(shape(format!( + "layer {l}: {} physical rows; the plan counts {counted}", + b.capacity() + ))); + } + } + let rope_base = spec + .layers + .iter() + .find_map(|s| match &s.mixer { + Mixer::Gqa(g) if g.rope.mode == RopeMode::Neox => Some(g.rope.base), + _ => None, + }) + .ok_or_else(|| shape("no NeoX layer in the validated description"))?; + let table = RopeTable::new(&RopeSpec::window(rope_base, 128))?; + let rope = RopeRows::new(gpu.stream(), &table, 128, ctx)?; + let slots = Arc::new(DeviceTensor::upload( + gpu.stream(), + &map.stage_view(), + run.len(), + 256, + )?); + let ffn = FfnBody::new_configured( + gpu, + file, + w, + (&spec.layers, &map, &slots), + RoutedDims { + embd: hidden, + rms_eps: spec.rms_eps, + batch_rows: crate::prefill::T_MAX, + }, + CardFamily { + what: WHAT, + n_expert: 256, + n_used: 6, + stacks: names::routed_stacks, + program: Program::FlintBody, + }, + FfnConfig { + residual: ResidualOrder::RoutedSharedResidual, + expert_input: ExpertInput::AmaxF16, + }, + )?; + let hang = Hang { + tiers, + family: &crate::tier::NODE_FAMILY, + open: LegOpen { + n_expert: 256, + layers: cfg.len(), + hidden, + n_used: 6, + rows: 1, + }, + cols: crate::prefill::T_MAX, + card_dontneed: host_cfg.card_dontneed, + bytes: CardBytes::File, + }; + let mut hybrid = hosted_hybrid_tiered( + (gpu, file, plan), + (run.clone(), map), + (host_cfg, set, &glue), + ( + HostWidths { + embd: hidden, + ff: 512, + n_used: 6, + }, + 1, + ), + Some(&hang), + |src| { + run.clone() + .map(|l| host::layer(src, spec, l)) + .collect::, _>>() + }, + )?; + prompt_walk::prepare_host(gpu, &mut hybrid, crate::prefill::T_MAX)?; + let embed = EmbedTable::new(Arc::clone(file), &names::token_embd(), hidden)?; + if embed.n_vocab() != spec.vocab as usize { + return Err(shape(format!( + "embedding vocabulary {} differs from {}", + embed.n_vocab(), + spec.vocab + ))); + } + let prompt_reserve = inputs + .prompt_batch_bytes(plan.machine) + .map_err(|e| GpuError::plan(WHAT, e))?; + let mut body = Self { + shell: TierShell::new(hybrid, glue, hidden, WHAT), + k: Kernels::load(gpu)?, + attention: AttnScratch::new(gpu, &bindings, 1)?, + batch_attention: AttnScratch::new(gpu, &bindings, crate::prefill::T_MAX)?, + s: StepRows::new(gpu.stream(), hidden)?, + units: BatchBufs::new(gpu, hidden, crate::prefill::T_MAX, 1)?, + cfg, + bindings, + stores, + rope, + ffn, + embed, + row: vec![0.0; hidden], + slots, + at: 0, + ctx, + hidden, + eps: spec.rms_eps, + slots_planned: seqs.get(), + slots_made: 1, + prompt_reserve, + }; + refuse_scratch_past( + WHAT, + plan, + card, + body.scratch_bytes(), + "step rows, attention, routed blocks, rope, host boundary and slot map", + )?; + body.check_prompt_reserve()?; + body.start_residency(gpu)?; + Ok(body) + } + + pub fn ctx(&self) -> usize { + self.ctx + } + pub fn kinds(&self) -> Vec { + self.cfg.clone() + } + pub fn store_bytes(&self) -> usize { + self.stores.iter().map(Cache::bytes).sum() + } + pub fn scratch_bytes(&self) -> usize { + self.s.bytes() + + self.attention.bytes() + + self.k.bytes() + + self.ffn.bytes() + + self.rope.table.num_bytes() + + self.shell.hybrid.boundary().device_bytes() + + self.slots.buf().num_bytes() + } + pub fn prompt_bytes(&self) -> usize { + self.batch_attention.bytes() + self.units.bytes() + self.ffn.batch_bytes() + } + pub(crate) fn check_prompt_reserve(&self) -> Result<(), GpuError> { + refuse_reserve_past( + WHAT, + "prompt batch buffers", + self.prompt_reserve, + self.prompt_bytes() as u64, + "attention rows, common walk units and shared routed batch buffers", + ) + } + + /// Rollback never restores a ring. Its depth is measured from the greatest written end, not the last cut. + pub fn cut_keeps(&self, pos: u32, n: u32) -> bool { + let end = self.at.max(pos); + n <= end + && self.bindings.iter().all(|b| { + runtime::stores::cut_depth_keeps( + u64::from(end - n), + b.capacity() as u64, + self.ctx as u64, + b.window() as u64, + ) + }) + } + pub fn keepable_at(&self, pos: u32, n: u32) -> u32 { + if self.cut_keeps(pos, n) { n } else { 0 } + } + pub fn stand_for_cut(&mut self, pos: u32) { + self.at = self.at.max(pos); + } + pub fn hybrid(&self) -> &Hybrid { + self.shell.hybrid() + } + pub fn hybrid_mut(&mut self) -> &mut Hybrid { + self.shell.hybrid_mut() + } + pub fn taps(&self, gpu: &Gpu) -> Result, GpuError> { + self.shell.taps(gpu) + } +} + +impl ChainBody for Body { + type Input = StepInput; + type Host = Body; + const UNHOSTED_POOL: bloomery_levers::UnhostedPool = bloomery_levers::UnhostedPool::NotCarried; + fn arch() -> Arch { + Arch::Flint + } + fn decode_input(&mut self, token: u32, pos: u32) -> Result { + if pos as usize >= self.ctx { + return Err(shape(format!("position {pos} past context {}", self.ctx))); + } + self.embed.row_into(token, &mut self.row)?; + Ok(StepInput { pos }) + } + fn refresh(&mut self, stream: &CudaStream, input: &StepInput) -> Result<(), GpuError> { + self.s.x.copy_from_host(stream, &self.row)?; + self.s.pos.copy_from_host(stream, &[input.pos])?; + self.s.n_keys.copy_from_host(stream, &[input.pos + 1])?; + self.at = self.at.max(input.pos + 1); + Ok(()) + } + fn enqueue_chain(&mut self, gpu: &Gpu, w: &Weights, head: &mut Head) -> Result<(), GpuError> { + crate::step::enqueue(gpu, w, self, head) + } + fn reset(&mut self, gpu: &Gpu) -> Result<(), GpuError> { + self.shell.hybrid.settle(gpu.stream())?; + self.at = 0; + Ok(()) + } + fn head_eps(&self) -> f32 { + self.eps + } + fn resident_bytes(&self) -> usize { + self.store_bytes() + + self.scratch_bytes() + + self.prompt_bytes() + + self + .shell + .taps + .as_ref() + .map_or(0, |t| t.iter().map(DeviceBuffer::num_bytes).sum()) + } + fn layers(&self) -> Range { + 0..self.cfg.len() + } + fn host(&mut self) -> Option<&mut Body> { + Some(self) + } +} + +impl TierBody for Body { + type Experts = HostRun; + fn hybrid(&self) -> &Hybrid { + self.shell.hybrid() + } + fn hybrid_mut(&mut self) -> &mut Hybrid { + self.shell.hybrid_mut() + } + fn residency_parts(&mut self) -> Option> { + self.shell.residency_parts(&self.slots) + } + fn serve_chain(&mut self, chain: Chain) -> Result<(), GpuError> { + self.shell.hybrid.serve_captured_of(chain) + } +} + +impl Rollback for Body { + fn rollback(&mut self, pos: u32) -> Result<(), GpuError> { + if !self.cut_keeps(self.at, pos) { + return Err(shape(format!( + "rollback to {pos} from written end {} loses ring keys; reset and refeed", + self.at + ))); + } + Ok(()) + } +} + +/// One resident sequence's device stores and greatest written end. +pub struct Seq { + stores: Vec, + at: u32, +} + +impl Slots for Body { + type Seq = Seq; + fn new_seq(&mut self, gpu: &Gpu) -> Result { + if self.slots_made >= self.slots_planned { + return Err(shape(format!( + "sequence {} exceeds {} planned resident sequences", + self.slots_made + 1, + self.slots_planned + ))); + } + let need = self.store_bytes(); + let free = gpu.alloc_room()?; + if need > free { + return Err(shape(format!( + "sequence needs {need} store bytes, card has {free}" + ))); + } + let stores = self + .bindings + .iter() + .map(|b| Cache::new(gpu, b)) + .collect::, _>>()?; + self.slots_made += 1; + Ok(Seq { stores, at: 0 }) + } + fn swap_seq(&mut self, _gpu: &Gpu, seq: &mut Seq) -> Result<(), GpuError> { + if self.stores.len() != seq.stores.len() + || self + .stores + .iter() + .zip(&seq.stores) + .any(|(a, b)| a.bytes() != b.bytes()) + { + return Err(shape("parked sequence has another store layout")); + } + std::mem::swap(&mut self.stores, &mut seq.stores); + std::mem::swap(&mut self.at, &mut seq.at); + Ok(()) + } + fn seq_bytes(&self) -> usize { + self.store_bytes() + } +} + +impl Drop for Body { + fn drop(&mut self) { + self.shell.hybrid.stop_swap(); + } +} + +pub fn set_taps(m: &mut FlintModel, on: bool) -> Result<(), GpuError> { + m.set_mode(StepMode::Eager); + let (gpu, _, body) = m.body_parts(WHAT)?; + body.shell.set_taps(gpu, body.cfg.len(), on) +} + +fn refuse_store_count( + plan: &Plan<'_>, + card: usize, + one: u64, + slots: NonZeroUsize, +) -> Result<(), GpuError> { + let want = one.saturating_mul(slots.get() as u64); + let actual = plan + .cards + .get(card) + .ok_or_else(|| shape(format!("no card {card} in plan")))? + .kv_bytes; + if want != actual { + return Err(shape(format!( + "{slots} resident sequences need {want} store bytes; plan counts {actual}" + ))); + } + Ok(()) +} + +pub(crate) fn shape(detail: impl Into) -> GpuError { + GpuError::shape(WHAT, detail) +} diff --git a/crates/gpu-flint/src/lib.rs b/crates/gpu-flint/src/lib.rs new file mode 100644 index 000000000..c8a33bb62 --- /dev/null +++ b/crates/gpu-flint/src/lib.rs @@ -0,0 +1,16 @@ +//! Flint's composition of the common head128 attention, shared routed block and host tier. +//! Device-linked consumers build this crate with `cargo oxide`. + +pub mod body; +pub mod prefill; +mod step; +pub mod tier; + +pub use body::{Body, FlintModel, Seq, StepInput, set_taps}; +pub use prefill::{prefill, set_prefill_group}; + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod two_card_tests; diff --git a/crates/gpu-flint/src/prefill.rs b/crates/gpu-flint/src/prefill.rs new file mode 100644 index 000000000..a0e50c733 --- /dev/null +++ b/crates/gpu-flint/src/prefill.rs @@ -0,0 +1,128 @@ +//! Flint attention plugged into the common prompt walk and shared FFN. + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch}; +use bloomery_gpu::head::Head; +use bloomery_gpu::rope_table::RopeRows; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError}; +use bloomery_gpu_deepseek41::prompt_walk::{Batch, BatchAttention, UnitRows}; +use runtime::layer::{FfnKind, Layer}; + +use crate::body::{Body, FlintModel, shape}; + +/// The plan and the ring both hold one common prompt unit. +pub const T_MAX: usize = model::arch::flint::place::PROMPT_ROWS; +const _: () = assert!(T_MAX as u64 == runtime::stores::PROMPT_BATCH); + +/// Run a prompt through real attention and dense/routed blocks, then the final head. +pub fn prefill( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, + ids: &[u32], + at: u32, +) -> Result { + let to = + runtime::prompt::call_end(at, ids.len(), body.ctx).map_err(|e| shape(e.to_string()))?; + for &id in ids { + if id as usize >= body.embed.n_vocab() { + return Err(shape(format!( + "prompt token {id} past vocabulary {}", + body.embed.n_vocab() + ))); + } + } + // A failed call may already have overwritten ring rows. Reset, not a lower logical position, repairs that state. + body.at = body.at.max(to); + let Body { + shell, + cfg, + bindings, + stores, + k, + batch_attention, + rope, + units, + ffn, + embed, + hidden, + ctx, + .. + } = body; + let mut attention = Attention { + gpu, + w, + cfg, + bindings, + stores, + k, + scratch: batch_attention, + rope, + }; + let mut walk = Batch { + gpu, + w, + bufs: units, + attention: &mut attention, + ffn, + hybrid: &mut shell.hybrid, + embed, + head, + taps: shell.taps.as_deref_mut(), + embd: *hidden, + ctx: *ctx, + layers: cfg.len(), + }; + let mut cursor = at; + walk.prompt(ids, &mut cursor) +} + +struct Attention<'a> { + gpu: &'a Gpu, + w: &'a Weights, + cfg: &'a [Layer], + bindings: &'a [Binding], + stores: &'a mut [Cache], + k: &'a Kernels, + scratch: &'a mut Scratch, + rope: &'a RopeRows, +} + +impl BatchAttention for Attention<'_> { + fn attend( + &mut self, + l: usize, + _u: usize, + m: usize, + rows: UnitRows<'_>, + ) -> Result<(), GpuError> { + // SAFETY: the common walk stages consecutive positions and each count is position + 1. + unsafe { + self.bindings[l].enqueue( + self.gpu, + self.w, + self.k, + self.scratch, + &mut self.stores[l], + rows.x, + rows.x1, + rows.pos, + rows.n_keys, + m, + self.rope, + ) + } + } + fn ffn(&self, l: usize) -> FfnKind { + self.cfg[l].ffn + } +} + +/// Change prompt grouping between calls; allocation stays within the plan's reserve. +pub fn set_prefill_group(m: &mut FlintModel, group: usize) -> Result { + let (gpu, _, body) = m.body_parts("flint prompt group")?; + let made = body.units.set_group(gpu, body.hidden, group)?; + body.check_prompt_reserve()?; + Ok(made) +} diff --git a/crates/gpu-flint/src/step.rs b/crates/gpu-flint/src/step.rs new file mode 100644 index 000000000..a20320782 --- /dev/null +++ b/crates/gpu-flint/src/step.rs @@ -0,0 +1,183 @@ +//! One-token walk: common attention, then the dense or routed block, and the output head. + +use bloomery_gpu::attention_h128::{Binding, Cache, Kernels, Scratch}; +use bloomery_gpu::head::Head; +use bloomery_gpu::host::StepLeg; +use bloomery_gpu::host::run::HostRun; +use bloomery_gpu::rope_table::RopeRows; +use bloomery_gpu::weights::Weights; +use bloomery_gpu::{Gpu, GpuError}; +use bloomery_gpu_deepseek41::shared_routed; +use cuda_core::DeviceBuffer; +use runtime::layer::{FfnKind, Layer}; +use runtime::sched::{self, At, LayerProgram, Overlap, PortKind}; + +use crate::body::{Body, FfnBody, StepRows, shape}; + +pub(crate) fn enqueue( + gpu: &Gpu, + w: &Weights, + body: &mut Body, + head: &mut Head, +) -> Result<(), GpuError> { + let Body { + shell, + cfg, + bindings, + stores, + k, + attention, + s, + rope, + ffn, + .. + } = body; + let mut port = StepLeg::new(gpu.stream(), &mut shell.hybrid); + let mut program = Step { + gpu, + w, + cfg, + bindings, + stores, + k, + attention, + s, + rope, + ffn, + taps: shell.taps.as_deref_mut(), + head, + }; + sched::walk( + Overlap { + units: 1, + cols: 1, + port: PortKind::Step, + }, + program.cfg.len(), + &mut port, + &mut program, + ) +} + +struct Step<'a> { + gpu: &'a Gpu, + w: &'a Weights, + cfg: &'a [Layer], + bindings: &'a [Binding], + stores: &'a mut [Cache], + k: &'a Kernels, + attention: &'a mut Scratch, + s: &'a mut StepRows, + rope: &'a RopeRows, + ffn: &'a mut FfnBody, + taps: Option<&'a mut [DeviceBuffer]>, + head: &'a mut Head, +} + +impl Step<'_> { + fn tap(&mut self, l: usize) -> Result<(), GpuError> { + let last = l + 1 == self.cfg.len(); + if let Some(taps) = self.taps.as_deref_mut() { + let out = if last { + &*self.head.input_mut() + } else { + &self.s.x + }; + taps[l].copy_from_device_async(out, self.gpu.stream())?; + } + Ok(()) + } +} + +impl<'a> LayerProgram for Step<'a> { + type Port = StepLeg<'a, HostRun>; + + fn begin(&mut self, unit: usize) -> Result<(), GpuError> { + one(unit) + } + + fn front(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + let l = at.layer; + // SAFETY: refresh stages the one position and its count, position + 1. + unsafe { + self.bindings[l].enqueue( + self.gpu, + self.w, + self.k, + self.attention, + &mut self.stores[l], + &self.s.x, + &mut self.s.x1, + &self.s.pos, + &self.s.n_keys, + 1, + self.rope, + )?; + } + match self.cfg[l].ffn { + FfnKind::Moe => { + shared_routed::front(self.gpu, self.w, self.ffn, &self.s.x1, (port.hybrid(), l)) + } + FfnKind::Dense => { + let dest = if l + 1 == self.cfg.len() { + self.head.input_mut() + } else { + &mut self.s.x + }; + shared_routed::dense( + self.gpu, + self.w, + self.ffn, + l, + (&self.s.x1, &mut self.s.xn, &mut self.s.delta), + dest, + )?; + self.tap(l) + } + } + } + + fn shadow(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + if !self.cfg[at.layer].host_leg() { + return Ok(()); + } + shared_routed::shadow( + self.gpu, + self.w, + self.ffn, + &self.s.x1, + (port.hybrid().boundary(), at.layer), + ) + } + + fn back(&mut self, port: &mut Self::Port, at: At) -> Result<(), GpuError> { + one(at.unit)?; + let l = at.layer; + if !self.cfg[l].host_leg() { + return Ok(()); + } + let dest = if l + 1 == self.cfg.len() { + self.head.input_mut() + } else { + &mut self.s.x + }; + shared_routed::back(self.gpu, self.ffn, port.hybrid(), l, dest)?; + self.tap(l)?; + port.hybrid().row_enqueued(l, at.unit) + } + + fn end(&mut self, unit: usize) -> Result<(), GpuError> { + one(unit)?; + self.head.enqueue(self.gpu, self.w) + } +} + +fn one(unit: usize) -> Result<(), GpuError> { + if unit == 0 { + Ok(()) + } else { + Err(shape(format!("step walk has one unit, not unit {unit}"))) + } +} diff --git a/crates/gpu-flint/src/tests.rs b/crates/gpu-flint/src/tests.rs new file mode 100644 index 000000000..5be7729eb --- /dev/null +++ b/crates/gpu-flint/src/tests.rs @@ -0,0 +1,252 @@ +//! Full-file consumer transitions, compared against an untouched resident sequence. + +use std::num::NonZeroUsize; + +use bloomery_gpu::GpuError; +use bloomery_gpu::host::served::prompt_pass; +use bloomery_gpu::host::swap::Residency; +use bloomery_gpu::model::StepMode; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::PlanInputs; +use model::placement::{PlanLevers, workstation}; + +use crate::{Body, FlintModel}; + +const CTX: usize = 1536; +const PREFIX: usize = 512; +const WRITTEN: usize = 1025; +const SLOTS: usize = 3; +const WHAT: &str = "flint full-file transitions"; + +type Error = Box; + +struct Observation { + token: u32, + pos: u32, + logits: Vec, +} + +fn observe(m: &FlintModel, token: u32) -> Result { + let logits = m.logits()?; + assert!( + logits.iter().all(|x| x.is_finite()), + "{WHAT}: finite final logits" + ); + Ok(Observation { + token, + pos: m.pos(), + logits: logits.into_iter().map(f32::to_bits).collect(), + }) +} + +fn equal(label: &str, actual: &Observation, expected: &Observation) { + assert_eq!(actual.pos, expected.pos, "{label}: sequence position"); + assert_eq!(actual.token, expected.token, "{label}: next-token argmax"); + assert_eq!( + actual.logits.len(), + expected.logits.len(), + "{label}: vocabulary" + ); + let mismatch = actual + .logits + .iter() + .zip(&expected.logits) + .position(|(a, b)| a != b); + assert!( + mismatch.is_none(), + "{label}: final logit bits first differ at {mismatch:?}" + ); +} + +fn prompt(m: &mut FlintModel, ids: &[u32]) -> Result { + prompt_pass(m, |m| { + m.run_rows(ids.len(), WHAT, |gpu, w, body, head, at| { + crate::prefill::prefill(gpu, w, body, head, ids, at) + }) + })? + .ok_or(GpuError::State { + what: WHAT, + missing: "the prompt's final token", + }) +} + +fn stream(vocab: u32, seed: u32) -> Vec { + let mut state = seed; + (0..=WRITTEN) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + state % vocab + }) + .collect() +} + +/// Two decode observations straddle the same warmed context in every arm. The remainder wraps +/// the 1024-row window ring, while leaving a saved decode result exactly at its rollback boundary. +fn feed(m: &mut FlintModel, ids: &[u32]) -> Result<([Observation; 2], Observation), GpuError> { + prompt(m, &ids[..PREFIX])?; + let first = m.step(&ids[PREFIX..PREFIX + 1])?; + let first = observe(m, first)?; + let second = m.step(&ids[PREFIX + 1..PREFIX + 2])?; + let second = observe(m, second)?; + let last = prompt(m, &ids[PREFIX + 2..WRITTEN])?; + Ok(([first, second], observe(m, last)?)) +} + +#[test] +#[ignore = "needs a CUDA card and the complete Flint GGUF in BLOOMERY_MODEL"] +fn hw_full_file_capture_slots_reset_and_ring_boundary() -> Result<(), Error> { + let path = std::env::var_os("BLOOMERY_MODEL") + .ok_or("flint full-file transitions: BLOOMERY_MODEL must name the complete Flint GGUF")?; + let file = Split::open(path)?; + let inputs = PlanInputs::read(&file)?; + assert_eq!( + inputs.spec.layers.len(), + 49, + "the released full model, not a small fixture" + ); + assert_eq!( + inputs.spec.hidden, 2048, + "the released full model residual width" + ); + let ids = stream(inputs.spec.vocab, 0x4f1a_29c3); + let foreign = stream(inputs.spec.vocab, 0xe72d_810b); + assert_ne!( + &ids[..PREFIX], + &foreign[..PREFIX], + "the interleaved slot has a distinct history" + ); + let census = bloomery_gpu::census()?; + let [device] = census.as_slice() else { + return Err(format!( + "{WHAT}: exactly one CUDA device must be visible, found {}", + census.len() + ) + .into()); + }; + let machine = workstation::plan_on( + workstation::spec_of_device(device), + inputs.spec.layers.len(), + ); + let slots = NonZeroUsize::new(SLOTS).expect("three resident sequences"); + let plan = inputs.plan_with_slots(&machine, CTX as u64, &PlanLevers::default(), slots)?; + let host = HostCfg { + populate: false, + lock: false, + card_dontneed: false, + r8: false, + route_trace: false, + }; + let mut m = Body::open_placed_slots(file, &plan, &inputs, 0, host, Residency::Off, slots)?; + m.add_slots(SLOTS)?; + + // Slot 2 is a fresh sequence run without interleaving: its outputs are the control, not a guessed numerical band. + m.select_slot(2)?; + m.set_mode(StepMode::Eager); + let (control_decode, control_end) = feed(&mut m, &ids)?; + let control_next = m.step(&ids[WRITTEN..WRITTEN + 1])?; + let control_next = observe(&m, control_next)?; + + m.select_slot(0)?; + m.set_mode(StepMode::Graph); + let (captured_decode, captured_end) = feed(&mut m, &ids)?; + for (row, (actual, expected)) in captured_decode.iter().zip(&control_decode).enumerate() { + equal( + &format!("eager versus captured decode row {row}"), + actual, + expected, + ); + } + equal( + "eager versus captured after ring wrap", + &captured_end, + &control_end, + ); + assert!(m.has_capture(), "slot 0 actually captured a decode chain"); + let capture_nodes = m.step_graph_nodes()?.len(); + + // A second captured slot must not redirect the first slot's graph or its cache handles. + m.select_slot(1)?; + feed(&mut m, &foreign)?; + assert!(m.has_capture(), "slot 1 captured its own decode chain"); + m.select_slot(0)?; + assert_eq!( + m.pos(), + WRITTEN as u32, + "switching preserved slot 0's position" + ); + assert!( + m.has_capture(), + "slot 0's capture remained parked with its stores" + ); + assert_eq!( + m.step_graph_nodes()?.len(), + capture_nodes, + "slot 0's captured chain survived the switch" + ); + let resumed = m.step(&ids[WRITTEN..WRITTEN + 1])?; + equal( + "resumed slot versus untouched control", + &observe(&m, resumed)?, + &control_next, + ); + + // Reset after overwrite, then refeed through the retained capture, must reproduce a fresh sequence. + m.reset()?; + assert_eq!(m.pos(), 0, "reset rewound the selected sequence"); + assert!(m.has_capture(), "reset retains the reusable decode capture"); + let (refed_decode, refed_end) = feed(&mut m, &ids)?; + for (actual, expected) in refed_decode.iter().zip(&control_decode) { + equal( + "reset/refeed decode versus fresh sequence", + actual, + expected, + ); + } + equal( + "reset/refeed wrapped state versus fresh sequence", + &refed_end, + &control_end, + ); + + // At written end 1025, the ring holds positions 1..1025. Replaying position 513 + // needs 1..513, but position 512 needs overwritten position 0: depth 512, not 513. + let safe = WRITTEN as u32 - 512; + let lost = safe - 1; + assert!( + m.body(WHAT)?.cut_keeps(m.pos(), safe), + "depth 512 is retained" + ); + assert!( + !m.body(WHAT)?.cut_keeps(m.pos(), lost), + "depth 513 requires reset/refeed" + ); + m.rollback(safe)?; + assert_eq!( + m.pos(), + safe, + "retained rollback moved the logical position" + ); + let refusal = m.rollback(lost); + assert!( + matches!(&refusal, Err(GpuError::Shape { what, detail }) + if *what == "flint Body" && detail.contains("ring keys") && detail.contains("reset and refeed")), + "lost-window rollback must name the ring and recovery, got {refusal:?}" + ); + assert_eq!(m.pos(), safe, "refused rollback moved no sequence position"); + let replayed = m.step(&ids[safe as usize..safe as usize + 1])?; + equal( + "retained boundary replay", + &observe(&m, replayed)?, + &control_decode[1], + ); + + // Reset/refeed of slot 0 did not erase the parked control's position or continuation. + m.select_slot(2)?; + assert_eq!( + m.pos(), + (WRITTEN + 1) as u32, + "reset did not rewind a parked sequence" + ); + Ok(()) +} diff --git a/crates/gpu-flint/src/tier.rs b/crates/gpu-flint/src/tier.rs new file mode 100644 index 000000000..6286348fe --- /dev/null +++ b/crates/gpu-flint/src/tier.rs @@ -0,0 +1,65 @@ +//! Flint's checked description and host views for the common card-leg tier. + +use std::ops::Range; + +use bloomery_gpu::GpuError; +use bloomery_gpu::host::leg_tier::{self, LegFamily, LegWidths}; +use bloomery_gpu::host::node::NodeFamily; +use gguf::Split; +use model::ModelError; +use model::arch::flint::{host, place::PlanInputs}; +use model::moe::HostLayer; +use model::placement::ModelTensors; +use model::placement::accepts::Program; +use model::r8file::R8Source; +use models::LayerSpec; + +/// Maximum tokens per expert-tier launch; the placement reserves this capacity. +pub const CHUNK: usize = model::arch::flint::place::TIER_CHUNK as usize; + +struct Flint; + +impl LegFamily for Flint { + const WHAT: &'static str = "flint tier"; + const WORD: &'static str = "flint"; + const ROWS: usize = 1; + const CHUNK: usize = CHUNK; + type Inputs = PlanInputs; + + fn describe(file: &Split) -> Result { + PlanInputs::describe(file).map_err(|e| GpuError::plan(Self::WHAT, e)) + } + + fn into_model(inputs: PlanInputs) -> ModelTensors { + inputs.model + } + + fn layers(inputs: &PlanInputs) -> &[LayerSpec] { + &inputs.spec.layers + } + + fn widths(inputs: &PlanInputs) -> LegWidths { + LegWidths { + n_expert: 256, + n_used: 6, + n_embd: inputs.spec.hidden as usize, + ff: 512, + program: Program::FlintBody, + } + } + + fn expert_input(_inputs: &PlanInputs) -> bloomery_gpu::card_leg::ExpertInput { + bloomery_gpu::card_leg::ExpertInput::AmaxF16 + } + + fn routed( + src: R8Source<'_>, + inputs: &PlanInputs, + run: Range, + ) -> Result, ModelError> { + run.map(|l| host::layer(src, &inputs.spec, l)).collect() + } +} + +/// The tier service uses the same Program and host arithmetic as the stage card. +pub const NODE_FAMILY: NodeFamily = leg_tier::node_family::(); diff --git a/crates/gpu-flint/src/two_card_tests.rs b/crates/gpu-flint/src/two_card_tests.rs new file mode 100644 index 000000000..ccc90411c --- /dev/null +++ b/crates/gpu-flint/src/two_card_tests.rs @@ -0,0 +1,397 @@ +//! The real six-slot tier protocol against the same plan's one-card expert union. + +use std::path::Path; + +use bloomery_gpu::fault::{Fault, FaultSite}; +use bloomery_gpu::host::served::prompt_pass; +use bloomery_gpu::host::tier::TierStats; +use bloomery_gpu::model::StepMode; +use bloomery_gpu::{GpuError, hybrid::HOST}; +use bloomery_levers::HostCfg; +use gguf::Split; +use model::arch::flint::place::{PlanInputs, tier_batch}; +use model::placement::{Plan, PlanLevers, union_plan, workstation}; +use models::Ffn; + +use crate::{Body, FlintModel, set_taps}; + +const WHAT: &str = "flint two-card full-file acceptance"; +const CTX: usize = 1024; +const PROMPT: usize = 22; +const IDS: usize = 96; +const MODEL_BYTES: u64 = 25_054_688_704; +// PIN(2026-10-25): full-file SHA from the pinned Flint diagnostic model receipt. +const MODEL_SHA256: &str = "63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f"; + +type Error = Box; + +#[derive(Debug, PartialEq, Eq)] +struct Observation { + position: u32, + argmax: u32, + logits: Vec, + layers: Vec, +} + +fn observe(m: &FlintModel, argmax: u32) -> Result { + let logits = m.logits()?; + let layers = m.body(WHAT)?.taps(m.gpu())?; + assert_eq!(logits.len(), 150_144, "{WHAT}: full vocabulary"); + assert_eq!( + layers.len(), + 49 * 2048, + "{WHAT}: every layer's merge output" + ); + assert!( + logits.iter().chain(&layers).all(|v| v.is_finite()), + "{WHAT}: completed finite outputs, not an unfilled tier row" + ); + Ok(Observation { + position: m.pos(), + argmax, + logits: logits.into_iter().map(f32::to_bits).collect(), + layers: layers.into_iter().map(f32::to_bits).collect(), + }) +} + +fn same(label: &str, want: &[Observation], got: &[Observation]) { + assert_eq!(got.len(), want.len(), "{label}: completed calls"); + for (call, (w, g)) in want.iter().zip(got).enumerate() { + assert_eq!( + (g.position, g.argmax), + (w.position, w.argmax), + "{label}: call {call} position and token" + ); + for (name, a, b) in [ + ("logits", &g.logits, &w.logits), + ("layer outputs", &g.layers, &w.layers), + ] { + let first = a.iter().zip(b).position(|(a, b)| a != b); + assert!( + a.len() == b.len() && first.is_none(), + "{label}: call {call} {name} first differing bits {first:?}" + ); + } + } +} + +fn prompt(m: &mut FlintModel, ids: &[u32]) -> Result { + prompt_pass(m, |m| { + m.run_rows(ids.len(), WHAT, |gpu, w, body, head, at| { + crate::prefill(gpu, w, body, head, ids, at) + }) + })? + .ok_or(GpuError::State { + what: WHAT, + missing: "the batched prompt's last token", + }) +} + +fn tier_stats(m: &FlintModel) -> Result { + let tiers = m.body(WHAT)?.hybrid().tiers(); + assert_eq!(tiers.len(), 1, "{WHAT}: one actual expert tier"); + Ok(tiers[0].stats()) +} + +fn sent_every(before: &TierStats, after: &TierStats, layers: &[usize], first: usize) { + for &layer in layers { + let i = layer - first; + assert!( + after.layer_hits[i] > before.layer_hits[i], + "{WHAT}: real routing sent no tier slot at layer {layer}" + ); + } +} + +/// Read the actual router's six ranked picks and the handoff's published tier places. +fn published(m: &FlintModel, layer: usize) -> Result, GpuError> { + let gpu = m.gpu(); + gpu.context().bind_to_thread()?; + let body = m.body(WHAT)?; + let picks = body.ffn.route_ids().to_host_vec(gpu.stream())?; + assert_eq!(picks.len(), 6, "{WHAT}: six real routed slots"); + assert!(picks.iter().all(|&id| id < 256), "{WHAT}: valid expert ids"); + for (i, id) in picks.iter().enumerate() { + assert!(!picks[..i].contains(id), "{WHAT}: distinct top-six picks"); + } + let side = body.ffn.tier().expect("the stage has a tier join"); + let places = side.tsel(0)?.to_host_vec(gpu.stream())?; + let tier = &body.hybrid().tiers()[0]; + let ids = tier + .set() + .ids(layer) + .expect("last routed layer in tier set"); + let expected: Vec = picks + .iter() + .map(|id| ids.iter().position(|i| i == id).map_or(HOST, |i| i as u32)) + .collect(); + assert_eq!( + places, expected, + "{WHAT}: rank-preserving six-slot publication" + ); + let at = body.s.pos.to_host_vec(gpu.stream())?; + assert_eq!( + at, + [m.pos() - 1], + "{WHAT}: replay refreshed its position input" + ); + gpu.stream().synchronize()?; + Ok(picks) +} + +fn history( + m: &mut FlintModel, + ids: &[u32], + mode: StepMode, + tier_layers: &[usize], +) -> Result, GpuError> { + m.reset()?; + m.set_mode(mode); + let tiered = !tier_layers.is_empty(); + let first = if tiered { + m.body(WHAT)?.hybrid().tiers()[0].set().layers().start + } else { + 0 + }; + let before = tiered.then(|| tier_stats(m)).transpose()?; + let token = prompt(m, &ids[..PROMPT])?; + let mut out = vec![observe(m, token)?]; + let after_prompt = tiered.then(|| tier_stats(m)).transpose()?; + if let (Some(b), Some(a)) = (&before, &after_prompt) { + sent_every(b, a, tier_layers, first); + let batch = m.body(WHAT)?.hybrid().tier_batch_stats(); + assert_eq!(batch.len(), 1, "{WHAT}: actual tier batch port"); + assert!( + batch[0].served > 0 && batch[0].settles > 0, + "{WHAT}: batched prompt published and completed tier work" + ); + } + let mut previous_picks = None; + let mut changed_picks = false; + let mut graph_nodes = None; + for &id in &ids[PROMPT..] { + let token = m.step(&[id])?; + out.push(observe(m, token)?); + if mode == StepMode::Graph { + assert!(m.has_capture(), "{WHAT}: graph mode really captured"); + let nodes = m.step_graph_nodes()?.len(); + assert!(nodes > 0, "{WHAT}: nonempty captured decode"); + if let Some(previous) = graph_nodes { + assert_eq!( + nodes, previous, + "{WHAT}: changed inputs reuse the captured chain" + ); + } + graph_nodes = Some(nodes); + } + if let Some(&last) = tier_layers.last() { + let picks = published(m, last)?; + changed_picks |= previous_picks.as_ref().is_some_and(|p| *p != picks); + previous_picks = Some(picks); + } + } + assert_eq!( + m.pos(), + IDS as u32, + "{WHAT}: consumed the whole teacher-forced history" + ); + if let Some(before) = after_prompt { + let after = tier_stats(m)?; + sent_every(&before, &after, tier_layers, first); + assert!( + after.issued > before.issued && after.settles > before.settles, + "{WHAT}: decode published and completed actual tier work" + ); + assert!( + changed_picks, + "{WHAT}: replay refreshed actual routing, not just tensor shapes" + ); + if mode == StepMode::Graph { + assert!( + after.captures > before.captures && after.replays > before.replays, + "{WHAT}: the second physical card captured and replayed its leg" + ); + } + } + Ok(out) +} + +fn open(path: &Path, plan: &Plan<'_>, inputs: &PlanInputs) -> Result { + let host = HostCfg { + populate: false, + lock: false, + card_dontneed: false, + r8: false, + route_trace: false, + }; + let mut m = Body::open_placed(Split::open(path)?, plan, inputs, 0, host)?; + set_taps(&mut m, true)?; + Ok(m) +} + +#[test] +#[ignore = "needs exactly two visible CUDA devices, the complete pinned Flint GGUF in BLOOMERY_MODEL, and host RAM for its host experts"] +fn hw_two_card_full_file_union_capture_publication_fault_reset() -> Result<(), Error> { + let path = std::env::var_os("BLOOMERY_MODEL") + .ok_or("flint two-card acceptance: BLOOMERY_MODEL must name the complete pinned GGUF")?; + let path = Path::new(&path); + assert_eq!( + std::fs::metadata(path)?.len(), + MODEL_BYTES, + "{WHAT}: complete file" + ); + assert_eq!( + gguf::digest::sha256_file(path)?, + MODEL_SHA256, + "{WHAT}: pinned model bytes" + ); + let file = Split::open(path)?; + let inputs = PlanInputs::read(&file)?; + assert_eq!( + ( + inputs.spec.layers.len(), + inputs.spec.hidden, + inputs.spec.vocab + ), + (49, 2048, 150_144), + "{WHAT}: released geometry" + ); + assert_eq!( + inputs.model.experts_used, 6, + "{WHAT}: production top-six program" + ); + let census = bloomery_gpu::census()?; + let [stage, tier] = census.as_slice() else { + return Err(format!( + "{WHAT}: exactly two CUDA devices must be visible, found {}", + census.len() + ) + .into()); + }; + let stage = workstation::spec_of_device(stage); + let tier = workstation::spec_of_device(tier); + assert!( + !stage.same_device(&tier), + "{WHAT}: two distinct physical devices" + ); + // A reduced budget makes real tier assignments while leaving room for the union reference. + // It is bounded by each actual free reading, not an assumed card model or the file's length. + let budget = (8u64 << 30) + .min(stage.free_bytes.unwrap_or(stage.usable_bytes()) / 3) + .min(tier.free_bytes.unwrap_or(tier.usable_bytes()) / 2); + let machine = workstation::plan_tiers(49, stage, &[tier], None, tier_batch(&inputs)?)?; + let mut flat = machine.clone(); + flat.tiers.clear(); + let levers = PlanLevers { + card_budget_bytes: Some(budget).into(), + }; + let plan = inputs.plan(&machine, CTX as u64, &levers)?; + let routed: Vec = inputs + .spec + .layers + .iter() + .enumerate() + .filter_map(|(i, l)| matches!(l.ffn, Ffn::Moe(_)).then_some(i)) + .collect(); + assert_eq!(plan.tier_n_l.len(), 1, "{WHAT}: one planned tier"); + for &l in &routed { + assert!( + plan.n_l[l] > 0 && plan.tier_n_l[0][l] > 0, + "{WHAT}: layer {l} must assign actual experts to both physical cards, budget {budget}" + ); + } + let union = union_plan(&plan, &flat)?; + assert!( + union.cards[0].headroom_bytes >= i128::from(flat.cards[0].margin_bytes), + "{WHAT}: the actual stage must fit the union and its reserves" + ); + let mut state = 0x4f1a_29c3u32; + let ids: Vec = (0..IDS) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + state % inputs.spec.vocab + }) + .collect(); + assert!( + ids.windows(2).all(|w| w[0] != w[1]), + "{WHAT}: changed decode inputs" + ); + let reference = { + let mut m = open(path, &union, &inputs)?; + assert!( + m.body(WHAT)?.hybrid().tiers().is_empty(), + "{WHAT}: reference computes the union on the stage" + ); + history(&mut m, &ids, StepMode::Eager, &[])? + }; + let mut m = open(path, &plan, &inputs)?; + let actual_stage = m.gpu().device_id()?; + let actual_tier = m.body(WHAT)?.hybrid().tiers()[0].gpu()?.device_id()?; + assert_ne!( + actual_stage.uuid, actual_tier.uuid, + "{WHAT}: not same-GPU wrappers" + ); + assert_eq!(actual_stage, stage.device.expect("census stage identity")); + assert_eq!(actual_tier, tier.device.expect("census tier identity")); + let eager = history(&mut m, &ids, StepMode::Eager, &routed)?; + same("two-card eager versus same-plan union", &reference, &eager); + let graph = history(&mut m, &ids, StepMode::Graph, &routed)?; + same( + "two-card capture versus same-plan union", + &reference, + &graph, + ); + + let fault = Fault::at(routed[0] as u32, FaultSite::QuantColumn); + m.gpu().context().bind_to_thread()?; + assert_eq!(m.gpu().fault()?, None, "{WHAT}: stage starts clean"); + { + let gpu = m.body(WHAT)?.hybrid().tiers()[0].gpu()?; + gpu.context().bind_to_thread()?; + let x = cuda_core::DeviceBuffer::from_host(gpu.stream(), &[f32::NAN; 256])?; + let mut act = bloomery_gpu::Q8Act::with_slots(gpu.stream(), 1, 256)?; + gpu.enqueue_quantize_q8_1_layer(&x, &mut act, routed[0])?; + gpu.stream().synchronize()?; + assert_eq!(gpu.fault()?, Some(fault), "{WHAT}: real tier kernel fault"); + } + m.gpu().context().bind_to_thread()?; + assert_eq!( + m.gpu().fault()?, + None, + "{WHAT}: tier fault is not a stage injection" + ); + let failed = m.step(&[ids[0]]); + assert!( + matches!(failed, Err(GpuError::Fault { fault: got, .. }) if got == fault), + "{WHAT}: actual readback must carry the tier kernel fault, got {failed:?}" + ); + assert_eq!( + m.poisoned(), + Some(fault), + "{WHAT}: fault poisoned the model" + ); + let position = m.pos(); + assert!( + matches!(m.step(&[ids[1]]), Err(GpuError::Poisoned { fault: got, .. }) if got == fault), + "{WHAT}: poisoned model must refuse further work" + ); + assert_eq!(m.pos(), position, "{WHAT}: refusal advanced no sequence"); + m.reset()?; + assert_eq!(m.poisoned(), None, "{WHAT}: reset lifted fault poison"); + assert_eq!(m.pos(), 0, "{WHAT}: reset rewound sequence"); + m.set_mode(StepMode::Graph); + let token = prompt(&mut m, &ids[..PROMPT])?; + let mut recovered = vec![observe(&m, token)?]; + let token = m.step(&ids[PROMPT..PROMPT + 1])?; + recovered.push(observe(&m, token)?); + same( + "fault/reset/refeed versus clean union", + &reference[..2], + &recovered, + ); + assert!( + m.has_capture(), + "{WHAT}: recovered model has a real decode capture" + ); + Ok(()) +} diff --git a/crates/gpu-gates/Cargo.toml b/crates/gpu-gates/Cargo.toml index 91dd37c52..cd59ba8b6 100644 --- a/crates/gpu-gates/Cargo.toml +++ b/crates/gpu-gates/Cargo.toml @@ -60,6 +60,7 @@ bloomery-gpu-mimo2 = { path = "../gpu-mimo2", optional = true } bloomery-gpu-solar2 = { path = "../gpu-solar2", optional = true } # The MiniMax-M3 program, reached only through the `minimaxm3` feature (its sparse-attention gate). bloomery-gpu-minimaxm3 = { path = "../gpu-minimaxm3", optional = true } +bloomery-gpu-flint = { path = "../gpu-flint", optional = true } # The EmbeddingGemma 2 encoder, reached only through the `gemma-embedding2` feature (its end-to-end # gate). bloomery-gpu-gemma-embedding2 = { path = "../gpu-gemma-embedding2", optional = true } @@ -152,6 +153,7 @@ glm5next = ["deepseek41", "dep:bloomery-gpu-glm5next", "bloomery-app/glm5next"] mimo2 = ["gpu", "dep:bloomery-gpu-mimo2", "bloomery-app/mimo2"] solar2 = ["gpu", "dep:bloomery-gpu-solar2", "bloomery-app/solar2"] minimaxm3 = ["gpu", "dep:bloomery-gpu-minimaxm3", "bloomery-app/minimaxm3"] +flint = ["gpu", "dep:bloomery-gpu-flint", "bloomery-app/flint"] gemma-embedding2 = ["gpu", "gemma4a", "dep:bloomery-gpu-vision", "dep:bloomery-gpu-gemma-embedding2", "dep:bloomery-serve"] gemma4a = ["gpu", "dep:bloomery-gpu-gemma4a", "dep:bloomery-vision"] @@ -171,7 +173,7 @@ required-features = ["deepseek41"] [[bin]] name = "bloomery-serve" path = "src/bin/bloomery_serve.rs" -required-features = ["glm5next", "clef", "mimo2", "solar2", "minimaxm3", "gemma-embedding2", "hf"] +required-features = ["glm5next", "clef", "mimo2", "solar2", "minimaxm3", "flint", "gemma-embedding2", "hf"] [[bin]] name = "bloomery-serve-ds41" diff --git a/crates/gpu-gates/src/bin/bloomery_serve.rs b/crates/gpu-gates/src/bin/bloomery_serve.rs index 8239b7f97..25e290fe8 100644 --- a/crates/gpu-gates/src/bin/bloomery_serve.rs +++ b/crates/gpu-gates/src/bin/bloomery_serve.rs @@ -1,5 +1,5 @@ //! `bloomery-serve` — one server binary, one seat a model: `--model -//! ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide` picks the seat +//! ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide` picks the seat //! (`serve_seats`, the shared seat modules), every other flag the seat's own, //! parsed by the seat's own parser, so each seat's flag set is exactly its //! per-model binary's. The @@ -7,7 +7,7 @@ //! picks the seat, and a bare start or `--help` prints [`drive::MODELS`], //! one line a seat with a working `--hf` example. //! -//! bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide] +//! bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide] //! [-m PATH | --model-file PATH | --hf [:]] //! [--head [--head-config ]] //! @@ -40,7 +40,8 @@ //! glm5-next the glm seat, qwen4exp the qwen38 seat, qwen3moe and qwen35moe //! the qwen3 seat, mimo2 the mimo2 seat, solar-open2 the solar2 seat, minimax-m3 //! the minimaxm3 seat, -//! nemotron_h and nemotron_h_moe the nemotron seat, raon1 the raon1 seat, muse-spark the musespark seat; +//! nemotron_h and nemotron_h_moe the nemotron seat, raon1 the raon1 seat, muse-spark the musespark seat, +//! and flint the flint seat; //! a dense qwen35 file generates through `--model qwen3`. With no model //! word it keeps the decision selection rules: a file with no head is //! refused by name, saying what would seat it (`--head`, a row's `--hf` @@ -164,19 +165,20 @@ mod drive { use serve::preflight::Driver; use crate::serve_seats::{ - decide, ds41, glm, mimo2, minimaxm3, musespark, nemotron, qwen3, qwen38, raon1, solar2, + decide, ds41, flint, glm, mimo2, minimaxm3, musespark, nemotron, qwen3, qwen38, raon1, + solar2, }; const NAME: &str = "bloomery-serve"; - const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|decide] \ + const USAGE: &str = "usage: bloomery-serve [--model ds41|qwen38|glm|qwen3|mimo2|solar2|minimaxm3|nemotron|raon1|musespark|flint|decide] \ [-m PATH | --model-file PATH | --hf [:]] [--head \ [--head-config ]] [--embd-model FILE | --embd-hf \ [:]] [--embd-ctx N] [--embd-cache-bytes N] [--embd-mmproj FILE | \ --embd-no-mmproj] [--embd-image-min-tokens N] [--embd-image-max-tokens N] \ [llama-server's placement flags: -dev, -sm, -mg, -ts, -ngl, \ --cpu-moe, -fit, -fitt, -devd] | --list-devices | --version"; @@ -196,6 +198,7 @@ models (the --model word is optional with a model file: the file's architecture nemotron Nemotron 3.5 Bolt -m raon1 Raon-1-mini -m musespark Muse Spark Open 816B -m + flint Flint -m decide a decision model by its head --hf bartowski/Cloudflare_clef-flash-GGUF:Q5_K_M decide a decision model in its file --hf ggml-org/lev-GGUF:Q4_K_M"; @@ -221,6 +224,7 @@ models (the --model word is optional with a model file: the file's architecture Nemotron, Raon1, MuseSpark, + Flint, } /// A `--model` word: a generative seat's, or the decide seat's. @@ -243,9 +247,10 @@ models (the --model word is optional with a model file: the file's architecture "nemotron" => Ok(Word::Seat(Model::Nemotron)), "raon1" => Ok(Word::Seat(Model::Raon1)), "musespark" => Ok(Word::Seat(Model::MuseSpark)), + "flint" => Ok(Word::Seat(Model::Flint)), serve::decide::WORD => Ok(Word::Decide), other => Err(format!( - "--model is ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark or {}, not {other}", + "--model is ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, flint or {}, not {other}", serve::decide::WORD ) .into()), @@ -262,7 +267,7 @@ models (the --model word is optional with a model file: the file's architecture impl Model { /// Every generative seat. - const ALL: [Model; 10] = [ + const ALL: [Model; 11] = [ Model::Ds41, Model::Qwen38, Model::Glm, @@ -273,6 +278,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron, Model::Raon1, Model::MuseSpark, + Model::Flint, ]; /// The seat's word, as errors and the usage name it. @@ -288,6 +294,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron => "nemotron", Model::Raon1 => "raon1", Model::MuseSpark => "musespark", + Model::Flint => "flint", } } @@ -313,6 +320,7 @@ models (the --model word is optional with a model file: the file's architecture ), Model::Raon1 => matches!(Arch::from_name(arch), Ok(Arch::Raon1)), Model::MuseSpark => matches!(Arch::from_name(arch), Ok(Arch::MuseSpark)), + Model::Flint => matches!(Arch::from_name(arch), Ok(Arch::Flint)), } } } @@ -330,7 +338,7 @@ models (the --model word is optional with a model file: the file's architecture } let v = it.next().ok_or_else(|| { format!( - "--model needs a value (ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark or {}): {USAGE}", + "--model needs a value (ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, flint or {}): {USAGE}", serve::decide::WORD ) })?; @@ -605,6 +613,7 @@ models (the --model word is optional with a model file: the file's architecture Model::Nemotron => nemotron::run(&rest, &side), Model::Raon1 => raon1::run(&rest, &side), Model::MuseSpark => musespark::run(&rest, &side), + Model::Flint => flint::run(&rest, &side), }; ended(r) } diff --git a/crates/gpu-gates/src/bin/gate_flint_attn.rs b/crates/gpu-gates/src/bin/gate_flint_attn.rs index 486bb3ed6..0c2ad26d6 100644 --- a/crates/gpu-gates/src/bin/gate_flint_attn.rs +++ b/crates/gpu-gates/src/bin/gate_flint_attn.rs @@ -25,10 +25,12 @@ fn main() -> std::process::ExitCode { #[cfg(feature = "gpu")] mod gate { + use bloomery_gpu::attention_h128::Kernels as LoadedAttention; use bloomery_gpu::fault::{Fault, FaultSite}; use bloomery_gpu::flash_gqa::{GROUP, HEAD, window_segments}; use bloomery_gpu::flash_gqa_h128_swa::{H128SwaArgs, H128SwaKernels}; use bloomery_gpu::qk_rms::{HEAD as RMS_HEAD, QkRmsArgs, QkRmsKernels}; + use bloomery_gpu::rope_neox::NeoxArgs; use bloomery_gpu::{Gpu, GpuError, LAYER_NONE}; use bloomery_gpu_gates::rounding::butterfly; use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, verdict}; @@ -631,6 +633,315 @@ mod gate { Ok(ok) } + fn sample(pos: usize, head: usize, dim: usize, salt: usize) -> f32 { + ((pos * 17 + head * 13 + dim * 7 + salt) % 97) as f32 / 97.0 - 0.5 + } + + fn turn_at(values: &[f32], table: &[f32], pos: usize, rotate: bool) -> Vec { + let mut out = values.to_vec(); + if rotate { + for d in 0..HEAD / 2 { + let (x, y) = (values[d], values[d + HEAD / 2]); + let (c, s) = (table[pos * HEAD + 2 * d], table[pos * HEAD + 2 * d + 1]); + out[d] = x.mul_add(c, -(y * s)); + out[d + HEAD / 2] = x.mul_add(s, y * c); + } + } + out + } + + /// Append across a wrap before a causal multirow pass. The full logical planes are the + /// independent oracle; only the device ring maps absolute rows modulo its capacity. + fn loaded_case( + gpu: &Gpu, + loaded: &LoadedAttention, + rotate: bool, + window: usize, + rows: usize, + m: usize, + ) -> Result { + let stream = gpu.stream(); + let ctx = 3072usize; + let start = 2047usize; + let kv = 2; + let heads = kv * GROUP; + let gq_h: Vec = (0..HEAD).map(|d| 0.6 + d as f32 * 0.001).collect(); + let gk_h: Vec = (0..HEAD).map(|d| 0.8 - d as f32 * 0.001).collect(); + let mut table_h = vec![0.0f32; ctx * HEAD]; + for p in 0..ctx { + for d in 0..HEAD / 2 { + let angle = p as f64 * 10000.0f64.powf(-((2 * d) as f64) / HEAD as f64); + table_h[p * HEAD + 2 * d] = angle.cos() as f32; + table_h[p * HEAD + 2 * d + 1] = angle.sin() as f32; + } + } + let mut logical_k = vec![0u16; kv * ctx * HEAD]; + let mut logical_v = vec![0u16; kv * ctx * HEAD]; + for kh in 0..kv { + for p in 0..start + m { + let raw: Vec = (0..HEAD).map(|d| sample(p, kh, d, 11)).collect(); + let key = turn_at(&rms_head(&raw, &gk_h, EPS), &table_h, p, rotate); + for d in 0..HEAD { + logical_k[(kh * ctx + p) * HEAD + d] = f32_to_f16_bits(key[d]); + logical_v[(kh * ctx + p) * HEAD + d] = + f32_to_f16_bits(sample(p, kh, d, 29) * 4.0); + } + } + } + let guard = 2 * HEAD; + let mut cache_k_h = vec![SENTINEL; kv * rows * HEAD + guard]; + let mut cache_v_h = cache_k_h.clone(); + for kh in 0..kv { + for p in start.saturating_sub(rows)..start { + let physical = (kh * rows + p % rows) * HEAD; + let logical = (kh * ctx + p) * HEAD; + cache_k_h[physical..physical + HEAD] + .copy_from_slice(&logical_k[logical..logical + HEAD]); + cache_v_h[physical..physical + HEAD] + .copy_from_slice(&logical_v[logical..logical + HEAD]); + } + } + let mut expect_k = cache_k_h.clone(); + let mut expect_v = cache_v_h.clone(); + let mut q_h = Vec::new(); + let mut raw_k = Vec::new(); + let mut raw_v = Vec::new(); + let mut expect_q = Vec::new(); + for t in 0..m { + let p = start + t; + for h in 0..heads { + let raw: Vec = (0..HEAD).map(|d| sample(p, h, d, 3)).collect(); + expect_q.extend(turn_at(&rms_head(&raw, &gq_h, EPS), &table_h, p, rotate)); + q_h.extend(raw); + } + for kh in 0..kv { + raw_k.extend((0..HEAD).map(|d| sample(p, kh, d, 11))); + raw_v.extend((0..HEAD).map(|d| sample(p, kh, d, 29) * 4.0)); + let physical = (kh * rows + p % rows) * HEAD; + let logical = (kh * ctx + p) * HEAD; + expect_k[physical..physical + HEAD] + .copy_from_slice(&logical_k[logical..logical + HEAD]); + expect_v[physical..physical + HEAD] + .copy_from_slice(&logical_v[logical..logical + HEAD]); + } + } + let pos_h: Vec = (start..start + m).map(|p| p as u32).collect(); + let counts_h: Vec = pos_h.iter().map(|p| p + 1).collect(); + let mut q = DeviceBuffer::from_host(stream, &q_h)?; + let mut kd = DeviceBuffer::from_host(stream, &raw_k)?; + let v = DeviceBuffer::from_host(stream, &raw_v)?; + let gq = DeviceBuffer::from_host(stream, &gq_h)?; + let gk = DeviceBuffer::from_host(stream, &gk_h)?; + let table = DeviceBuffer::from_host(stream, &table_h)?; + let pos = DeviceBuffer::from_host(stream, &pos_h)?; + let counts = DeviceBuffer::from_host(stream, &counts_h)?; + let sinks_h: Vec = (0..heads).map(|h| 7.0 + h as f32 * 0.3).collect(); + let sinks = DeviceBuffer::from_host(stream, &sinks_h)?; + let mut ck = DeviceBuffer::from_host(stream, &cache_k_h)?; + let mut cv = DeviceBuffer::from_host(stream, &cache_v_h)?; + let segs = window_segments(window); + let mut pv = DeviceBuffer::zeroed(stream, m * heads * segs * HEAD)?; + let mut pm = DeviceBuffer::zeroed(stream, m * heads * segs * 2)?; + let mut out = DeviceBuffer::from_host(stream, &vec![12345.0f32; m * heads * HEAD + HEAD])?; + gpu.take_fault()?; + // SAFETY: consecutive positions, distinct modulo rows, and m <= rows-window+1 + // for rings. All readable historical keys above were initialized. + unsafe { + loaded.enqueue_append( + stream, + NeoxArgs { + q: &mut q, + k: &mut kd, + v: &v, + gq: &gq, + gk: &gk, + table: &table, + pos: &pos, + eps: EPS, + n_head: heads, + n_kv: kv, + ctx, + m, + fault: gpu.unlabelled_sink(), + cache_k: &mut ck, + cache_v: &mut cv, + }, + rows, + rotate, + )?; + } + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &counts, + scale: score_scale(), + n_kv: kv, + ctx, + m, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + )?; + stream.synchronize()?; + let got_q = q.to_host_vec(stream)?; + let got = out.to_host_vec(stream)?; + let cache_same = ck.to_host_vec(stream)? == expect_k && cv.to_host_vec(stream)? == expect_v; + let query_same = bits_equal(&got_q, &expect_q); + let mut error = 0.0f32; + let mut no_sink_gap = 0.0f32; + for t in 0..m { + let end = start + t + 1; + for (h, &sink) in sinks_h.iter().enumerate() { + let b = (t * heads + h) * HEAD; + let base = (h / GROUP) * ctx * HEAD; + let keys = &logical_k[base..base + ctx * HEAD]; + let values = &logical_v[base..base + ctx * HEAD]; + let reference = attend( + &expect_q[b..b + HEAD], + keys, + values, + window_first(end, window), + end, + score_scale(), + Some(sink), + ); + error = error.max(max_abs(&got[b..b + HEAD], &reference)); + if t == 0 && h == 0 { + let no_sink = attend( + &expect_q[b..b + HEAD], + keys, + values, + window_first(end, window), + end, + score_scale(), + None, + ); + no_sink_gap = max_abs(&reference, &no_sink); + } + } + } + let canary = got[m * heads * HEAD..].iter().all(|x| *x == 12345.0); + let clean = gpu.take_fault()?.is_none(); + let mut ok = clause( + "loaded attention logical/ring", + query_same && cache_same && canary && clean && error <= BAND, + &format!( + "rotate={rotate} window={window} rows={rows} m={m} q={query_same} cache={cache_same} canary={canary} clean={clean} max_abs={error:.3e}" + ), + ); + ok &= clause( + "loaded attention sink witness", + no_sink_gap > 1e-5, + &format!("max_abs without sink={no_sink_gap:.3e}"), + ); + if rows < ctx { + ok &= refused( + "ring unsafe append-then-attend chunk", + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &counts, + scale: score_scale(), + n_kv: kv, + ctx, + m: rows - window + 2, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + ), + "invalid ring geometry", + ); + let bad_counts = DeviceBuffer::from_host(stream, &[0u32, ctx as u32 + 1])?; + loaded.enqueue_attention( + stream, + H128SwaArgs { + q: &q, + kc: &ck, + vc: &cv, + n_keys: &bad_counts, + scale: score_scale(), + n_kv: kv, + ctx, + m: 2, + window, + sinks: Some(&sinks), + part_v: &mut pv, + part_ms: &mut pm, + fault: gpu.unlabelled_sink(), + y: &mut out, + }, + heads, + rows, + )?; + stream.synchronize()?; + let invalid = out.to_host_vec(stream)?; + ok &= clause( + "ring invalid logical count", + invalid[..2 * heads * HEAD].iter().all(|v| v.is_nan()) + && gpu.take_fault()? == Some(Fault::at(LAYER_NONE, FaultSite::KeyCount)), + "zero and count past logical ctx are NaN with key_count", + ); + let invalid_pos = DeviceBuffer::from_host(stream, &[ctx as u32])?; + let before_k = ck.to_host_vec(stream)?; + let before_v = cv.to_host_vec(stream)?; + // SAFETY: the out-of-range position is never appended. + unsafe { + loaded.enqueue_append( + stream, + NeoxArgs { + q: &mut q, + k: &mut kd, + v: &v, + gq: &gq, + gk: &gk, + table: &table, + pos: &invalid_pos, + eps: EPS, + n_head: heads, + n_kv: kv, + ctx, + m: 1, + fault: gpu.unlabelled_sink(), + cache_k: &mut ck, + cache_v: &mut cv, + }, + rows, + rotate, + )?; + } + stream.synchronize()?; + ok &= clause( + "ring invalid absolute position", + q.to_host_vec(stream)?[..heads * HEAD] + .iter() + .all(|v| v.is_nan()) + && ck.to_host_vec(stream)? == before_k + && cv.to_host_vec(stream)? == before_v + && gpu.take_fault()? == Some(Fault::at(LAYER_NONE, FaultSite::CachePos)), + "position at logical ctx is NaN and preserves all physical rows", + ); + } + Ok(ok) + } + pub fn run() -> Result<(), GateError> { bloomery_levers::at_main(&[])?; let gpu = Gpu::new()?; @@ -653,6 +964,13 @@ mod gate { ok &= flash_fault(&gpu, &flash, 9, 8, "flash count past ctx")?; ok &= flash_refusals(&gpu, &flash)?; ok &= rms_case(&gpu, &rms)?; + let loaded = LoadedAttention::load(&gpu)?; + for rotate in [false, true] { + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 1024, 3)?; + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 1024, 512)?; + ok &= loaded_case(&gpu, &loaded, rotate, 0, 3072, 3)?; + ok &= loaded_case(&gpu, &loaded, rotate, WINDOW, 3072, 3)?; + } if ok { Ok(()) } else { Err(checks_failed()) } } } diff --git a/crates/gpu-gates/src/bin/gate_flint_router.rs b/crates/gpu-gates/src/bin/gate_flint_router.rs index 790d7f152..f339bf6cf 100644 --- a/crates/gpu-gates/src/bin/gate_flint_router.rs +++ b/crates/gpu-gates/src/bin/gate_flint_router.rs @@ -1,18 +1,6 @@ -//! Flint's 256-expert top-6 sigmoid router (`sigmoid_256_6`), model-free. -//! -//! One token. The weight's column 0 holds the logits and the other columns -//! are 0; the activation is 1 at column 0, so each logit is one product. -//! Bias is 0 and the scale is 1. The logits fall strictly through the first -//! sixteen experts and sit at −20 after that, so the sixth and the seventh -//! do not tie. -//! -//! The slot count checked here is 6, not the instance's `N_USED`. The ids -//! equal the host top 6 of the kernel's scores plus bias (larger value, and -//! a tie to the larger index) and the host sigmoid ranking of the planted -//! logits. The weights are bit-identical to the host renorm of those scores -//! (f64 slot sum, narrowed, `renorm_divisor`, times the scale) and differ -//! from the first 6 weights of a top-7 renorm. A NaN logit raises `Router`; -//! a clean launch raises nothing. +//! The actual 256/top-6 selected-sum-floor binding, through RouterEntry. +//! Optional selection-only bias, active floor/zero sums, ties, exact rows, +//! nonfinite refusal and graph replay. This gate needs an NVIDIA device. #[cfg(feature = "gpu")] fn main() -> std::process::ExitCode { @@ -21,49 +9,83 @@ fn main() -> std::process::ExitCode { #[cfg(feature = "gpu")] mod gate { - use bloomery_gpu::route_body::sigmoid_256_6; - use bloomery_gpu::route_core::{renorm_divisor, sigmoid}; + use bloomery_gpu::gemm::Gemm32Kernels; + use bloomery_gpu::route_body::sigmoid_floor_256_6::{RouterKernels, RouterOut}; + use bloomery_gpu::route_core::sigmoid; + use bloomery_gpu::routed_block::{RouteIn, RoutedRows, RouterEntry}; use bloomery_gpu::{DeviceTensor, Fault, FaultSite, Gpu, LAYER_NONE}; use bloomery_gpu_gates::{GateError, bits_equal, checks_failed, verdict}; use cuda_core::DeviceBuffer; - /// Experts, the pick this gate names, and the activation width. The pick - /// is not read from the instance. const EXPERTS: usize = 256; const USED: usize = 6; - const TOP7: usize = 7; - const K: usize = 32; - const SCALE: f32 = 1.0; + const K: usize = 64; + const TOKENS: usize = 5; + const SCALE: f32 = 1.75; + const FLOOR: f32 = 1.0 / 16384.0; - fn logit(expert: usize) -> f32 { - if expert < 16 { - 20.0 - expert as f32 - } else { - -20.0 - } + struct Cx { + gpu: Gpu, + router: RouterKernels, + wide: Gemm32Kernels, + } + + #[derive(Default)] + struct Pick { + scores: Vec, + ids: Vec, + weights: Vec, } - fn weight_matrix() -> Vec { - let mut w = vec![0.0f32; EXPERTS * K]; - for e in 0..EXPERTS { - w[e * K] = logit(e); + impl Pick { + fn same(&self, other: &Self) -> bool { + bits_equal(&self.scores, &other.scores) + && self.ids == other.ids + && bits_equal(&self.weights, &other.weights) } - w } - /// The top `k` of `key`, larger value first, a tie to the larger index. fn top_k(key: &[f32], k: usize) -> Vec { - let mut idx: Vec = (0..key.len() as u32).collect(); - idx.sort_by(|&a, &b| key[b as usize].total_cmp(&key[a as usize]).then(b.cmp(&a))); - idx.truncate(k); - idx + let mut ids: Vec = (0..EXPERTS as u32).collect(); + ids.sort_by(|&a, &b| key[b as usize].total_cmp(&key[a as usize]).then(b.cmp(&a))); + ids.truncate(k); + ids } - /// Unbiased scores at `ids`, summed in slot order in f64, narrowed, guarded, scaled. + // Independent contract expression: do not reuse the device divisor. fn weights_of(scores: &[f32], ids: &[u32], scale: f32) -> Vec { - let g: Vec = ids.iter().map(|&i| scores[i as usize]).collect(); - let div = renorm_divisor(g.iter().fold(0.0f64, |s, &v| s + f64::from(v)) as f32); - g.into_iter().map(|v| v / div * scale).collect() + let sum = ids + .iter() + .fold(0.0f64, |sum, &e| sum + f64::from(scores[e as usize])); + let div = (sum as f32).max(FLOOR); + ids.iter() + .map(|&e| scores[e as usize] / div * scale) + .collect() + } + + fn matrix(logits: &[f32]) -> Vec { + let mut w = vec![0.0; EXPERTS * K]; + for (e, &logit) in logits.iter().enumerate() { + w[e * K] = logit; + } + w + } + + fn activations(factors: &[f32]) -> Vec { + let mut x = vec![0.0; factors.len() * K]; + for (t, &factor) in factors.iter().enumerate() { + x[t * K] = factor; + } + x + } + + fn read_one(cx: &Cx, out: &RouterOut) -> Result { + let s = cx.gpu.stream(); + Ok(Pick { + scores: out.probs.to_host_vec(s)?, + ids: out.ids.to_host_vec(s)?, + weights: out.weights.to_host_vec(s)?, + }) } fn clause(name: &str, pass: bool, detail: &str) -> bool { @@ -71,101 +93,391 @@ mod gate { pass } - pub fn run() -> Result<(), GateError> { - bloomery_levers::at_main(&[])?; - let gpu = Gpu::new()?; - println!("gate_flint_router: device {}", gpu.device_name()?); - let s = gpu.stream(); - let rk = sigmoid_256_6::RouterKernels::load(gpu.context())?; - let planted: Vec = (0..EXPERTS).map(logit).collect(); - let by_logit = top_k( - &planted.iter().copied().map(sigmoid).collect::>(), - USED, - ); - let w_host = weight_matrix(); - let x_host = { - let mut x = vec![0.0f32; K]; - x[0] = 1.0; - x + fn valid_case( + cx: &Cx, + name: &str, + logits: &[f32], + bias: Option<&[f32]>, + ) -> Result { + let s = cx.gpu.stream(); + let mut wh = matrix(logits); + if name == "dense" { + for e in 0..EXPERTS { + for j in 0..K { + wh[e * K + j] = ((e * 17 + j * 11) % 67) as f32 / 67.0 - 0.5; + } + } + } + let w = DeviceTensor::upload(s, &wh, EXPERTS, K)?; + let factors = [1.0, 0.0, 2.0, -1.0, 0.5]; + let mut xh = activations(&factors); + if name == "dense" { + for t in 0..TOKENS { + for j in 1..K { + xh[t * K + j] = ((t * 13 + j * 7) % 31) as f32 / 31.0 - 0.5; + } + } + } + let x = DeviceBuffer::from_host(s, &xh)?; + let b = bias.map(|b| DeviceBuffer::from_host(s, b)).transpose()?; + let input = RouteIn { + w: &w, + bias: b.as_ref(), + scale: SCALE, + fault: cx.gpu.unlabelled_sink(), }; - let bias_host = vec![0.0f32; EXPERTS]; - let w = DeviceTensor::upload(s, &w_host, EXPERTS, K)?; - let x = DeviceBuffer::from_host(s, &x_host)?; - let bias = DeviceBuffer::from_host(s, &bias_host)?; - let mut out = sigmoid_256_6::RouterOut::new(s)?; - gpu.take_fault()?; - rk.enqueue_router(s, &w, &x, &bias, SCALE, &mut out, gpu.unlabelled_sink())?; + let mut out = RouterOut::new(s)?; + let mut single = Pick::default(); + let mut first = Pick::default(); + let mut tickets = true; + cx.gpu.take_fault()?; + for t in 0..TOKENS { + let xt = DeviceBuffer::from_host(s, &xh[t * K..(t + 1) * K])?; + cx.router.route(s, &xt, &input, &mut out)?; + s.synchronize()?; + let pick = read_one(cx, &out)?; + tickets &= out.tickets(s)? == 0; + if t == 0 { + first = read_one(cx, &out)?; + } + single.scores.extend(pick.scores); + single.ids.extend(pick.ids); + single.weights.extend(pick.weights); + } + let mut probs = DeviceBuffer::zeroed(s, EXPERTS * TOKENS)?; + let mut ids = DeviceBuffer::zeroed(s, USED * TOKENS)?; + let mut weights = DeviceBuffer::zeroed(s, USED * TOKENS)?; + cx.router.route_rows( + (s, &cx.gpu, &cx.wide), + &x, + &input, + TOKENS, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, + )?; s.synchronize()?; - let scores = out.probs.to_host_vec(s)?; - let ids = out.ids.to_host_vec(s)?; - let weights = out.weights.to_host_vec(s)?; - let clean = gpu.take_fault()?; - - let by_score = top_k( - &scores - .iter() - .zip(&bias_host) - .map(|(&p, &b)| p + b) - .collect::>(), - USED, - ); - let len_ok = ids.len() == USED && weights.len() == USED; - let ids_ok = len_ok && ids == by_score && ids == by_logit; - let expect = weights_of(&scores, &by_score, SCALE); - let w_ok = weights.len() == USED && expect.len() == USED && bits_equal(&weights, &expect); - let by7 = top_k( - &scores + let batch = Pick { + scores: probs.to_host_vec(s)?, + ids: ids.to_host_vec(s)?, + weights: weights.to_host_vec(s)?, + }; + let clean = cx.gpu.take_fault()?.is_none(); + let mut expected = true; + for (t, &factor) in factors.iter().enumerate() { + let scores = &single.scores[t * EXPERTS..(t + 1) * EXPERTS]; + let keys: Vec = scores .iter() - .zip(&bias_host) - .map(|(&p, &b)| p + b) - .collect::>(), - TOP7, - ); - let w7 = weights_of(&scores, &by7, SCALE); - let differ = - weights.len() == USED && w7.len() == TOP7 && !bits_equal(&weights, &w7[..USED]); - + .enumerate() + .map(|(e, &p)| bias.map_or(p, |b| p + b[e])) + .collect(); + let want_ids = top_k(&keys, USED); + expected &= single.ids[t * USED..(t + 1) * USED] == want_ids; + if name != "dense" { + let planted_keys: Vec = logits + .iter() + .enumerate() + .map(|(e, &logit)| { + let p = sigmoid(logit * factor); + bias.map_or(p, |b| p + b[e]) + }) + .collect(); + expected &= single.ids[t * USED..(t + 1) * USED] == top_k(&planted_keys, USED); + } + expected &= bits_equal( + &single.weights[t * USED..(t + 1) * USED], + &weights_of(scores, &want_ids, SCALE), + ); + } let mut ok = clause( - "router ids top 6", - ids_ok && clean.is_none(), + &format!("{name} selection and unbiased floor weights"), + expected && clean && tickets, &format!( - "len={} ids={ids:?} by_score={by_score:?} by_logit={by_logit:?} fault={clean:?}", - ids.len() + "bias={} clean={clean} tickets_zero={tickets}", + bias.is_some() ), ); ok &= clause( - "router weights renorm of the top 6", - w_ok && clean.is_none(), - &format!("bit_identical={w_ok} slots={}", weights.len()), + &format!("{name} token-major rows"), + single.same(&batch), + "bit_identical", + ); + if name.starts_with("tiny") { + let sum: f32 = first.ids.iter().map(|&e| first.scores[e as usize]).sum(); + let legacy: Vec = first + .ids + .iter() + .map(|&e| first.scores[e as usize] / (sum + 1e-20) * SCALE) + .collect(); + ok &= clause( + "tiny floor active", + sum > 0.0 + && sum < FLOOR + && !bits_equal(&first.weights, &legacy) + && first.weights.iter().sum::() < SCALE, + "not epsilon renormalization", + ); + } + if name == "zero" || name == "ties" { + ok &= clause( + &format!("{name} descending-id ties"), + first.ids == [255, 254, 253, 252, 251, 250], + "six distinct ids", + ); + if name == "zero" { + ok &= clause( + "zero selected sum", + first.weights.iter().all(|&w| w == 0.0), + "finite zero weights", + ); + } + } + if let Some(bias) = bias { + let keys: Vec = first + .scores + .iter() + .zip(bias) + .map(|(&p, &b)| p + b) + .collect(); + ok &= clause( + "bias changes selection not weight source", + first.ids != top_k(&first.scores, USED) + && !bits_equal(&first.weights, &weights_of(&keys, &first.ids, SCALE)), + "unbiased selected scores consumed", + ); + } else if name == "normal" { + let top7 = top_k(&first.scores, 7); + ok &= clause( + "six not seven selected scores", + !bits_equal( + &first.weights, + &weights_of(&first.scores, &top7, SCALE)[..USED], + ), + "top-seven renormalization differs", + ); + } + + let x1 = DeviceBuffer::from_host(s, &xh[..K])?; + let graph = cx + .gpu + .capture(|st| cx.router.route(st, &x1, &input, &mut out))?; + let mut replay = graph.node_count() == 1; + for _ in 0..2 { + graph.launch(s)?; + s.synchronize()?; + replay &= read_one(cx, &out)?.same(&first) && out.tickets(s)? == 0; + } + ok &= clause( + &format!("{name} single capture"), + replay, + "one node, two exact replays, zero tickets", ); + let graph = cx.gpu.capture(|st| { + cx.router.route_rows( + (st, &cx.gpu, &cx.wide), + &x, + &input, + TOKENS, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, + ) + })?; + let mut replay = graph.node_count() == 3; + for _ in 0..2 { + graph.launch(s)?; + s.synchronize()?; + let now = Pick { + scores: probs.to_host_vec(s)?, + ids: ids.to_host_vec(s)?, + weights: weights.to_host_vec(s)?, + }; + replay &= now.same(&single); + } + replay &= cx.gpu.take_fault()?.is_none(); ok &= clause( - "router weights differ from a top-7 renorm", - differ && clean.is_none(), - &format!("differ={differ}"), + &format!("{name} rows capture"), + replay, + "three nodes, two exact replays, no fault", ); + Ok(ok) + } - let mut nan_w = w_host; - nan_w[40 * K] = f32::NAN; - let nan_w = DeviceTensor::upload(s, &nan_w, EXPERTS, K)?; - let mut nan_out = sigmoid_256_6::RouterOut::new(s)?; - gpu.take_fault()?; - rk.enqueue_router( - s, - &nan_w, + fn invalid_case( + cx: &Cx, + name: &str, + logit: Option, + bias: Option, + scale: f32, + ) -> Result { + let s = cx.gpu.stream(); + let mut logits = vec![0.0; EXPERTS]; + if let Some(logit) = logit { + logits[40] = logit; + } + let w = DeviceTensor::upload(s, &matrix(&logits), EXPERTS, K)?; + let mut bh = vec![0.0; EXPERTS]; + if let Some(bias) = bias { + bh[40] = bias; + } + let b = bias.map(|_| DeviceBuffer::from_host(s, &bh)).transpose()?; + let x = DeviceBuffer::from_host(s, &activations(&[1.0, 1.0]))?; + let input = RouteIn { + w: &w, + bias: b.as_ref(), + scale, + fault: cx.gpu.unlabelled_sink(), + }; + let mut out = RouterOut::new(s)?; + cx.gpu.take_fault()?; + cx.router.route(s, &x, &input, &mut out)?; + s.synchronize()?; + let one = cx.gpu.take_fault()?; + let one_poisoned = out.weights.to_host_vec(s)?.iter().all(|w| !w.is_finite()); + let tickets = out.tickets(s)?; + let mut probs = DeviceBuffer::zeroed(s, EXPERTS * 2)?; + let mut ids = DeviceBuffer::zeroed(s, USED * 2)?; + let mut weights = DeviceBuffer::zeroed(s, USED * 2)?; + cx.router.route_rows( + (s, &cx.gpu, &cx.wide), &x, - &bias, - SCALE, - &mut nan_out, - gpu.unlabelled_sink(), + &input, + 2, + RoutedRows { + probs: &mut probs, + ids: &mut ids, + weights: &mut weights, + }, )?; s.synchronize()?; - let nan_fault = gpu.take_fault()?; + let rows = cx.gpu.take_fault()?; + let rows_poisoned = weights.to_host_vec(s)?.iter().all(|w| !w.is_finite()); let want = Some(Fault::at(LAYER_NONE, FaultSite::Router)); - ok &= clause( - "router NaN logit", - nan_fault == want, - &format!("fault={nan_fault:?} want={want:?}"), - ); + Ok(clause( + name, + one == want && rows == want && one_poisoned && rows_poisoned && tickets == 0, + &format!( + "one={one:?} rows={rows:?} poisoned={} tickets={tickets}", + one_poisoned && rows_poisoned + ), + )) + } + + fn host_boundaries(cx: &Cx) -> Result { + let s = cx.gpu.stream(); + let w = DeviceTensor::upload(s, &matrix(&vec![0.0; EXPERTS]), EXPERTS, K)?; + let x = DeviceBuffer::zeroed(s, K)?; + let short_bias = DeviceBuffer::zeroed(s, EXPERTS - 1)?; + let mut out = RouterOut::new(s)?; + let sink = cx.gpu.unlabelled_sink(); + let short = cx + .router + .enqueue_router(s, &w, &x, Some(&short_bias), SCALE, &mut out, sink) + .is_err(); + let mut probs = DeviceBuffer::zeroed(s, EXPERTS)?; + let mut ids = DeviceBuffer::zeroed(s, USED)?; + let mut weights = DeviceBuffer::zeroed(s, USED)?; + let empty = cx + .router + .enqueue_router_rows( + s, + &cx.gpu, + &cx.wide, + &w, + &x, + None, + SCALE, + 0, + &mut probs, + &mut ids, + &mut weights, + sink, + ) + .is_err(); + let short_rows = cx + .router + .enqueue_router_rows( + s, + &cx.gpu, + &cx.wide, + &w, + &x, + None, + SCALE, + 2, + &mut probs, + &mut ids, + &mut weights, + sink, + ) + .is_err(); + let legacy = + bloomery_gpu::route_body::sigmoid_256_6::RouterKernels::load(cx.gpu.context())?; + let mut legacy_out = bloomery_gpu::route_body::sigmoid_256_6::RouterOut::new(s)?; + let missing = legacy + .route( + s, + &x, + &RouteIn { + w: &w, + bias: None, + scale: SCALE, + fault: sink, + }, + &mut legacy_out, + ) + .is_err(); + Ok(clause( + "host boundaries", + short && empty && short_rows && missing, + "short bias, empty/short rows, legacy missing bias refused", + )) + } + + pub fn run() -> Result<(), GateError> { + bloomery_levers::at_main(&[])?; + let gpu = Gpu::new()?; + println!("gate_flint_router: device {}", gpu.device_name()?); + let router = RouterKernels::load(gpu.context())?; + let wide = Gemm32Kernels::load(gpu.context())?; + let cx = Cx { gpu, router, wide }; + let normal: Vec = (0..EXPERTS) + .map(|e| if e < 16 { 2.0 - e as f32 * 0.25 } else { -20.0 }) + .collect(); + let tiny: Vec = (0..EXPERTS) + .map(|e| -30.0 - (e % 16) as f32 * 0.25) + .collect(); + let mut bias = vec![0.0; EXPERTS]; + for (e, b) in bias.iter_mut().enumerate().skip(240) { + *b = 2.0 + e as f32 / 256.0; + } + let mut ok = valid_case(&cx, "normal", &normal, None)?; + ok &= valid_case(&cx, "dense", &normal, None)?; + ok &= valid_case(&cx, "biased", &normal, Some(&bias))?; + ok &= valid_case(&cx, "tiny", &tiny, None)?; + ok &= valid_case(&cx, "tiny-biased", &tiny, Some(&bias))?; + ok &= valid_case(&cx, "zero", &vec![-200.0; EXPERTS], None)?; + ok &= valid_case(&cx, "ties", &vec![0.0; EXPERTS], None)?; + for (name, value) in [ + ("NaN", f32::NAN), + ("+infinity", f32::INFINITY), + ("-infinity", f32::NEG_INFINITY), + ] { + ok &= invalid_case(&cx, &format!("{name} logit"), Some(value), None, SCALE)?; + ok &= invalid_case( + &cx, + &format!("{name} selection bias"), + None, + Some(value), + SCALE, + )?; + } + ok &= invalid_case(&cx, "nonfinite weight scale", None, None, f32::INFINITY)?; + ok &= host_boundaries(&cx)?; if ok { Ok(()) } else { Err(checks_failed()) } } } diff --git a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs index 3d30bbeda..6e72e0d28 100644 --- a/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_glm5next_twocard.rs @@ -192,7 +192,8 @@ mod gate { use crate::glm5next_tier; use crate::quiet::Quiet; - use crate::tier_twocard::{self, Leg, Sent, plus, same, sent_between, sent_every, union_plan}; + use crate::tier_twocard::{self, Leg, Sent, plus, same, sent_between, sent_every}; + use model::placement::union_plan; const NAME: &str = "gate_glm5next_twocard"; /// The card budget both plans are made under (module header: the union diff --git a/crates/gpu-gates/src/bin/gate_minimaxm3_prefill.rs b/crates/gpu-gates/src/bin/gate_minimaxm3_prefill.rs index ba0461931..e2eea9a2d 100644 --- a/crates/gpu-gates/src/bin/gate_minimaxm3_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_minimaxm3_prefill.rs @@ -37,7 +37,7 @@ //! The two-card arm (`--arm bp`, `just gate-gpu-minimaxm3-twocard` under `BLOOMERY_CARD=both`): plan (b′) (the A6000 the //! stage card, the 3090 an expert tier holding each layer's next ids after the stage's prefix, the host the rest), planned //! under one card budget no environment moves (`seat::TWO_BUDGET`), in the real tier only. Before the common clauses it holds, against the reference — the plan with every tier segment on the stage card -//! (`tier_twocard::union_plan`), the A6000 alone, opened first and released: +//! (`model::placement::union_plan`), the A6000 alone, opened first and released: //! - `two_card_load`: the plan puts experts on the tier, and the load's tier opened on the 3090 beside the stage card with //! the plan's experts on each layer; //! - `two_card_first_tokens_are_union`: [`TWO_WALK`] decode steps of the walk read the reference's ids and logits rows bit @@ -490,12 +490,7 @@ mod gate { ) -> Result<(), GateError> { // The walk's positions too: the reference and the two-card load hold the same cache rows. let ctx = ctx.max(TWO_WALK + 1); - let mut two = seat::two_card( - path, - (ctx, tier_now), - (TWO_WALK, crate::e2e::walk), - crate::tier_twocard::union_plan, - )?; + let mut two = seat::two_card(path, (ctx, tier_now), (TWO_WALK, crate::e2e::walk))?; let (pass, layers) = two_card_clauses(&mut two)?; let mut loaded = two.loaded; let dims = loaded.dims(); diff --git a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs index 5137e5828..62dc44357 100644 --- a/crates/gpu-gates/src/bin/gate_musespark_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_musespark_prefill.rs @@ -83,7 +83,7 @@ //! ids and logits bits, every logit finite, and the walk sent every tier layer a routed slot (or the run proves nothing //! there and fails by name); //! - `two_card_first_token_near_union`: the first token's layers against the reference — the plan with every tier -//! segment on the stage card ([`tier_twocard::union_plan`]), the A6000 alone, opened first and released — each layer +//! segment on the stage card ([`model::placement::union_plan`]), the A6000 alone, opened first and released — each layer //! teacher-forced: the step runs that layer alone (`GpuModel::set_layer_span`) on the reference's `l_out` of the layer //! before it (`GpuModel::set_input_override`; layer 0 reads the embedding), keeps the reference's picks and keeps //! `ffn_moe_out` and `l_out` within [`TWO_STEP_BAND`]; the token sent the tier a slot on one of the tier's layers. The @@ -155,9 +155,7 @@ mod gate { use crate::gate_card; use crate::lcpp_e2e::{self, Dims, Seat, fnv1a}; use crate::prefill_clauses::{self, Case}; - use crate::tier_twocard::{ - Sent, sent_between, sent_every, tier_counts, tier_on_3090, union_plan, - }; + use crate::tier_twocard::{Sent, sent_between, sent_every, tier_counts, tier_on_3090}; use app::Prompt; use bloomery_gpu::arch::musespark::{Body, LayerNodes, LayerRoute}; use bloomery_gpu::host::served::TierBody; @@ -171,6 +169,7 @@ mod gate { use model::arch::Arch; use model::arch::musespark::hparams::Hparams; use model::arch::musespark::place::{self, PlanInputs}; + use model::placement::union_plan; use model::placement::{Machine, Plan, PlanLevers, workstation}; type Model = GpuModel; diff --git a/crates/gpu-gates/src/bin/gate_qwen3moe_twocard.rs b/crates/gpu-gates/src/bin/gate_qwen3moe_twocard.rs index 8fa38388f..c054bdd01 100644 --- a/crates/gpu-gates/src/bin/gate_qwen3moe_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_qwen3moe_twocard.rs @@ -87,8 +87,9 @@ mod gate { use crate::e2e::walk; use crate::q3place::PlaceQ3; use crate::tier_twocard::{ - Leg, Sent, same, sent_between, sent_every, tier_counts, tier_on_3090, union_plan, + Leg, Sent, same, sent_between, sent_every, tier_counts, tier_on_3090, }; + use model::placement::union_plan; const NAME: &str = "gate_qwen3moe_twocard"; /// The card budget both plans are made under in the real tier (module header). diff --git a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs index 9bcd34bc7..a660da817 100644 --- a/crates/gpu-gates/src/bin/gate_solar2_twocard.rs +++ b/crates/gpu-gates/src/bin/gate_solar2_twocard.rs @@ -88,7 +88,8 @@ mod gate { use crate::e2e::walk; use crate::quiet::Quiet; - use crate::tier_twocard::{Leg, Sent, same, sent_between, sent_every, union_plan}; + use crate::tier_twocard::{Leg, Sent, same, sent_between, sent_every}; + use model::placement::union_plan; const NAME: &str = "gate_solar2_twocard"; /// The card budget both plans are made under in the real tier (module header: the union it leaves fits the A6000). @@ -298,6 +299,7 @@ mod gate { host: levers.host(), residency: Residency::Off, group: 1, + slots: std::num::NonZeroUsize::MIN, }; let pass = bits(&file, &cfg, &inputs)?; println!("{NAME}: {}", tier::tally_line()); diff --git a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs index cbf918642..ea316e6e5 100644 --- a/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs +++ b/crates/gpu-gates/src/bin/shared/mimo2_twocard.rs @@ -2,7 +2,7 @@ //! `BLOOMERY_CARD=both`): plan (b′) (`workstation::plan_bp`: the A6000 the stage card, the 3090 an expert tier holding //! each routed layer's next ids after the stage's prefix, the host the rest), planned under one card budget no //! environment moves ([`TWO_BUDGET`]), against the reference — the same plan with every tier segment on the stage card -//! (`tier_twocard::union_plan`), the A6000 alone, opened first and released. Its clauses: +//! (`model::placement::union_plan`), the A6000 alone, opened first and released. Its clauses: //! - `two_card_load`: the plan puts experts on the tier, and the load's tier opened on the 3090 beside the stage card //! with the plan's experts on each layer; //! - `two_card_first_tokens_are_union`: [`TWO_WALK`] decode steps of the walk read the reference's ids and logits rows @@ -26,11 +26,9 @@ use bloomery_gpu_mimo2::{Body, Mimo2Model, T_MAX, prefill, set_prefill_group}; use gguf::Split; use model::arch::mimo2::place::PlanInputs; use model::placement::workstation; -use model::placement::{Machine, PlanLevers}; +use model::placement::{Machine, PlanLevers, union_plan}; -use crate::tier_twocard::{ - Leg, Sent, same, sent_between, sent_every, tier_counts, tier_on_3090, union_plan, -}; +use crate::tier_twocard::{Leg, Sent, same, sent_between, sent_every, tier_counts, tier_on_3090}; /// What the arm's errors name. const WHAT: &str = "gate_mimo2_e2e two-card"; diff --git a/crates/gpu-gates/src/bin/shared/minimaxm3_seat.rs b/crates/gpu-gates/src/bin/shared/minimaxm3_seat.rs index 1573268fd..df9603efe 100644 --- a/crates/gpu-gates/src/bin/shared/minimaxm3_seat.rs +++ b/crates/gpu-gates/src/bin/shared/minimaxm3_seat.rs @@ -27,7 +27,7 @@ use gguf::Split; use model::arch::minimaxm3::hparams::FfnKind; use model::arch::minimaxm3::place::PlanInputs; use model::placement::workstation::{self, HostRead, PLAN_ROOM}; -use model::placement::{Machine, Plan, PlanLevers}; +use model::placement::{Machine, PlanLevers}; use refset::arch::minimaxm3; use std::num::NonZeroUsize; use std::path::Path; @@ -235,15 +235,11 @@ pub struct TwoCard { pub opened: std::time::Instant, } -/// A plan's union on the stage card alone (the gate's `tier_twocard::union_plan`): every tier segment joined to the stage -/// card's, `flat` the machine without the tier. -pub type Union = for<'a, 'b> fn(&'b Plan<'a>, &'a Machine) -> Result, GateError>; - /// The steps' token walk: `walk(steps, vocab)` gives the ids the reference decodes. pub type Walk = fn(usize, usize) -> Vec; /// The two-card arm's loads at `ctx` positions, one after the other: plan (b′) (`workstation::plan_bp`, the stage the A6000, -/// the tier the 3090) under one card budget no environment moves ([`TWO_BUDGET`]); the reference, that plan's `union` on the A6000 alone, its ids and logits rows read over the walk +/// the tier the 3090) under one card budget no environment moves ([`TWO_BUDGET`]); the reference, that plan's union on the A6000 alone, its ids and logits rows read over the walk /// `walk(steps, vocab)` and released; then plan (b′) itself. A budget the environment sets, a union past the A6000's usable /// bytes, a reference opened elsewhere or holding a tier, and the fixture tier (its cut's routed layers fit the stage card, /// so the tier would hold nothing) are refused by name. @@ -251,7 +247,6 @@ pub fn two_card( file: &Path, (ctx, tier): (usize, Tier), (steps, walk): (usize, Walk), - union: Union, ) -> Result { if tier == Tier::Fixture { return Err("the two-card arm runs in the real tier: the fixture cut's routed layers fit the stage card".into()); @@ -302,7 +297,7 @@ pub fn two_card( } let host = levers.host(); let reference = { - let uplan = union(&plan, &flat)?; + let uplan = model::placement::union_plan(&plan, &flat)?; let mut r = Body::open_placed(minimaxm3_folder::model_at(file)?, &uplan, &inputs, 0, host)?; r.set_mode(StepMode::Graph); if !r.body("minimaxm3 seat")?.hybrid().tiers().is_empty() { diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs b/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs new file mode 100644 index 000000000..e74fdfcf8 --- /dev/null +++ b/crates/gpu-gates/src/bin/shared/serve_seats/flint.rs @@ -0,0 +1,88 @@ +//! Flint's actual slot-capable body behind the common hybrid HTTP seat. +//! Chat, tools, reasoning and end-of-generation tokens use the canonical model contract. + +use bloomery_gpu_gates::generate::Place; +use bloomery_gpu_gates::{GateError, record, side_seat}; +use gguf::Split; +use model::arch::Arch; +use model::arch::flint::place; +use model::arch::place_hybrid::PlanInputs; +use model::placement::PlacementError; +use model::placement::workstation::TierBatchBytes; +use serve::ServeError; + +use super::hybrid::Family; + +const NAME: &str = "bloomery-serve-flint"; +const USAGE: &str = "usage: bloomery-serve --model flint [-m PATH | --hf [:]] \ + [--host H] [--port P] [--ctx C] [--parallel N] [--alias NAME] \ + [--chat-template-file PATH] [--place W] [--plan] [--queue-depth Q] \ + [--slot-save-path DIR] [--api-key KEY] [--api-key-file FNAME]"; + +/// The common placement, host load, residency, prompt-group and caller settings. +pub const ACTS_ON: &[&str] = &[ + bloomery_levers::CARD_BUDGET, + bloomery_levers::RESIDENCY, + bloomery_levers::PREFILL_GROUP, + bloomery_levers::HOST_POPULATE, + bloomery_levers::HOST_LOCK, + bloomery_levers::CARD_DONTNEED, + bloomery_levers::R8, + bloomery_levers::PIN_MAIN, + bloomery_levers::HOST_LANES, + bloomery_levers::SCREEN, + bloomery_levers::SCHEMAJUMP, +]; + +/// Architecture-specific facts; the common owner plans and serves every slot. +pub struct Flint; + +impl Family for Flint { + type Body = bloomery_gpu_flint::Body; + type H = (); + + const NAME: &'static str = self::NAME; + const USAGE: &'static str = self::USAGE; + const ACTS_ON: &'static [&'static str] = self::ACTS_ON; + const ALIAS: &'static str = "flint"; + const FAMILY: &'static str = "flint"; + const SERVES: &'static str = "flint"; + + fn accepts(arch: Arch) -> bool { + matches!(arch, Arch::Flint) + } + + fn read(split: &Split) -> Result, PlacementError> { + place::PlanInputs::read(split) + } + + fn configure(inputs: &mut PlanInputs<()>, group: usize) -> Result<(), GateError> { + Ok(place::configure_group(inputs, group)?) + } + + fn register() { + record::at_main(NAME, record::BLOOMERY_SERVE_HYBRID); + } + + fn listening() -> record::Record { + record::Record::new(&record::LISTENING_HYBRID) + } + + fn load_record() -> record::Record { + record::Record::new(&record::LOAD_HYBRID) + } + + fn tier_batch( + place: Place, + inputs: &PlanInputs<()>, + ) -> Result, GateError> { + Ok((!place.tier_cards().is_empty()) + .then(|| place::tier_batch(inputs)) + .transpose()?) + } +} + +/// Opens the real Flint session and serves resident sequences by select and step. +pub fn run(args: &[String], side: &side_seat::Flags) -> Result { + super::hybrid::run::(args, side) +} diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs index 495f265e3..0a91647cc 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/hybrid.rs @@ -1,6 +1,6 @@ -//! The seat of a body that runs one chain on one card, fed in prompt batches: -//! the one owner of the open, the load record, the seat's calls and the think -//! tags, for the family whose body holds one sequence ([`solar2`](super::solar2)). +//! A common hybrid HTTP seat over one chain on one card: opening, planning, +//! resident sequence selection, load records and prompt calls. Its body's +//! sequence capability preserves named refusals for single-sequence families. //! //! A seat gives its family the facts the run needs — the architectures it //! accepts, its plan inputs, the words its refusals name and the records it @@ -13,8 +13,8 @@ use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, Instant}; -use app::arch::HybridCfg; -use app::{Keep, Loaded, OpenArgs, OpenLog, Prompt, Session, SessionError}; +use app::arch::{Batch, HybridCfg, SequenceSlots}; +use app::{Loaded, OpenArgs, OpenLog, Prompt, Session, SessionError}; use bloomery_gpu::GpuModel; use bloomery_gpu::model::{ChainBody, StepMode}; use bloomery_gpu_gates::GateError; @@ -40,7 +40,7 @@ use super::tiered::{self, Open, Tiered}; pub trait Family: Sized + 'static { /// The body the seat serves: one chain on one card, fed in prompt /// batches. - type Body: ChainBody + app::Open, Cfg = HybridCfg> + Prompt + Keep; + type Body: Batch>; /// The family's hyperparameters: what its plan inputs are read for. type H; @@ -64,6 +64,11 @@ pub trait Family: Sized + 'static { /// The file's plan inputs, its own reading. fn read(split: &Split) -> Result, PlacementError>; + /// Price the chosen prompt group before the common context probes. + fn configure(_inputs: &mut PlanInputs, _group: usize) -> Result<(), GateError> { + Ok(()) + } + /// The records the seat prints, registered once at `main`. fn register(); @@ -118,6 +123,7 @@ struct Log { opened: Duration, ctx: usize, group: usize, + slots: usize, host_experts: u64, card_experts: u64, graph_nodes: usize, @@ -160,7 +166,7 @@ impl OpenLog for Log { } impl Log { - /// The `load` record of `m`, the one slot ready. + /// The `load` record of `m`, with every planned slot ready. fn record(&self, m: &GpuModel) { if let Some((card, rate)) = &self.lane { record::lane(card, rate).eprint(); @@ -170,7 +176,7 @@ impl Log { .w("arch", ::arch().name()) .u("resident_bytes", m.resident_bytes()) .u("ctx", self.ctx) - .u("slots", 1) + .u("slots", self.slots) .u("layers", m.layers().len()) .u("host_experts", self.host_experts) .u("card_experts", self.card_experts) @@ -196,6 +202,7 @@ fn open_session(a: Open<()>) -> Result, GateError> { opened: Duration::ZERO, ctx: a.ctx, group: a.group, + slots: a.slots.get(), pin_main: a.pin_main, pinned, host_experts: 0, @@ -220,16 +227,26 @@ fn open_session(a: Open<()>) -> Result, GateError> { host: a.host, residency: a.residency, group: a.group, + slots: a.slots, }, }; - let s = Loaded::::open(file, args, &mut log)? + let mut s = Loaded::::open(file, args, &mut log)? .ok_or("the open stopped at its plan")? .ready(&mut log)?; + ::Sequences::add(&mut s, a.slots)?; log.record(s.model()); Ok(Hybrid { s, ctx: a.ctx }) } impl Seat for Hybrid { + fn slots(&self) -> usize { + ::Sequences::count(&self.s) + } + + fn select(&mut self, slot: usize) -> Result<(), GateError> { + Ok(::Sequences::select(&mut self.s, slot)?) + } + fn pos(&self) -> usize { self.s.model().pos() as usize } @@ -309,7 +326,8 @@ impl Tiered for Hybrid { const ALIAS: &'static str = F::ALIAS; const FAMILY: &'static str = F::FAMILY; const SERVES: &'static str = F::SERVES; - const RESIDENT: bool = false; + const RESIDENT: bool = <::Sequences as SequenceSlots>::RESIDENT; + const ONE_PASS: bool = false; fn accepts(arch: Arch) -> bool { F::accepts(arch) @@ -319,6 +337,10 @@ impl Tiered for Hybrid { F::read(split) } + fn configure(inputs: &mut PlanInputs, group: usize) -> Result<(), GateError> { + F::configure(inputs, group) + } + fn flags(_flag: &str, _value: &str, _out: &mut ()) -> Result { Ok(false) } diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs index b5b52bb18..edf2105a2 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/mod.rs @@ -37,6 +37,8 @@ mod drafted; // DSpark draft, its slot rule and the draft word (`served::ServedDraft`). #[cfg(feature = "deepseek41")] pub mod ds41; +#[cfg(all(feature = "deepseek41", feature = "flint"))] +pub mod flint; #[cfg(feature = "glm5next")] pub mod glm; #[cfg(feature = "deepseek41")] @@ -80,7 +82,7 @@ pub mod minimaxm3_folder; pub mod musespark; // The seat of a body that runs one chain on one card, fed in prompt batches // (`solar2`): its open, its load record, its calls and its think tags. -#[cfg(all(feature = "deepseek41", feature = "solar2"))] +#[cfg(all(feature = "deepseek41", any(feature = "solar2", feature = "flint")))] pub mod hybrid; // The nemotron seat sits behind `deepseek41` for the same reason: the server // surface it binds through is scoped to it, and it runs no V4.1 code. The diff --git a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs index 766804fc5..0ba36629b 100644 --- a/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs +++ b/crates/gpu-gates/src/bin/shared/serve_seats/tiered.rs @@ -228,6 +228,12 @@ pub trait Tiered: Seat + Sized + 'static { /// The file's plan inputs, its own `place::read`. fn read(split: &Split) -> Result, PlacementError>; + /// Price the selected prompt schedule before any context probe or allocation. + /// A family whose schedule has fixed buffers keeps its header-derived inputs. + fn configure(_inputs: &mut PlanInputs, _group: usize) -> Result<(), GateError> { + Ok(()) + } + /// One of the seat's own flags (`SeatFlags::parse`'s `own`): `Ok(true)` took it. fn flags(flag: &str, value: &str, out: &mut Self::Flags) -> Result; @@ -487,7 +493,8 @@ pub fn run(args: &[String], side: &side_seat::Flags) -> Result(plan: &Plan<'a>, flat: &'a Machine) -> Result, GateError> { - let model = plan.model; - let tier = Device::Card(plan.machine.cards.len()); - let mut out = plan.clone(); - for row in &mut out.rows { - let t = &model.tensors[row.tensor]; - if t.role != Role::RoutedExperts { - continue; - } - let ids = |d: Device| -> Vec { - row.segments - .iter() - .filter(|s| s.device == d) - .filter_map(|s| s.experts.as_ref()) - .flat_map(|e| e.ids().iter().copied()) - .collect() - }; - let on_tier = ids(tier); - if on_tier.is_empty() { - continue; - } - let mut union = ids(Device::Card(0)); - union.extend(on_tier); - let n = union.len(); - let card = placement::routed_row( - row.tensor, - t, - 0, - ExpertList::new(union, model.experts)?, - model, - )?; - let segments = card - .segments - .into_iter() - .filter(|s| s.device == Device::Card(0)) - .chain( - row.segments - .iter() - .filter(|s| !matches!(s.device, Device::Card(_))) - .cloned(), - ) - .collect(); - *row = Row { - segments, - ..row.clone() - }; - if let Some(l) = t.layer.and_then(|l| out.n_l.get_mut(l)) { - *l = u64::try_from(n)?; - } - } - out.machine = flat; - out.cards.truncate(flat.cards.len()); - out.tier_n_l.clear(); - Ok(out) -} /// Every token, kept count and logits row a leg read, in order. #[derive(Default)] diff --git a/crates/gpu-gates/src/record.rs b/crates/gpu-gates/src/record.rs index 9a4254951..9b6451f1b 100644 --- a/crates/gpu-gates/src/record.rs +++ b/crates/gpu-gates/src/record.rs @@ -2006,6 +2006,43 @@ pub static LISTENING_SOLAR2: Kind = Kind { ], }; +/// A common hybrid seat's loaded model and resident sequence stores. +pub static LOAD_HYBRID: Kind = Kind { + name: "load_hybrid", + head: "load", + doc: "A hybrid seat's architecture, resident bytes, per-slot context, resident slots, layers, host and card expert counts, captured step nodes, prompt group, caller pin and load wall.", + parts: &[ + key("arch", Word, ""), + key("resident_bytes", U64, "B"), + key("ctx", U64, "positions"), + key("slots", U64, ""), + key("layers", U64, ""), + key("host_experts", U64, "experts"), + key("card_experts", U64, "experts"), + key("graph_nodes", U64, ""), + key("group", U64, "batches"), + key("pin_main", Word, ""), + key("pinned", Bool, ""), + lit(" in "), + pos("load_s", F64(1), "s"), + lit(" s"), + ], +}; + +/// The common hybrid seat's listening address. +pub static LISTENING_HYBRID: Kind = Kind { + name: "listening_hybrid", + head: "bloomery-serve:", + doc: "A hybrid seat's placement, per-slot context, resident slots and listening address.", + parts: &[ + key("place", Word, ""), + key("ctx", U64, "positions"), + key("slots", U64, ""), + lit(" listening on http://"), + pos("addr", Word, ""), + ], +}; + /// The MiniMax-M3 seat's load. pub static LOAD_MINIMAXM3: Kind = Kind { name: "load_minimaxm3", @@ -3060,6 +3097,19 @@ pub static BLOOMERY_SERVE_SOLAR2: &[&Kind] = kinds![ groups::bind::CACHE, ]; +/// The common hybrid seat's placement, residency, load, address and request records. +pub static BLOOMERY_SERVE_HYBRID: &[&Kind] = kinds![ + groups::side_seat::SEAT, + groups::tiered::LEVER, + &[&PLACE_UNSET], + groups::tiered::RUN, + groups::residency38::HOST, + groups::fetch::FETCH, + groups::fetch::SET, + &[&LANE, &LOAD_HYBRID, &LOAD_PHASES, &LISTENING_HYBRID], + groups::bind::CACHE, +]; + /// What the MiniMax-M3 seat of `bloomery-serve` prints, all on stderr: the /// placement and why (`place unset`), a `--hf` repo's resolve lines (printed by /// the server before the seat registers), the residency lever's word and why, @@ -4340,11 +4390,11 @@ mod tests { #[test] fn every_binary_registers_the_groups_its_entries_require() { use groups::{ - bind, drafted, ds41_draft, ds41_split, fetch, host_stats, nvtier_family, + bind, drafted, ds41_draft, ds41_split, fetch, host_stats, hybrid, nvtier_family, qwen3moe_place, residency38, serve_client, side_seat, tiered, vision_seat, }; type Group = &'static [&'static Kind]; - let table: [(&str, Group, &[Group]); 17] = [ + let table: [(&str, Group, &[Group]); 18] = [ ( "ds41 seat: Flags::encoder, Draft::from_levers, print_widths, bind::spawn", BLOOMERY_SERVE_DS41, @@ -4412,6 +4462,20 @@ mod tests { bind::CACHE, ], ), + ( + "hybrid seat: residency_pick, settle_pick, hybrid::open_session, bind::spawn", + BLOOMERY_SERVE_HYBRID, + &[ + side_seat::SEAT, + tiered::LEVER, + tiered::RUN, + residency38::HOST, + hybrid::GROUP, + fetch::FETCH, + fetch::SET, + bind::CACHE, + ], + ), ( "minimaxm3 seat: residency_pick, residency_set, bind::spawn", BLOOMERY_SERVE_MINIMAXM3, diff --git a/crates/gpu/src/arch/musespark/body/tier.rs b/crates/gpu/src/arch/musespark/body/tier.rs index 0c56518f0..cce7df8ac 100644 --- a/crates/gpu/src/arch/musespark/body/tier.rs +++ b/crates/gpu/src/arch/musespark/body/tier.rs @@ -30,6 +30,7 @@ use model::arch::models::{LayerSpec, Moe}; use model::arch::musespark::place::{self, PlanInputs}; use model::moe::HostLayer; use model::placement::ModelTensors; +use model::placement::accepts::Program; use model::r8file::R8Source; use crate::card_slots::{CardAcc, CardSlotKernels, ColPlaces}; @@ -87,7 +88,7 @@ impl LegFamily for MuseSpark { n_used: hp.n_used, n_embd: hp.latent, ff: hp.expert_width(), - program: model::placement::accepts::Program::MuseSparkBody, + program: Program::MuseSparkBody, } } diff --git a/crates/gpu/src/arch/qwen3moe/tier_leg.rs b/crates/gpu/src/arch/qwen3moe/tier_leg.rs index de8854b21..b47367c6b 100644 --- a/crates/gpu/src/arch/qwen3moe/tier_leg.rs +++ b/crates/gpu/src/arch/qwen3moe/tier_leg.rs @@ -1,8 +1,7 @@ //! The Qwen3-30B and Qwen3.6 expert tier: one card beside the stage card, //! hung through the common card leg ([`crate::host::leg_tier`]). What is -//! Qwen's here is the node family — the file's reading, `silu(g)·u`, and the -//! placed leg's arms — and the join of the tier's slots with the stage -//! card's. +//! Qwen's here is the node family — the file's reading and `silu(g)·u` — +//! and the join of the tier's slots with the stage card's. //! //! The join is the host leg's combine (`hostleg`) with the tier's slots in //! the same slot order. A slot the stage card holds is the stage card's down; diff --git a/crates/gpu/src/attention_h128.rs b/crates/gpu/src/attention_h128.rs new file mode 100644 index 000000000..ee8fbd1ca --- /dev/null +++ b/crates/gpu/src/attention_h128.rs @@ -0,0 +1,617 @@ +//! Loaded head-128 GQA: independent rotation and readable-window policies over f16 KV stores. +//! +//! KV retention belongs to the runtime's Store policy. This binding owns no logical cursor: +//! the caller supplies absolute positions and causal counts, and exchanges only cache planes +//! between slots. Ring appends are cut to `capacity - window + 1` rows before attention, so +//! a later row never overwrites a key still readable by the first row of the chunk. + +use crate::attention_h128_ring; +use crate::flash_gqa::{GROUP, HEAD, window_segments}; +use crate::flash_gqa_h128_swa::{H128SwaArgs, H128SwaKernels}; +use crate::gemm::{GEMM_MAX_SLOTS, Gemm32Kernels, GemmInput, GemmKernels, GemmRoute}; +use crate::qk_rms::{QkRmsArgs, QkRmsKernels}; +use crate::rope_neox::{NeoxArgs, RopeNeoxKernels}; +use crate::rope_table::RopeRows; +use crate::site::{self, SiteTy, SiteWeight, WideActs, WideForms, WideKernels}; +use crate::weights::Weights; +use crate::{Gpu, GpuError}; +use cuda_core::DeviceBuffer; +use gguf::Split; +use model::arch::models::{LayerSpec, Mixer, OutGate, RopeMode}; +use model::arch::names; +use model::placement::kernels::attention::{Forms, Head128Scratch}; + +const WHAT: &str = "attention_h128"; +const SITES: &[SiteTy] = &[ + SiteTy::Q3K, + SiteTy::Q4K, + SiteTy::Q5K, + SiteTy::Q6K, + SiteTy::Iq4Xs, + SiteTy::Q5_0, + SiteTy::Q8_0, + SiteTy::F32, + SiteTy::Mxfp8, + SiteTy::Fp8B128, + SiteTy::Bf16, +]; + +struct Projection { + name: String, + ty: SiteTy, + rows: usize, +} + +impl Projection { + fn load( + file: &Split, + w: &Weights, + name: String, + rows: usize, + cols: usize, + ) -> Result { + let ty = site::file_site(file, WHAT, &name, (rows, cols), SITES)?; + site::site(w, WHAT, &name, (rows, cols), ty)?; + Ok(Self { name, ty, rows }) + } + + #[allow( + clippy::too_many_arguments, + reason = "one typed projection's borrowed launch state (rust-quality R8)" + )] + fn enqueue( + &self, + gpu: &Gpu, + w: &Weights, + k: &WideKernels<'_>, + a: &WideActs, + x: &DeviceBuffer, + route: &GemmRoute, + m: usize, + y: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + site::gemm( + gpu, + k, + (self.ty, w, SiteWeight::Resident(&self.name)), + self.rows, + &a.input(x), + (route, GemmInput::PerSlot), + m, + y, + ) + } +} + +/// A layer's loaded sites, gains, sinks, and independent attention policies. +pub struct Binding { + layer: usize, + hidden: usize, + heads: usize, + kv: usize, + ctx: usize, + capacity: usize, + window: usize, + rotate: bool, + eps: f32, + norm: String, + qnorm: String, + knorm: String, + sinks: String, + q: Projection, + k: Projection, + v: Projection, + o: Projection, +} + +fn vector(file: &Split, w: &Weights, name: String, len: usize) -> Result { + let (_, t) = file + .find(&name) + .ok_or_else(|| GpuError::tensor(WHAT, &name, "a vector"))?; + if t.dims != [len as u64] { + return Err(GpuError::shape( + WHAT, + format!("{name} has shape {:?}; need [{len}]", t.dims), + )); + } + let mut values = vec![0.0; len]; + gguf::quant::dequant_row(t.ty, file.data(&name)?, &mut values) + .map_err(::model::ModelError::from)?; + if let Some(i) = values.iter().position(|x| !x.is_finite()) { + return Err(GpuError::shape(WHAT, format!("{name}[{i}] is not finite"))); + } + site::vector(w, WHAT, name, len) +} + +impl Binding { + /// Bind actual file and resident formats once. No architecture-name dispatch is involved. + #[allow( + clippy::too_many_arguments, + reason = "the loaded layer and its geometry (rust-quality R8)" + )] + pub fn load( + file: &Split, + w: &Weights, + layer: usize, + spec: &LayerSpec, + hidden: usize, + eps: f32, + ctx: usize, + ) -> Result { + let Mixer::Gqa(g) = &spec.mixer else { + return Err(GpuError::shape(WHAT, "a GQA layer is required")); + }; + let heads = g.heads as usize; + let kv = g.kv_heads as usize; + let rotate = match g.rope.mode { + RopeMode::None if g.rope.dims == 0 && g.rope.yarn.is_none() => false, + RopeMode::Neox + if g.rope.dims as usize == HEAD + && g.rope.yarn.is_none() + && g.rope.base.is_finite() + && g.rope.base > 0.0 => + { + true + } + _ => return Err(GpuError::shape(WHAT, "need NoPE or whole-head plain NeoX")), + }; + let window = g.window.map_or(0, |n| n as usize); + if hidden == 0 + || ctx == 0 + || u32::try_from(ctx).is_err() + || kv == 0 + || kv.checked_mul(GROUP) != Some(heads) + || g.head_dim as usize != HEAD + || g.value_dim as usize != HEAD + || !g.qk_norm + || !g.sinks + || g.o_norm + || g.out_gate != OutGate::None + || g.select.is_some() + || g.value_scale.is_some() + || !eps.is_finite() + || eps <= 0.0 + || g.window == Some(0) + { + return Err(GpuError::shape( + WHAT, + format!("unsupported head128 layer {layer}: {g:?}"), + )); + } + // The same capacity owner sizes the plan and the physical allocation. + let capacity = + runtime::stores::Rows::Ring.count(g.window.map(u64::from), ctx as u64) as usize; + let qwidth = heads + .checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "query width overflow"))?; + let kvwidth = kv + .checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "KV width overflow"))?; + Ok(Self { + layer, + hidden, + heads, + kv, + ctx, + capacity, + window, + rotate, + eps, + norm: vector(file, w, names::attn_norm(layer), hidden)?, + qnorm: vector(file, w, names::attn_q_norm(layer), HEAD)?, + knorm: vector(file, w, names::attn_k_norm(layer), HEAD)?, + sinks: vector(file, w, names::attn_sinks(layer), heads)?, + q: Projection::load(file, w, names::attn_q(layer), qwidth, hidden)?, + k: Projection::load(file, w, names::attn_k(layer), kvwidth, hidden)?, + v: Projection::load(file, w, names::attn_v(layer), kvwidth, hidden)?, + o: Projection::load(file, w, names::attn_output(layer), hidden, qwidth)?, + }) + } + + #[must_use] + pub fn capacity(&self) -> usize { + self.capacity + } + #[must_use] + pub fn window(&self) -> usize { + self.window + } + /// Maximum append-then-attend chunk. Plane rows have no overwrite hazard. + #[must_use] + pub fn chunk_rows(&self) -> usize { + if self.capacity == self.ctx { + self.ctx + } else { + self.capacity - self.window + 1 + } + } + #[must_use] + pub fn projection_types(&self) -> [SiteTy; 4] { + [self.q.ty, self.k.ty, self.v.ty, self.o.ty] + } + + /// Full attention sub-layer: RMS, Q/K/V, head RMS and optional NeoX, append, + /// sink-folded attention, O projection, residual. Every scratch cell read is written. + /// + /// # Safety + /// `pos[0..m]` are consecutive absolute positions and `n_keys[t] == pos[t] + 1`. + /// Cache rows for earlier readable positions are initialized. The runtime Store owner + /// has established retention before a rewind; a wrapped ring cannot restore overwritten keys. + #[allow( + clippy::too_many_arguments, + reason = "borrowed layer launch state (rust-quality R8)" + )] + pub unsafe fn enqueue( + &self, + gpu: &Gpu, + w: &Weights, + k: &Kernels, + s: &mut Scratch, + cache: &mut Cache, + x: &DeviceBuffer, + x1: &mut DeviceBuffer, + pos: &DeviceBuffer, + n_keys: &DeviceBuffer, + m: usize, + rope: &RopeRows, + ) -> Result<(), GpuError> { + if m == 0 + || m > s.rows + || m > self.chunk_rows() + || s.hidden != self.hidden + || s.heads != self.heads + || s.kv != self.kv + || cache.capacity != self.capacity + || cache.kv != self.kv + || x.len() < m * self.hidden + || x1.len() < m * self.hidden + || pos.len() < m + || n_keys.len() < m + || (self.rotate && (rope.width != HEAD || rope.rows() < self.ctx)) + { + return Err(GpuError::shape( + WHAT, + format!( + "layer {} incompatible scratch/cache/rows: m={m}", + self.layer + ), + )); + } + let stream = gpu.stream(); + let fault = gpu.layer_sink(self.layer)?; + let wide = WideKernels { + gemm: &k.gemm, + g32: Some(&k.g32), + }; + gpu.elem().enqueue_rms_norm( + stream, + x, + w.f32_buf(WHAT, &self.norm)?, + self.eps, + self.hidden, + m, + &mut s.xn, + )?; + k.gemm.enqueue_route_dense(stream, m, &mut s.route, fault)?; + s.input.quantize( + gpu, + &wide, + fault, + &s.xn, + m, + &[self.q.ty, self.k.ty, self.v.ty], + )?; + self.q + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.q)?; + self.k + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.k)?; + self.v + .enqueue(gpu, w, &wide, &s.input, &s.xn, &s.route, m, &mut s.v)?; + let gq = w.f32_buf(WHAT, &self.qnorm)?; + let gk = w.f32_buf(WHAT, &self.knorm)?; + // SAFETY: positions and initialized retained rows are the caller's contract; + // chunk_rows prevents wrapped appends from destroying an earlier row's window. + unsafe { + k.enqueue_append( + stream, + NeoxArgs { + q: &mut s.q, + k: &mut s.k, + v: &s.v, + gq, + gk, + table: &rope.table, + pos, + eps: self.eps, + n_head: self.heads, + n_kv: self.kv, + ctx: self.ctx, + m, + fault, + cache_k: &mut cache.k, + cache_v: &mut cache.v, + }, + self.capacity, + self.rotate, + )?; + } + k.enqueue_attention( + stream, + H128SwaArgs { + q: &s.q, + kc: &cache.k, + vc: &cache.v, + n_keys, + scale: 1.0 / (HEAD as f32).sqrt(), + n_kv: self.kv, + ctx: self.ctx, + m, + window: self.window, + sinks: Some(w.f32_buf(WHAT, &self.sinks)?), + part_v: &mut s.part_v, + part_ms: &mut s.part_ms, + fault, + y: &mut s.attended, + }, + self.heads, + self.capacity, + )?; + s.output + .quantize(gpu, &wide, fault, &s.attended, m, &[self.o.ty])?; + self.o.enqueue( + gpu, + w, + &wide, + &s.output, + &s.attended, + &s.route, + m, + &mut s.delta, + )?; + gpu.elem() + .enqueue_add(stream, x, &s.delta, m * self.hidden, x1) + } +} + +/// Only the physical cache travels with a slot. Logical retention stays in runtime::Store. +pub struct Cache { + pub k: DeviceBuffer, + pub v: DeviceBuffer, + capacity: usize, + kv: usize, +} + +impl Cache { + pub fn new(gpu: &Gpu, binding: &Binding) -> Result { + let n = binding + .kv + .checked_mul(binding.capacity) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "cache allocation overflow"))?; + Ok(Self { + k: DeviceBuffer::zeroed(gpu.stream(), n)?, + v: DeviceBuffer::zeroed(gpu.stream(), n)?, + capacity: binding.capacity, + kv: binding.kv, + }) + } + #[must_use] + pub fn bytes(&self) -> usize { + self.k.num_bytes() + self.v.num_bytes() + } +} + +/// Loaded launch modules. Their module handles allocate no device scratch. +pub struct Kernels { + gemm: GemmKernels, + g32: Gemm32Kernels, + qk: QkRmsKernels, + rope: RopeNeoxKernels, + flash: H128SwaKernels, + ring: attention_h128_ring::Kernels, +} + +impl Kernels { + pub fn load(gpu: &Gpu) -> Result { + Ok(Self { + gemm: GemmKernels::load(gpu.context())?, + g32: Gemm32Kernels::load(gpu.context())?, + qk: QkRmsKernels::load(gpu.context())?, + rope: RopeNeoxKernels::load(gpu.context())?, + flash: H128SwaKernels::load(gpu.context())?, + ring: attention_h128_ring::Kernels::load(gpu.context())?, + }) + } + #[must_use] + pub fn bytes(&self) -> usize { + 0 + } + + /// Head RMS and append, with independent rotation and physical capacity. + /// # Safety + /// Valid absolute positions are distinct modulo `rows`. + pub unsafe fn enqueue_append( + &self, + stream: &cuda_core::CudaStream, + args: NeoxArgs<'_>, + rows: usize, + rotate: bool, + ) -> Result<(), GpuError> { + if rows != args.ctx { + // SAFETY: this method's caller guarantees distinct physical rows. + return unsafe { self.ring.append(stream, args, rows, rotate) }; + } + if rotate { + // SAFETY: distinct modulo the full plane means distinct plane positions. + unsafe { self.rope.enqueue_head_norm_neox_append(stream, args) } + } else { + let NeoxArgs { + q, + k, + v, + gq, + gk, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + .. + } = args; + self.qk.enqueue( + stream, + QkRmsArgs { + q, + k, + v, + gq, + gk, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + }, + ) + } + } + + pub fn enqueue_attention( + &self, + stream: &cuda_core::CudaStream, + args: H128SwaArgs<'_>, + heads: usize, + rows: usize, + ) -> Result<(), GpuError> { + if rows == args.ctx { + self.flash.enqueue(stream, args, heads) + } else { + self.ring.attention(stream, args, heads, rows) + } + } +} + +/// Reused step/prompt storage. No allocation occurs during enqueue. +pub struct Scratch { + rows: usize, + hidden: usize, + heads: usize, + kv: usize, + xn: DeviceBuffer, + q: DeviceBuffer, + k: DeviceBuffer, + v: DeviceBuffer, + attended: DeviceBuffer, + delta: DeviceBuffer, + part_v: DeviceBuffer, + part_ms: DeviceBuffer, + input: WideActs, + output: WideActs, + route: GemmRoute, +} + +fn layout( + bindings: &[Binding], + rows: usize, +) -> Result<(usize, usize, usize, usize, WideForms, WideForms), GpuError> { + let first = bindings + .first() + .ok_or_else(|| GpuError::shape(WHAT, "at least one binding required"))?; + if !(1..=GEMM_MAX_SLOTS).contains(&rows) + || bindings + .iter() + .any(|b| b.hidden != first.hidden || b.heads != first.heads || b.kv != first.kv) + { + return Err(GpuError::shape( + WHAT, + format!("scratch needs 1..={GEMM_MAX_SLOTS} rows and equal layer widths"), + )); + } + let mut input = WideForms::NONE; + let mut output = WideForms::NONE; + let mut segs = 1; + for b in bindings { + input = input.union(WideActs::wants_of(&[b.q.ty, b.k.ty, b.v.ty])); + output = output.union(WideActs::wants(b.o.ty)); + segs = segs.max(window_segments(b.window)); + } + // The f32 source buffers are already owned above; no duplicate form is needed. + input.f32 = false; + output.f32 = false; + Ok((first.hidden, first.heads, first.kv, segs, input, output)) +} + +impl Scratch { + pub fn new(gpu: &Gpu, bindings: &[Binding], rows: usize) -> Result { + let (hidden, heads, kv, segs, input, output) = layout(bindings, rows)?; + let _ = Self::planned_bytes(bindings, rows)?; + let stream = gpu.stream(); + Ok(Self { + rows, + hidden, + heads, + kv, + xn: DeviceBuffer::zeroed(stream, rows * hidden)?, + q: DeviceBuffer::zeroed(stream, rows * heads * HEAD)?, + k: DeviceBuffer::zeroed(stream, rows * kv * HEAD)?, + v: DeviceBuffer::zeroed(stream, rows * kv * HEAD)?, + attended: DeviceBuffer::zeroed(stream, rows * heads * HEAD)?, + delta: DeviceBuffer::zeroed(stream, rows * hidden)?, + part_v: DeviceBuffer::zeroed(stream, rows * heads * segs * HEAD)?, + part_ms: DeviceBuffer::zeroed(stream, rows * heads * segs * 2)?, + input: WideActs::new(stream, input, rows, hidden)?, + output: WideActs::new(stream, output, rows, heads * HEAD)?, + route: GemmRoute::new(stream, rows, 1)?, + }) + } + /// Exact device allocation, including typed activation forms and the dense route table. + pub fn planned_bytes(bindings: &[Binding], rows: usize) -> Result { + let (hidden, heads, kv, segs, input, output) = layout(bindings, rows)?; + let shape = Head128Scratch { + hidden: hidden as u64, + query: (heads * HEAD) as u64, + key_value: (kv * HEAD) as u64, + heads: heads as u64, + segments: segs as u64, + input: Forms { + q128: input.q128, + q32: input.q32, + }, + output: Forms { + q128: output.q128, + q32: output.q32, + }, + }; + shape + .bytes(rows as u64) + .and_then(|n| usize::try_from(n).ok()) + .ok_or_else(|| GpuError::shape(WHAT, "scratch allocation overflow")) + } + #[must_use] + pub fn bytes(&self) -> usize { + [ + &self.xn, + &self.q, + &self.k, + &self.v, + &self.attended, + &self.delta, + &self.part_v, + &self.part_ms, + ] + .iter() + .map(|b| b.num_bytes()) + .sum::() + + self.input.bytes() + + self.output.bytes() + + self.route.bytes() + } +} diff --git a/crates/gpu/src/attention_h128_ring.rs b/crates/gpu/src/attention_h128_ring.rs new file mode 100644 index 000000000..bda87f776 --- /dev/null +++ b/crates/gpu/src/attention_h128_ring.rs @@ -0,0 +1,880 @@ +//! Ring-addressed head-128 primitives. Logical positions index RoPE; only KV addresses wrap. + +use crate::GpuError; +use crate::elem::rms_scale_of_mean_checked; +use crate::fault::{FaultSink, FaultSite}; +use crate::flash::f32_to_f16_bits; +use crate::flash::{MERGE_BATCH, half_bits_to_f32, online_fold}; +use crate::flash_gqa::{ + GROUP, HEAD, KEY_TILE, SEG_KEYS, SEGMENTS, seg_span, window_cut, window_segments, +}; +use crate::flash_gqa_h128_swa::{H128SwaArgs, fold_tile}; +use crate::launch_u32; +use crate::rope_neox::NeoxArgs; +use crate::rope_neox::neox_pair; +use cuda_core::{CudaContext, CudaStream, LaunchConfig1D}; +use cuda_device::{ + DisjointSlice, SharedArray, kernel, launch_bounds, launch_contract, thread, warp, +}; +use cuda_host::cuda_module; +use std::sync::Arc; + +/// u32 words of one key row, and the padded stride the key tile is staged at. +const ROW_WORDS: usize = HEAD / 2; +const K_STRIDE: usize = ROW_WORDS + 1; +/// u64 words of one value row. +const ROW_QWORDS: usize = HEAD / 4; +const ILP: usize = 4; +const THREADS: usize = GROUP * 32; +const THREADS_U32: u32 = THREADS as u32; +const MERGE_THREADS: u32 = HEAD as u32; +const _: () = assert!(THREADS_U32 as usize == THREADS && MERGE_THREADS as usize == HEAD); +const _: () = assert!(ROW_QWORDS == 32 && ROW_WORDS.is_multiple_of(ILP) && K_STRIDE % 2 == 1); +// The launch's window: 513 positions is nine 64-key segments, and a row of +// 600 live keys starts at key 87. +const _: () = { + assert!(window_segments(0) == SEGMENTS && window_segments(513) == 9); + let (first, n_w) = window_cut(600, 513); + assert!(first == 87 && n_w == 513 && first + n_w == 600); +}; + +#[inline(always)] +fn live_keys(count: u32, ctx: usize) -> usize { + let c = count as usize; + if c > ctx { 0 } else { c } +} + +#[cuda_module] +mod append_kernels { + use super::*; + const THREADS: usize = 64; + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(64)] + #[launch_contract( + domain = 1, + block = (64, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + q.len() >= m * n_head * 128, + k.len() >= m * n_kv * 128, + v.len() >= m * n_kv * 128, + gq.len() >= 128, + gk.len() >= 128, + pos.len() >= m, + cache_k.len() >= n_kv * rows * 128, + cache_v.len() >= n_kv * rows * 128 + ) + )] + pub fn qk_rms_append_ring( + gq: &[f32], + gk: &[f32], + pos: &[u32], + v: &[f32], + eps: f32, + n_head: u32, + n_kv: u32, + ctx: u32, + rows: u32, + m: u32, + fault: FaultSink, + mut q: DisjointSlice, + mut k: DisjointSlice, + mut cache_k: DisjointSlice, + mut cache_v: DisjointSlice, + ) { + static mut WSUM: SharedArray = SharedArray::UNINIT; + + let heads = (n_head + n_kv) as usize; + let b = thread::blockIdx_x() as usize; + if b >= m as usize * heads { + return; // block-uniform + } + let t = b / heads; + let h = b - t * heads; + let tid = thread::threadIdx_x() as usize; + let is_q = h < n_head as usize; + let kh = h.wrapping_sub(n_head as usize); + let base = if is_q { + (t * n_head as usize + h) * HEAD + } else { + (t * n_kv as usize + kh) * HEAD + }; + // SAFETY: t < m <= pos.len() by the launch contract. + let p = unsafe { *pos.get_unchecked(t) } as usize; + + // SAFETY: base + tid + 64 is inside the head's buffer by the launch + // contract. One thread per value pair. + let (x0, x1) = unsafe { + if is_q { + ( + *q.get_unchecked_mut(base + tid), + *q.get_unchecked_mut(base + tid + THREADS), + ) + } else { + ( + *k.get_unchecked_mut(base + tid), + *k.get_unchecked_mut(base + tid + THREADS), + ) + } + }; + let mut acc = f64::from(x0 * x0) + f64::from(x1 * x1); + acc += warp::shuffle_xor_f64(acc, 16); + acc += warp::shuffle_xor_f64(acc, 8); + acc += warp::shuffle_xor_f64(acc, 4); + acc += warp::shuffle_xor_f64(acc, 2); + acc += warp::shuffle_xor_f64(acc, 1); + // SAFETY: WSUM is this block's own shared allocation. Two slots, one + // per warp, written before the barrier that publishes them. + let ws = unsafe { SharedArray::as_raw_mut_ptr(&raw mut WSUM) }; + if warp::lane_id() == 0 { + // SAFETY: tid / 32 < 2. One lane per warp writes its slot. + unsafe { *ws.add(tid / 32) = acc }; + } + thread::sync_threads(); + // SAFETY: both slots were written before the barrier above. + let sum = unsafe { *ws.add(0) + *ws.add(1) }; + let mean = (sum / HEAD as f64) as f32; + let scale = rms_scale_of_mean_checked(mean, eps, fault, tid == 0); + + if p >= ctx as usize { + if tid == 0 { + fault.raise(FaultSite::CachePos); + } + // SAFETY: the positions read above. This thread owns them. + unsafe { + if is_q { + *q.get_unchecked_mut(base + tid) = f32::NAN; + *q.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } else { + *k.get_unchecked_mut(base + tid) = f32::NAN; + *k.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } + } + return; // block-uniform: p is the token's + } + // SAFETY: tid + 64 < 128 <= the gain's length. + let (g0, g1) = unsafe { + let g = if is_q { gq } else { gk }; + (*g.get_unchecked(tid), *g.get_unchecked(tid + THREADS)) + }; + let n0 = (scale * g0) * x0; + let n1 = (scale * g1) * x1; + + if is_q { + // SAFETY: the positions read above. This thread owns them. + unsafe { + *q.get_unchecked_mut(base + tid) = n0; + *q.get_unchecked_mut(base + tid + THREADS) = n1; + } + return; + } + // SAFETY: the positions read above. This thread owns them. + unsafe { + *k.get_unchecked_mut(base + tid) = n0; + *k.get_unchecked_mut(base + tid + THREADS) = n1; + } + let row = (kh * rows as usize + p % rows as usize) * HEAD; + // SAFETY: kh < n_kv and p % rows < rows, so the row is inside both rings. + // base + tid + 64 is inside v. One thread per pair. The tokens of + // one launch hold distinct positions, so no two blocks write one row. + unsafe { + *cache_k.get_unchecked_mut(row + tid) = f32_to_f16_bits(n0); + *cache_k.get_unchecked_mut(row + tid + THREADS) = f32_to_f16_bits(n1); + *cache_v.get_unchecked_mut(row + tid) = f32_to_f16_bits(*v.get_unchecked(base + tid)); + *cache_v.get_unchecked_mut(row + tid + THREADS) = + f32_to_f16_bits(*v.get_unchecked(base + tid + THREADS)); + } + } + + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(64)] + #[launch_contract( + domain = 1, + block = (64, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + q.len() >= m * n_head * 128, + k.len() >= m * n_kv * 128, + v.len() >= m * n_kv * 128, + gq.len() >= 128, + gk.len() >= 128, + table.len() >= ctx * 128, + pos.len() >= m, + cache_k.len() >= n_kv * rows * 128, + cache_v.len() >= n_kv * rows * 128 + ) + )] + pub fn head_norm_neox_append_ring( + gq: &[f32], + gk: &[f32], + table: &[f32], + pos: &[u32], + v: &[f32], + eps: f32, + n_head: u32, + n_kv: u32, + ctx: u32, + rows: u32, + m: u32, + fault: FaultSink, + mut q: DisjointSlice, + mut k: DisjointSlice, + mut cache_k: DisjointSlice, + mut cache_v: DisjointSlice, + ) { + static mut WSUM: SharedArray = SharedArray::UNINIT; + + let heads = (n_head + n_kv) as usize; + let b = thread::blockIdx_x() as usize; + if b >= m as usize * heads { + return; // block-uniform + } + let t = b / heads; + let h = b - t * heads; + let tid = thread::threadIdx_x() as usize; + let is_q = h < n_head as usize; + let kh = h.wrapping_sub(n_head as usize); + let base = if is_q { + (t * n_head as usize + h) * HEAD + } else { + (t * n_kv as usize + kh) * HEAD + }; + // SAFETY: t < m <= pos.len() by the launch contract. + let p = unsafe { *pos.get_unchecked(t) } as usize; + + // SAFETY: base + tid + 64 < (t·n + h + 1)·128 <= m·n·128, inside the + // head's buffer by the launch contract; one thread per value pair. + let (x0, x1) = unsafe { + if is_q { + ( + *q.get_unchecked_mut(base + tid), + *q.get_unchecked_mut(base + tid + THREADS), + ) + } else { + ( + *k.get_unchecked_mut(base + tid), + *k.get_unchecked_mut(base + tid + THREADS), + ) + } + }; + let mut acc = f64::from(x0 * x0) + f64::from(x1 * x1); + acc += warp::shuffle_xor_f64(acc, 16); + acc += warp::shuffle_xor_f64(acc, 8); + acc += warp::shuffle_xor_f64(acc, 4); + acc += warp::shuffle_xor_f64(acc, 2); + acc += warp::shuffle_xor_f64(acc, 1); + // SAFETY: WSUM is this block's own shared allocation; the raw form is + // the only way to reach it without a reference to a `static mut`. + // Two slots, one per warp, written before the barrier that publishes + // them. + let ws = unsafe { SharedArray::as_raw_mut_ptr(&raw mut WSUM) }; + if warp::lane_id() == 0 { + // SAFETY: tid / 32 < 2; one lane per warp writes its slot. + unsafe { *ws.add(tid / 32) = acc }; + } + thread::sync_threads(); + // SAFETY: both slots were written before the barrier above. + let sum = unsafe { *ws.add(0) + *ws.add(1) }; + let mean = (sum / HEAD as f64) as f32; + let scale = rms_scale_of_mean_checked(mean, eps, fault, tid == 0); + + if p >= ctx as usize { + if tid == 0 { + fault.raise(FaultSite::CachePos); + } + // SAFETY: the positions read above; this thread owns them. + unsafe { + if is_q { + *q.get_unchecked_mut(base + tid) = f32::NAN; + *q.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } else { + *k.get_unchecked_mut(base + tid) = f32::NAN; + *k.get_unchecked_mut(base + tid + THREADS) = f32::NAN; + } + } + return; // block-uniform: p is the token's + } + // SAFETY: tid + 64 < 128 <= the gain's length; p < ctx and 2·tid + 1 + // < 128, so the table pair is inside row p of the table's ctx rows. + let (g0, g1, c, s) = unsafe { + let g = if is_q { gq } else { gk }; + ( + *g.get_unchecked(tid), + *g.get_unchecked(tid + THREADS), + *table.get_unchecked(p * HEAD + 2 * tid), + *table.get_unchecked(p * HEAD + 2 * tid + 1), + ) + }; + let n0 = (scale * g0) * x0; + let n1 = (scale * g1) * x1; + let (y0, y1) = neox_pair(n0, n1, c, s); + + if is_q { + // SAFETY: the positions read above; this thread owns them. + unsafe { + *q.get_unchecked_mut(base + tid) = y0; + *q.get_unchecked_mut(base + tid + THREADS) = y1; + } + return; + } + // SAFETY: the positions read above; this thread owns them. + unsafe { + *k.get_unchecked_mut(base + tid) = y0; + *k.get_unchecked_mut(base + tid + THREADS) = y1; + } + let row = (kh * rows as usize + p % rows as usize) * HEAD; + // SAFETY: kh < n_kv and p % rows < rows, so row + 127 < n_kv·rows·128 <= the + // planes' lengths; base + tid + 64 < m·n_kv·128 <= v.len(). One + // thread per pair of the row; the tokens of one launch hold distinct + // positions (the launch contract; the engine's embedding launch + // writes consecutive ones), so no two blocks write one row. + unsafe { + *cache_k.get_unchecked_mut(row + tid) = f32_to_f16_bits(y0); + *cache_k.get_unchecked_mut(row + tid + THREADS) = f32_to_f16_bits(y1); + *cache_v.get_unchecked_mut(row + tid) = f32_to_f16_bits(*v.get_unchecked(base + tid)); + *cache_v.get_unchecked_mut(row + tid + THREADS) = + f32_to_f16_bits(*v.get_unchecked(base + tid + THREADS)); + } + } +} + +#[cuda_module] +mod flash_kernels { + use super::*; + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + rows >= 1, + rows <= ctx, + n_keys_buf.len() >= m, + segs >= 1, + q.len() >= m * n_kv * 8 * 128, + kc.len() >= n_kv * rows * 128, + vc.len() >= n_kv * rows * 128, + part_v.len() >= m * n_kv * 8 * segs * 128, + part_ms.len() >= m * n_kv * 8 * segs * 2 + ) + )] + pub fn gqa_flash_seg_h128_swa_ring( + q: &[f32], + kc: &[u16], + vc: &[u16], + n_keys_buf: &[u32], + scale: f32, + n_kv: u32, + ctx: u32, + segs: u32, + seg_keys: u32, + rows: u32, + m: u32, + window: u32, + mut part_v: DisjointSlice, + mut part_ms: DisjointSlice, + ) { + static mut QS: SharedArray = SharedArray::UNINIT; + static mut KS: SharedArray = SharedArray::UNINIT; + static mut VS: SharedArray = SharedArray::UNINIT; + static mut WS: SharedArray = SharedArray::UNINIT; + + let b = thread::blockIdx_x() as usize; + let nkv = n_kv as usize; + let n_seg = segs as usize; + let tokens = m as usize; + if b >= tokens * nkv * n_seg { + return; // block-uniform + } + let t = b % tokens; + let sk = b / tokens; + let seg = sk / nkv; + let kh = sk - seg * nkv; + let tid = thread::threadIdx_x() as usize; + let w = tid / 32; + let lane = warp::lane_id() as usize; + let n_head = nkv * GROUP; + let h = kh * GROUP + w; + let idx = (t * n_head + h) * n_seg + seg; + let ctx = ctx as usize; + // SAFETY: t < m <= n_keys_buf.len() by the launch contract. + let limit = live_keys(unsafe { *n_keys_buf.get_unchecked(t) }, ctx); + let (first, n_w) = window_cut(limit, window as usize); + let span = seg_span(n_w, n_seg, seg_keys as usize); + let lo = first + seg * span; + if lo >= first + n_w { + if lane == 0 { + // SAFETY: idx < m·n_kv·GROUP·segs, both slots inside part_ms. + unsafe { + *part_ms.get_unchecked_mut(2 * idx) = f32::NEG_INFINITY; + *part_ms.get_unchecked_mut(2 * idx + 1) = 0.0; + } + } + return; // block-uniform: the window cut is the block's + } + let hi = (lo + span).min(first + n_w); + + // SAFETY: each `static mut` is this block's own shared allocation. + // The raw form reaches it without a reference. Every index below is + // inside its array, and every write is ordered before its reads by a + // block barrier (or, for the weights, by the warp sync in `fold_tile`). + let (qs, ks, vs, ws) = unsafe { + ( + SharedArray::as_raw_mut_ptr(&raw mut QS), + SharedArray::as_raw_mut_ptr(&raw mut KS), + SharedArray::as_raw_mut_ptr(&raw mut VS), + SharedArray::as_raw_mut_ptr(&raw mut WS), + ) + }; + + let qb = (t * n_head + kh * GROUP) * HEAD; + let mut i = 0usize; + while i < 4 { + // SAFETY: qb + 4·tid + i is inside row t's group, and that row is + // inside q by the launch contract. The shared index is inside + // GROUP·HEAD. + unsafe { *qs.add(4 * tid + i) = *q.get_unchecked(qb + 4 * tid + i) }; + i += 1; + } + + let plane = kh * rows as usize * HEAD; + let k64 = kc.as_ptr() as *const u64; + let v64 = vc.as_ptr() as *const u64; + let mut mx = f32::NEG_INFINITY; + let mut s_sum = 0.0f32; + let mut acc = [0.0f32; 4]; + let mut t0 = lo; + while t0 < hi { + thread::sync_threads(); + let mut i = 0usize; + while i < 4 { + let e = tid + THREADS * i; + let key = e / ROW_QWORDS; + let word = e - key * ROW_QWORDS; + let (kw, vw) = if t0 + key < hi { + // SAFETY: t0 + key < hi <= limit <= ctx, so the row is + // mapped modulo rows inside key head kh's ring. A row is 32 u64 and the + // planes start 8-byte aligned (a device allocation). + unsafe { + let r = (plane + ((t0 + key) % rows as usize) * HEAD) / 4 + word; + (*k64.add(r), *v64.add(r)) + } + } else { + (0u64, 0u64) + }; + // SAFETY: key < 32 and word < 32, inside the staged tiles. + unsafe { + *ks.add(key * K_STRIDE + 2 * word) = kw as u32; + *ks.add(key * K_STRIDE + 2 * word + 1) = (kw >> 32) as u32; + *vs.add(key * ROW_QWORDS + word) = vw; + } + i += 1; + } + thread::sync_threads(); + + let live = t0 + lane < hi; + let mut a = [0.0f32; ILP]; + let mut wd = 0usize; + while wd < ROW_WORDS { + let mut p = 0usize; + while p < ILP { + // SAFETY: lane < 32 and wd + p < ROW_WORDS, inside the + // staged key tile. The query index is inside GROUP·HEAD. + let (kw, q0, q1) = unsafe { + ( + *ks.add(lane * K_STRIDE + wd + p), + *qs.add(w * HEAD + 2 * (wd + p)), + *qs.add(w * HEAD + 2 * (wd + p) + 1), + ) + }; + let k0 = half_bits_to_f32(kw as u16); + let k1 = half_bits_to_f32((kw >> 16) as u16); + a[p] = f32::mul_add(q0, k0, a[p]); + a[p] = f32::mul_add(q1, k1, a[p]); + p += 1; + } + wd += ILP; + } + let dot = (a[0] + a[1]) + (a[2] + a[3]); + let sc = if live { dot * scale } else { f32::NEG_INFINITY }; + + // SAFETY: the weight and value tiles are this block's, the value + // tile was published by the barrier above, and w < GROUP. + unsafe { fold_tile(sc, live, &mut mx, &mut s_sum, &mut acc, ws, vs, w, lane) }; + t0 += KEY_TILE; + } + + // SAFETY: idx is inside the partials. The four dims 4·lane .. +3 are + // this lane's alone, and lane 0 writes (max, sum). + unsafe { + let o = idx * HEAD + 4 * lane; + *part_v.get_unchecked_mut(o) = acc[0]; + *part_v.get_unchecked_mut(o + 1) = acc[1]; + *part_v.get_unchecked_mut(o + 2) = acc[2]; + *part_v.get_unchecked_mut(o + 3) = acc[3]; + if lane == 0 { + *part_ms.get_unchecked_mut(2 * idx) = mx; + *part_ms.get_unchecked_mut(2 * idx + 1) = s_sum; + } + } + } + + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + n_keys_buf.len() >= m, + segs >= 1, + sinks.len() >= n_head, + part_v.len() >= m * n_head * segs * 128, + part_ms.len() >= m * n_head * segs * 2, + y.len() >= m * n_head * 128 + ) + )] + pub fn gqa_flash_merge_h128_swa_ring( + part_v: &[f32], + part_ms: &[f32], + n_keys_buf: &[u32], + sinks: &[f32], + n_head: u32, + ctx: u32, + segs: u32, + seg_keys: u32, + m: u32, + window: u32, + fault: FaultSink, + mut y: DisjointSlice, + ) { + let b = thread::blockIdx_x() as usize; + let nh = n_head as usize; + if b >= m as usize * nh { + return; // block-uniform + } + let t = b / nh; + let h = b - t * nh; + let d = thread::threadIdx_x() as usize; + let n_seg = segs as usize; + // SAFETY: t < m <= n_keys_buf.len() by the launch contract. + let limit = live_keys(unsafe { *n_keys_buf.get_unchecked(t) }, ctx as usize); + if limit == 0 { + if d == 0 { + fault.raise(FaultSite::KeyCount); + } + // SAFETY: b·HEAD + d is inside y; thread d owns it. + unsafe { *y.get_unchecked_mut(b * HEAD + d) = f32::NAN }; + return; // block-uniform: the row is the block's + } + let (_, n_w) = window_cut(limit, window as usize); + let live = n_w.div_ceil(seg_span(n_w, n_seg, seg_keys as usize)); + let mut mx = f32::NEG_INFINITY; + let mut s = 0.0f32; + let mut acc = 0.0f32; + let mut j = 0usize; + while j < live { + let mut ms = [0.0f32; 2 * MERGE_BATCH]; + let mut vs = [0.0f32; MERGE_BATCH]; + for i in 0..MERGE_BATCH { + cuda_device::thread::__unroll_config::<0>(); + if j + i < live { + let idx = b * n_seg + j + i; + // SAFETY: idx is inside the row's segments, so both + // partials and the value dim are inside the buffers. + unsafe { + ms[2 * i] = *part_ms.get_unchecked(2 * idx); + ms[2 * i + 1] = *part_ms.get_unchecked(2 * idx + 1); + vs[i] = *part_v.get_unchecked(idx * HEAD + d); + } + } + } + for i in 0..MERGE_BATCH { + cuda_device::thread::__unroll_config::<0>(); + if j + i < live && ms[2 * i + 1] != 0.0 { + (mx, s, acc) = online_fold(mx, s, acc, ms[2 * i], ms[2 * i + 1], vs[i]); + } + } + j += MERGE_BATCH; + } + // SAFETY: h < n_head <= sinks.len() by the launch contract. + let sink = unsafe { *sinks.get_unchecked(h) }; + // The sink joins last, after every live segment. + let (_, s, acc) = online_fold(mx, s, acc, sink, 1.0, 0.0); + let out = if sink.is_nan() || sink == f32::INFINITY { + f32::NAN + } else { + acc * (1.0 / s) + }; + // SAFETY: b·HEAD + d is inside y; thread d owns it. + unsafe { *y.get_unchecked_mut(b * HEAD + d) = out }; + } +} + +/// New ring entries; legacy plane entries stay in their original modules. +pub(crate) struct Kernels { + append: append_kernels::LoadedModule, + flash: flash_kernels::LoadedModule, +} + +impl Kernels { + pub(crate) fn load(ctx: &Arc) -> Result { + // SAFETY: both embedded modules belong to this crate; every launch is checked below. + let append = unsafe { crate::shared_module!(append_kernels, ctx)? }; + // SAFETY: this crate owns this embedded module too. + let flash = unsafe { crate::shared_module!(flash_kernels, ctx)? }; + Ok(Self { append, flash }) + } + + /// # Safety + /// Valid positions are distinct modulo `rows`. + pub(crate) unsafe fn append( + &self, + stream: &CudaStream, + args: NeoxArgs<'_>, + rows: usize, + rotate: bool, + ) -> Result<(), GpuError> { + const WHAT: &str = "attention_h128::ring_append"; + let NeoxArgs { + q, + k, + v, + gq, + gk, + table, + pos, + eps, + n_head, + n_kv, + ctx, + m, + fault, + cache_k, + cache_v, + } = args; + if n_head == 0 || n_kv == 0 || ctx == 0 || rows == 0 || rows > ctx || m == 0 || m > rows { + return Err(GpuError::shape( + WHAT, + format!( + "need heads/kv >=1, 1<=rows<=ctx and 1<=m<=rows: heads={n_head} kv={n_kv} ctx={ctx} rows={rows} m={m}" + ), + )); + } + let plane = n_kv + .checked_mul(rows) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "cache shape overflow"))?; + let q_len = m + .checked_mul(n_head) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "query shape overflow"))?; + let kv_len = m + .checked_mul(n_kv) + .and_then(|n| n.checked_mul(HEAD)) + .ok_or_else(|| GpuError::shape(WHAT, "key shape overflow"))?; + for (name, got, need) in [ + ("q", q.len(), q_len), + ("k", k.len(), kv_len), + ("v", v.len(), kv_len), + ("gq", gq.len(), HEAD), + ("gk", gk.len(), HEAD), + ("pos", pos.len(), m), + ("cache_k", cache_k.len(), plane), + ("cache_v", cache_v.len(), plane), + ( + "table", + table.len(), + if rotate { + ctx.checked_mul(HEAD) + .ok_or_else(|| GpuError::shape(WHAT, "table shape overflow"))? + } else { + 0 + }, + ), + ] { + if got < need { + return Err(GpuError::shape( + WHAT, + format!("{name}.len() {got} < {need}"), + )); + } + } + let grid = launch_u32( + WHAT, + "grid", + m.checked_mul( + n_head + .checked_add(n_kv) + .ok_or_else(|| GpuError::shape(WHAT, "head count overflow"))?, + ) + .ok_or_else(|| GpuError::shape(WHAT, "grid overflow"))?, + )?; + let nh = launch_u32(WHAT, "heads", n_head)?; + let nkv = launch_u32(WHAT, "kv", n_kv)?; + let ctx = launch_u32(WHAT, "ctx", ctx)?; + let rows = launch_u32(WHAT, "rows", rows)?; + let m = launch_u32(WHAT, "m", m)?; + if rotate { + let prep = self + .append + .prepare_head_norm_neox_append_ring(LaunchConfig1D::new(grid, 64, 0))?; + self.append.head_norm_neox_append_ring( + stream, &prep, gq, gk, table, pos, v, eps, nh, nkv, ctx, rows, m, fault, q, k, + cache_k, cache_v, + )?; + } else { + let prep = self + .append + .prepare_qk_rms_append_ring(LaunchConfig1D::new(grid, 64, 0))?; + self.append.qk_rms_append_ring( + stream, &prep, gq, gk, pos, v, eps, nh, nkv, ctx, rows, m, fault, q, k, cache_k, + cache_v, + )?; + } + Ok(()) + } + + pub(crate) fn attention( + &self, + stream: &CudaStream, + args: H128SwaArgs<'_>, + n_head: usize, + rows: usize, + ) -> Result<(), GpuError> { + const WHAT: &str = "attention_h128::ring_attention"; + let H128SwaArgs { + q, + kc, + vc, + n_keys, + scale, + n_kv, + ctx, + m, + window, + sinks, + part_v, + part_ms, + fault, + y, + } = args; + if n_kv == 0 + || ctx == 0 + || rows == 0 + || rows > ctx + || m == 0 + || n_kv.checked_mul(GROUP) != Some(n_head) + || !scale.is_finite() + || window == 0 + || window > rows + || m > rows - window + 1 + { + return Err(GpuError::shape( + WHAT, + format!( + "invalid ring geometry: heads={n_head} kv={n_kv} ctx={ctx} rows={rows} m={m} window={window} scale={scale}" + ), + )); + } + let sinks = sinks.ok_or_else(|| GpuError::shape(WHAT, "per-head sinks required"))?; + let segs = window_segments(window); + let heads = m + .checked_mul(n_head) + .ok_or_else(|| GpuError::shape(WHAT, "heads overflow"))?; + let partials = heads + .checked_mul(segs) + .ok_or_else(|| GpuError::shape(WHAT, "partials overflow"))?; + for (name, got, need) in [ + ("q", q.len(), heads.checked_mul(HEAD)), + ( + "kc", + kc.len(), + n_kv.checked_mul(rows).and_then(|n| n.checked_mul(HEAD)), + ), + ( + "vc", + vc.len(), + n_kv.checked_mul(rows).and_then(|n| n.checked_mul(HEAD)), + ), + ("n_keys", n_keys.len(), Some(m)), + ("sinks", sinks.len(), Some(n_head)), + ("part_v", part_v.len(), partials.checked_mul(HEAD)), + ("part_ms", part_ms.len(), partials.checked_mul(2)), + ("y", y.len(), heads.checked_mul(HEAD)), + ] { + let need = + need.ok_or_else(|| GpuError::shape(WHAT, format!("{name} shape overflow")))?; + if got < need { + return Err(GpuError::shape( + WHAT, + format!("{name}.len() {got} < {need}"), + )); + } + } + let grid = launch_u32( + WHAT, + "grid", + m.checked_mul(n_kv) + .and_then(|n| n.checked_mul(segs)) + .ok_or_else(|| GpuError::shape(WHAT, "grid overflow"))?, + )?; + let nkv = launch_u32(WHAT, "kv", n_kv)?; + let ctx = launch_u32(WHAT, "ctx", ctx)?; + let rows = launch_u32(WHAT, "rows", rows)?; + let ns = launch_u32(WHAT, "segments", segs)?; + let sk = launch_u32(WHAT, "segment keys", SEG_KEYS)?; + let mu = launch_u32(WHAT, "m", m)?; + let window = launch_u32(WHAT, "window", window)?; + let nh = launch_u32(WHAT, "heads", n_head)?; + let prep = self + .flash + .prepare_gqa_flash_seg_h128_swa_ring(LaunchConfig1D::new(grid, THREADS_U32, 0))?; + self.flash.gqa_flash_seg_h128_swa_ring( + stream, + &prep, + q, + kc, + vc, + n_keys, + scale, + nkv, + ctx, + ns, + sk, + rows, + mu, + window, + &mut *part_v, + &mut *part_ms, + )?; + let prep = self + .flash + .prepare_gqa_flash_merge_h128_swa_ring(LaunchConfig1D::new( + launch_u32(WHAT, "merge grid", heads)?, + MERGE_THREADS, + 0, + ))?; + self.flash.gqa_flash_merge_h128_swa_ring( + stream, &prep, part_v, part_ms, n_keys, sinks, nh, ctx, ns, sk, mu, window, fault, y, + )?; + Ok(()) + } +} diff --git a/crates/gpu/src/card_experts.rs b/crates/gpu/src/card_experts.rs index 13b6365b0..a3fd228dd 100644 --- a/crates/gpu/src/card_experts.rs +++ b/crates/gpu/src/card_experts.rs @@ -45,6 +45,7 @@ use model::arch::place_hybrid::RingForms; use model::placement::accepts::{self, LegArms, Program}; use model::placement::kernels::{self, DownAct, DownForms}; +use crate::card_leg::ExpertInput; use crate::card_leg::{ self, CardLeg, LegBufs, LegRun, LegShape, LegStacks, LegSum, RouteBlock, RouteW, STACK_ROLES, }; @@ -78,6 +79,7 @@ struct CardKernels { leg: CardLeg, places: CardSlotKernels, arms: LegArms, + add_base: bool, } impl CardFamily { @@ -165,6 +167,12 @@ impl CardBatch { &self.bufs.pre } + /// The raw routed sum, before shared experts and residuals. + #[must_use] + pub fn acc(&self) -> &DeviceBuffer { + &self.bufs.acc + } + /// The buffers a tier layer's join reads and writes over the batch. pub fn join(&mut self) -> TierJoin<'_> { self.bufs.join(&self.sel) @@ -209,22 +217,6 @@ impl CardLayer { } } -/// The form the down stack of `layer`, of type `ty`, reads ([`kernels::down`]); -/// a type the table names no down entry for is refused by name (`what`). -fn down_act( - what: &'static str, - layer: &CardLayer, - ty: gguf::GgmlType, -) -> Result { - kernels::down(ty) - .map(|(_, act)| act) - .ok_or_else(|| GpuError::Tensor { - what, - name: layer.names[2].clone(), - need: "a down stack of a type the kernel table names a down entry for", - }) -} - /// The forms a load's leg buffers hold: the forms `downs` the card layers' /// downs read, or the 128-value [`DownAct::Q8Act`] on a load with no card /// layer (a tier card's join reads only the buffers' down rows). @@ -271,6 +263,7 @@ pub struct CardExperts { n: usize, /// The union of the resident layers' selected down activation forms. downs: DownForms, + expert_input: ExpertInput, /// The one-token step's buffers. step: CardBufs, /// The buffers of a pass of several rows, on a load that runs one. @@ -370,7 +363,11 @@ impl CardExperts { } let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; stacks.check(n_card, family.shape(n, ff, 1))?; - downs = downs.with(down_act(what, &layer, stacks.down_ty())?); + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + what, + "the selected card down's activation form", + ))?; + downs = downs.with(act); per.push(Some(layer)); } let downs = card_downs(downs); @@ -380,12 +377,14 @@ impl CardExperts { leg: CardLeg::load(gpu)?, places: CardSlotKernels::load(gpu.context())?, arms, + add_base: true, }, layers: per, tier: tier_k, ff, n, downs, + expert_input: ExpertInput::Direct, step: CardBufs::new(gpu.stream(), &family, (n, ff, 1), downs)?, pass: (rows > 1) .then(|| CardBufs::new(gpu.stream(), &family, (n, ff, rows), downs)) @@ -393,6 +392,21 @@ impl CardExperts { }) } + /// Select the loaded down-input policy before making prompt batches or captures. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + self.step.leg.set_expert_input(gpu, policy)?; + if let Some(pass) = &mut self.pass { + pass.leg.set_expert_input(gpu, policy)?; + } + self.expert_input = policy; + Ok(()) + } + + /// Leave only the raw routed sum when the caller owns residual association. + pub fn set_add_base(&mut self, add_base: bool) { + self.k.add_base = add_base; + } + /// Layer `l` computes experts on the card. #[must_use] pub fn has(&self, l: usize) -> bool { @@ -448,6 +462,12 @@ impl CardExperts { &self.step.pre } + /// The raw routed sum, before shared experts and residuals. + #[must_use] + pub fn acc(&self) -> &DeviceBuffer { + &self.step.acc + } + /// The buffers a tier layer's join reads and writes over the step's row, /// the slots' places `sel` as the handoff wrote them. pub fn step_join<'a>(&'a mut self, sel: &'a DeviceBuffer) -> TierJoin<'a> { @@ -463,6 +483,14 @@ impl CardExperts { }) } + /// The raw routed sum of the last pass. + pub fn pass_acc(&self) -> Result<&DeviceBuffer, GpuError> { + self.pass.as_ref().map(|p| &p.acc).ok_or(GpuError::State { + what: self.family.what, + missing: "the buffers of a pass of several rows", + }) + } + /// Device bytes of the step's buffers and a pass's. #[must_use] pub fn bytes(&self) -> usize { @@ -473,8 +501,10 @@ impl CardExperts { /// first-prompt only. pub fn batch(&self, gpu: &Gpu, cap: usize) -> Result { let stream = gpu.stream(); + let mut bufs = CardBufs::new(stream, &self.family, (self.n, self.ff, cap), self.downs)?; + bufs.leg.set_expert_input(gpu, self.expert_input)?; Ok(CardBatch { - bufs: CardBufs::new(stream, &self.family, (self.n, self.ff, cap), self.downs)?, + bufs, sel: DeviceBuffer::zeroed(stream, cap * self.family.n_used)?, ring: None, }) @@ -506,28 +536,31 @@ impl CardExperts { )); } let shape = (cb.bufs.cap(), self.family.n_used, self.n, self.ff); - let ring = RingGemm::new(gpu, shape, half_slots, RingForms::BLOCKS32)?; + let mut ring = RingGemm::new(gpu, shape, half_slots, RingForms::BLOCKS32)?; + ring.set_expert_input(gpu, self.expert_input)?; cb.ring = Some(ring); Ok(()) } - /// Device bytes of [`CardExperts::batch`]'s buffers for `cap` tokens - /// ([`CardBatch::bytes`]'s formula): the card leg's in the card layers' - /// down forms ([`kernels::leg_bytes`]), the card sum and the sum plus the - /// base, and the places. + /// The common pure descriptor of a prompt batch's leg and selected input protection. + #[must_use] + pub fn batch_layout(&self, cap: usize) -> model::placement::workstation::TierLegScratch { + model::placement::workstation::TierLegScratch { + n_embd: self.n as u64, + ff: self.ff as u64, + n_used: self.family.n_used as u64, + chunk: cap as u64, + downs: self.downs, + input_scales: self.expert_input == ExpertInput::AmaxF16, + } + } + + /// Device bytes of [`CardExperts::batch`] before an expert stream arms its ring. #[must_use] pub fn batch_bytes(&self, cap: usize) -> usize { - let shape = self.family.shape(self.n, self.ff, cap); - let leg = kernels::leg_bytes( - shape.cap as u64, - shape.slots_per_col as u64, - shape.n as u64, - shape.ff as u64, - self.downs, - ); - usize::try_from(leg).expect("the leg's bytes fit a usize") - + 2 * size_of::() * cap * self.n - + size_of::() * cap * self.family.n_used + let bytes = kernels::ffn::card_batch_bytes(self.batch_layout(cap)) + .expect("the card batch's dimensions have representable bytes"); + usize::try_from(bytes).expect("the card batch's bytes fit a usize") } /// Layer `l`'s card slots of the step (module doc, `m = 1`), the slots' @@ -783,6 +816,9 @@ impl CardExperts { self.k .leg .sum_rows(gpu, leg, &*sel, (l, io.m, n_card), sum)?; + if !self.k.add_base { + return Ok(()); + } gpu.elem() .enqueue_add(stream, acc, io.x1, io.m * self.n, pre) })(); @@ -826,7 +862,7 @@ impl CardKernels { acc: &mut bufs.acc, }); self.leg.enqueue(gpu, &mut bufs.leg, &run, sum)?; - if tiered { + if tiered || !self.add_base { return Ok(()); } let stream = gpu.stream(); @@ -941,6 +977,7 @@ mod tests { writer.finish().expect("a finished file"); let file = gguf::Split::open(&path).expect("the file"); // IQ3_S has a selected expert row arm, not a generic device format. + // Use the selected program's formats, not the generic type/role table. let model = ModelTensors { tensors, layers: triples.len(), @@ -998,6 +1035,18 @@ mod tests { /// quant: both one-form loads and mixed loads conserve bytes and execute /// step, pass, batch and tier chunks as the same common leg. Host slots /// stay untouched on the tier, including a chunk's short final run. + #[test] + fn the_leg_buffers_hold_the_card_downs_forms() { + let q8 = DownForms::of(DownAct::Q8Act); + let b32 = DownForms::of(DownAct::Q8Blocks32); + assert_eq!(card_downs(DownForms::NONE), q8); + assert_eq!(card_downs(q8), q8); + assert_eq!(card_downs(b32), b32); + assert!(!card_downs(b32).holds(DownAct::Q8Act)); + let both = q8.with(DownAct::Q8Blocks32); + assert_eq!(card_downs(both), both); + } + #[test] #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] fn hw_mixed_card_and_tier_wrappers_conserve_forms_and_rows() { @@ -1044,8 +1093,12 @@ mod tests { vec![(0..CARD as u32).collect(); triples.len()], ) .expect("the tier set"); - let mut tier = LegTier::new(&gpu, &specs, &set, &w, shape, FAMILY.program, tier_act) - .expect("the tier wrapper"); + let mut tier = LegTier::new(&gpu, &specs, &set, &w, shape, FAMILY.program, |moe| { + Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit(), + }) + }) + .expect("the tier wrapper"); assert_eq!(tier.block_bytes(), counted(shape.cap)); let host_x: Vec = (0..CAP * N).map(|i| (i % 71) as f32 / 71.0 - 0.5).collect(); let x = DeviceBuffer::from_host(stream, &host_x).expect("the rows"); @@ -1239,28 +1292,13 @@ mod tests { &w, family.shape(N, FF, 2), program, - tier_act, + |moe| Ok(Act::SwigluClamp { + limit: moe.act.swiglu_limit() + }), ), Err(GpuError::Tensor { .. }), ), "{program:?}: {triple:?}" ); } - - /// A load's leg buffers hold the forms its card layers' downs read — the - /// 32-value form for NVFP4's, both for a mix — and the 128-value form on a - /// load with no card layer. Mutant: the buffers back on the 128-value form - /// alone (an NVFP4 layer's down finds no q8 blocks of 32 and is refused at - /// its first run). - #[test] - fn the_leg_buffers_hold_the_card_downs_forms() { - let q8 = DownForms::of(DownAct::Q8Act); - let b32 = DownForms::of(DownAct::Q8Blocks32); - assert_eq!(card_downs(DownForms::NONE), q8); - assert_eq!(card_downs(q8), q8); - assert_eq!(card_downs(b32), b32); - assert!(!card_downs(b32).holds(DownAct::Q8Act)); - let both = q8.with(DownAct::Q8Blocks32); - assert_eq!(card_downs(both), both); - } } diff --git a/crates/gpu/src/card_leg.rs b/crates/gpu/src/card_leg.rs index 12a0138b4..f5655268d 100644 --- a/crates/gpu/src/card_leg.rs +++ b/crates/gpu/src/card_leg.rs @@ -69,6 +69,7 @@ //! down GEMM. use crate::card_slots::{CardAcc, CardSlotKernels, CardSlots}; +use crate::expert_input::{Protection, Slots}; use crate::fault::FaultSink; use crate::gemm::{ Gemm32Args, Gemm32Kernels, Gemm32Weight, GemmAct, GemmAct32, GemmArgs, GemmInput, GemmKernels, @@ -88,6 +89,7 @@ use crate::weights::{DevWeight, Weights}; use crate::{DeviceTensor, Gpu, GpuError, Q8Act}; use cuda_core::{CudaContext, CudaStream, DeviceBuffer}; use gguf::GgmlType; +pub use model::ops::ExpertInput; use model::placement::accepts::{self, LegArms}; use model::placement::kernels::{self, DownAct, DownEntry, DownForms, GateUpEntry, GemmEntry}; use std::sync::Arc; @@ -154,6 +156,7 @@ pub struct LegBufs { h: DeviceBuffer, act_h: ActForms, down: DeviceBuffer, + protection: Option, } impl LegBufs { @@ -202,9 +205,22 @@ impl LegBufs { h: DeviceBuffer::zeroed(stream, slots * shape.ff)?, act_h, down: DeviceBuffer::zeroed(stream, slots * shape.n)?, + protection: None, }) } + /// Select the loaded down-input policy before capture or execution. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + match policy { + ExpertInput::Direct => self.protection = None, + ExpertInput::AmaxF16 if self.protection.is_none() => { + self.protection = Some(Protection::new(gpu, self.shape.slots())?); + } + ExpertInput::AmaxF16 => {} + } + Ok(()) + } + /// The shape the buffers were made for. #[must_use] pub fn shape(&self) -> LegShape { @@ -215,7 +231,11 @@ impl LegBufs { /// their shape and forms. #[must_use] pub fn bytes(&self) -> usize { - self.act_x.device_bytes() + self.h.num_bytes() + self.act_h.bytes() + self.down.num_bytes() + self.act_x.device_bytes() + + self.h.num_bytes() + + self.act_h.bytes() + + self.down.num_bytes() + + self.protection.as_ref().map_or(0, Protection::bytes) } /// The down rows of the last run: `n` values a slot, slot-major, the @@ -405,6 +425,29 @@ impl RouteBlock<'_> { n_slots: usize, (io, rule): (RouteIo<'_>, Act), fault: FaultSink, + ) -> Result<(), GpuError> { + self.enqueue_input( + gpu, + (gemm, g32), + (route, top_k), + n_slots, + (io, rule), + None, + fault, + ) + } + + /// The routed block under its loaded expert-input policy. + #[allow(clippy::too_many_arguments, reason = "route execution contract")] + pub(crate) fn enqueue_input( + &self, + gpu: &Gpu, + (gemm, g32): (Option<&GemmKernels>, &Gemm32Kernels), + (route, top_k): (&GemmRoute, usize), + n_slots: usize, + (io, rule): (RouteIo<'_>, Act), + mut protection: Option<&mut Protection>, + fault: FaultSink, ) -> Result<(), GpuError> { const WHAT_B: &str = "card_leg::RouteBlock"; let stream = gpu.stream(); @@ -480,12 +523,34 @@ impl RouteBlock<'_> { )); } } + let (ids, counts) = route.expert_slots(); + let selected = Slots { + ids, + counts, + routed: true, + n_card: 0, + len: n_slots, + }; + if let Some(p) = protection.as_deref_mut() { + p.kernels.normalize( + gpu, + (g, u, &mut p.scales), + &selected, + self.ff, + Some(rule), + fault, + )?; + } let per_slot = GemmInput::PerSlot; match self.down { RouteW::Grouped(ty, w) => { let act = h128.ok_or_else(|| lacks("q8_1 blocks of 128 (the down)"))?; let gemm = grouped_k()?; - gemm.enqueue_swiglu_act_quant(stream, g, u, rule, n_slots, act, fault)?; + if protection.is_some() { + gpu.enqueue_quantize_gemm(g, n_slots, act, fault)?; + } else { + gemm.enqueue_swiglu_act_quant(stream, g, u, rule, n_slots, act, fault)?; + } gemm.enqueue_gemm( stream, GemmArgs { @@ -497,15 +562,21 @@ impl RouteBlock<'_> { input: per_slot, y, }, - ) + )?; } RouteW::Blocks32(w) => { let act = h32.ok_or_else(|| lacks("q8 blocks of 32 (the down)"))?; - match rule { - Act::SiluMul => { - g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)? + if protection.is_some() { + g32.enqueue_quantize_gemm32(stream, g, n_slots, act, fault)?; + } else { + match rule { + Act::SiluMul => { + g32.enqueue_swiglu_quant32(stream, g, u, n_slots, act, fault)? + } + _ => { + g32.enqueue_swiglu_act_quant32(stream, g, u, rule, n_slots, act, fault)? + } } - _ => g32.enqueue_swiglu_act_quant32(stream, g, u, rule, n_slots, act, fault)?, } g32.enqueue_gemm32( stream, @@ -517,9 +588,14 @@ impl RouteBlock<'_> { input: per_slot, y, }, - ) + )?; } } + if let Some(p) = protection { + p.kernels + .restore(gpu, y, &p.scales, &selected, self.hidden)?; + } + Ok(()) } } @@ -959,6 +1035,7 @@ struct Scratch<'a> { act_x: &'a mut Q8Act, h: &'a mut DeviceBuffer, act_h: &'a mut ActForms, + protection: &'a mut Option, } impl CardLeg { @@ -1005,6 +1082,7 @@ impl CardLeg { h, act_h, down, + protection, } = bufs; let shape = *shape; if sum.is_some() { @@ -1015,6 +1093,7 @@ impl CardLeg { act_x, h, act_h, + protection, }; self.launch( gpu, @@ -1053,6 +1132,7 @@ impl CardLeg { act_x, h, act_h, + protection, .. } = bufs; let scratch = Scratch { @@ -1060,6 +1140,7 @@ impl CardLeg { act_x, h, act_h, + protection, }; self.launch( gpu, @@ -1090,6 +1171,7 @@ impl CardLeg { act_x, h, act_h, + protection, .. } = bufs; let scratch = Scratch { @@ -1097,6 +1179,7 @@ impl CardLeg { act_x, h, act_h, + protection, }; self.launch( gpu, @@ -1161,6 +1244,7 @@ impl CardLeg { act_x, h, act_h, + protection, } = scratch; if run.m == 0 || run.m > cap { return Err(GpuError::Shape { @@ -1255,6 +1339,24 @@ impl CardLeg { } _ => return Err(st.gate.unrun()), } + let selected = Slots { + ids: run.sel, + counts: run.sel, + routed: false, + n_card: run.n_card, + len: slots, + }; + if let Some(p) = protection { + p.kernels.normalize( + gpu, + // Gate/up already activated h; None leaves the second operand unread. + (h, &*down, &mut p.scales), + &selected, + ff, + None, + fault, + )?; + } let q = QuantSel { x: h, cols: 0..slots, @@ -1369,6 +1471,9 @@ impl CardLeg { } _ => return Err(st.down.unrun()), } + if let Some(p) = protection { + p.kernels.restore(gpu, down, &p.scales, &selected, n)?; + } Ok(()) } } @@ -2069,6 +2174,104 @@ pub(crate) mod tests { } } + /// Prequantized inputs preserve the loaded protection policy and changed-input replay. + #[test] + #[ignore = "needs a CUDA device; `just gate-gpu-lib` runs it on the box"] + fn hw_prequantized_leg_preserves_protection_and_replay() { + let gpu = Gpu::new().expect("a GPU"); + let stream = gpu.stream(); + let leg = CardLeg::load(&gpu).expect("the leg"); + let ty = GgmlType::Q5_K; + let weights = weights(&gpu, SHAPE, &[ty]); + let names = [ + stack_name("gate", ty), + stack_name("up", ty), + stack_name("down", ty), + ]; + let stacks = + LegStacks::of(&weights, [&names[0], &names[1], &names[2]]).expect("the stacks"); + let mut ordinary = LegBufs::new(stream, SHAPE).expect("ordinary buffers"); + let mut quantized = LegBufs::new(stream, SHAPE).expect("quantized buffers"); + let mut host = vec![0.0f32; CAP * N]; + let mut normed = DeviceBuffer::zeroed(stream, host.len()).expect("normed rows"); + let mut act = Q8Act::with_tier_cols(stream, CAP, N).expect("quantized input"); + let sel = DeviceBuffer::from_host(stream, &SEL).expect("places"); + let sentinel = vec![SENT; CAP * SPC * N]; + let mut expected = DeviceBuffer::from_host(stream, &sentinel).expect("ordinary output"); + let mut actual = DeviceBuffer::from_host(stream, &sentinel).expect("quantized output"); + for policy in [ExpertInput::Direct, ExpertInput::AmaxF16] { + ordinary + .set_expert_input(&gpu, policy) + .expect("ordinary policy"); + quantized + .set_expert_input(&gpu, policy) + .expect("quantized policy"); + for m in [CAP, CAP - 1] { + let graph = gpu + .capture(|_| { + gpu.enqueue_quantize_q8_1_cols(&normed, &mut act, m, 0)?; + leg.enqueue_rows_quantized( + &gpu, + &mut quantized, + &QuantRun { + layer: 0, + stacks: &stacks, + act: &act, + m, + sel: &sel, + n_card: N_CARD, + rule: Act::SiluMul, + }, + &mut actual, + ) + }) + .expect("capture quantized input and protected leg"); + let amplitudes = match policy { + ExpertInput::Direct => [1.0f32, 2.0], + ExpertInput::AmaxF16 => [1.0f32, 8192.0], + }; + for amplitude in amplitudes { + for (i, value) in host.iter_mut().enumerate() { + *value = (((i * 2654435761usize) % 2001) as f32 / 1000.0 - 1.0) * amplitude; + } + normed.copy_from_host(stream, &host).expect("changed input"); + expected + .copy_from_host(stream, &sentinel) + .expect("reference canaries"); + actual + .copy_from_host(stream, &sentinel) + .expect("replay canaries"); + leg.enqueue_rows( + &gpu, + &mut ordinary, + &LegRun { + layer: 0, + stacks: &stacks, + normed: &normed, + m, + sel: &sel, + n_card: N_CARD, + rule: Act::SiluMul, + }, + &mut expected, + ) + .expect("ordinary input with the same policy"); + graph.launch(stream).expect("changed-input replay"); + let want = expected.to_host_vec(stream).expect("ordinary rows"); + let got = actual.to_host_vec(stream).expect("quantized rows"); + assert!(gpu.take_fault().expect("fault word").is_none()); + let active = m * SPC * N; + let case = format!("{policy:?}, {m} tokens, amplitude {amplitude}"); + assert_hand_rows(&case, &got[..active], &want[..active]); + assert!( + got[active..].iter().all(|&value| value == SENT), + "{case}: tail" + ); + } + } + } + } + /// A gate and up of `ty` beside a Q4_K down, a combination the qwen35moe /// program's arms admit ([`accepts::leg_arms`]): the leg's rows are the /// hand launches' bit for bit (the table's gate·up entry, the quantizer diff --git a/crates/gpu/src/expert_input.rs b/crates/gpu/src/expert_input.rs new file mode 100644 index 000000000..47f3a10f1 --- /dev/null +++ b/crates/gpu/src/expert_input.rs @@ -0,0 +1,257 @@ +//! Per-slot protection of quantized expert down inputs and restoration before routing. + +use crate::fault::{FaultSink, FaultSite}; +use crate::kquant::{Act, act::apply_checked}; +use crate::{Gpu, GpuError, launch_u32}; +use cuda_core::{DeviceBuffer, LaunchConfig1D}; +use cuda_device::{DisjointSlice, kernel, launch_bounds, launch_contract, thread, warp}; +use cuda_host::cuda_module; + +/// Slot selection: resident `_sel` places, or the route's unique slot list and refused count. +pub(crate) struct Slots<'a> { + pub ids: &'a DeviceBuffer, + pub counts: &'a DeviceBuffer, + pub routed: bool, + pub n_card: usize, + pub len: usize, +} + +#[cuda_module] +mod expert_input_kernels { + use super::*; + + #[inline(always)] + unsafe fn slot( + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + len: u32, + i: usize, + ) -> Option { + if i >= len as usize { + return None; + } + if routed != 0 { + // SAFETY: the route owns two counters; its slot list has len entries. + if i >= len as usize - unsafe { *counts.get_unchecked(1) } as usize { + return None; + } + // SAFETY: i < len <= ids.len(). Route slots are unique and below len. + Some(unsafe { *ids.get_unchecked(i) } as usize) + } else { + // SAFETY: i < len <= ids.len(). + (unsafe { *ids.get_unchecked(i) } < n_card).then_some(i) + } + } + + #[allow(clippy::too_many_arguments, reason = "flat device ABI")] + #[kernel] + #[launch_bounds(32)] + #[launch_contract(domain = 1, block = (32, 1, 1), requires = ( + ids.len() >= slots, counts.len() >= 2 * routed, h.len() >= slots * width, + scales.len() >= slots, u.len() >= activate * slots * width + ))] + pub fn normalize( + mut h: DisjointSlice, + u: &[f32], + mut scales: DisjointSlice, + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + slots: u32, + width: u32, + activate: u32, + act: u32, + limit: f32, + fault: FaultSink, + ) { + let i = thread::index_1d().get() / 32; + // SAFETY: the launch contract bounds ids and counts; route invariants bound slot ids. + let Some(s) = (unsafe { slot(ids, counts, routed, n_card, slots, i) }) else { + return; + }; + let lane = warp::lane_id() as usize; + let base = s * width as usize; + let mut max = 0.0f32; + let mut bad = false; + let mut d = lane; + while d < width as usize { + // SAFETY: s < slots, d < width; each lane owns its strided cells. + let mut v = unsafe { *h.get_unchecked_mut(base + d) }; + if activate != 0 { + // SAFETY: u holds slots whole rows when activate is selected. + v = apply_checked(act, limit, v, unsafe { *u.get_unchecked(base + d) }, fault); + // SAFETY: this lane alone owns this cell. + unsafe { *h.get_unchecked_mut(base + d) = v }; + } + bad |= !v.is_finite(); + max = max.max(v.abs()); + d += 32; + } + let max = warp::reduce_max_f32(max); + let bad = warp::ballot(bad) != 0; + // Keep the pinned graph's log/multiply/ceil/clamp/multiply/exp order. + let e = if bad { + f32::NAN + } else { + (((max * (1.0 / 1024.0)).ln() * std::f32::consts::LOG2_E) + .ceil() + .clamp(0.0, 127.0) + * std::f32::consts::LN_2) + .exp() + }; + if lane == 0 { + // SAFETY: this warp uniquely owns slot s. + unsafe { *scales.get_unchecked_mut(s) = e }; + if bad { + fault.raise(FaultSite::QuantColumn); + } + } + d = lane; + while d < width as usize { + // SAFETY: the same lane owns this cell, and s < slots. + unsafe { + let v = *h.get_unchecked_mut(base + d); + *h.get_unchecked_mut(base + d) = v / e; + } + d += 32; + } + } + + #[allow(clippy::too_many_arguments, reason = "flat device ABI")] + #[kernel] + #[launch_bounds(32)] + #[launch_contract(domain = 1, block = (32, 1, 1), requires = ( + ids.len() >= slots, counts.len() >= 2 * routed, y.len() >= slots * width, scales.len() >= slots + ))] + pub fn restore( + mut y: DisjointSlice, + scales: &[f32], + ids: &[u32], + counts: &[u32], + routed: u32, + n_card: u32, + slots: u32, + width: u32, + ) { + let i = thread::index_1d().get() / 32; + // SAFETY: launch bounds and route invariants are as in normalize. + let Some(s) = (unsafe { slot(ids, counts, routed, n_card, slots, i) }) else { + return; + }; + // SAFETY: s < slots <= scales.len(). + let e = unsafe { *scales.get_unchecked(s) }; + let mut d = warp::lane_id() as usize; + while d < width as usize { + // SAFETY: each lane owns its cells of the unique selected slot. + unsafe { + let v = *y.get_unchecked_mut(s * width as usize + d); + *y.get_unchecked_mut(s * width as usize + d) = v * e; + } + d += 32; + } + } +} + +pub(crate) struct ExpertInputKernels { + module: expert_input_kernels::LoadedModule, +} + +impl ExpertInputKernels { + pub fn load(gpu: &Gpu) -> Result { + // SAFETY: this crate owns the embedded module and enforces its launch contracts. + let module = unsafe { crate::shared_module!(expert_input_kernels, gpu.context())? }; + Ok(Self { module }) + } + + pub fn normalize( + &self, + gpu: &Gpu, + (h, u, scales): ( + &mut DeviceBuffer, + &DeviceBuffer, + &mut DeviceBuffer, + ), + slots: &Slots<'_>, + width: usize, + rule: Option, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "ExpertInput::normalize"; + let (act, limit) = rule.unwrap_or(Act::SiluMul).code(what)?; + let len = launch_u32(what, "slots", slots.len)?; + let width = launch_u32(what, "width", width)?; + let n_card = launch_u32(what, "n_card", slots.n_card)?; + let prep = self + .module + .prepare_normalize(LaunchConfig1D::new(len, 32, 0))?; + self.module.normalize( + gpu.stream(), + &prep, + h, + u, + scales, + slots.ids, + slots.counts, + u32::from(slots.routed), + n_card, + len, + width, + u32::from(rule.is_some()), + act, + limit, + fault, + )?; + Ok(()) + } + + pub fn restore( + &self, + gpu: &Gpu, + y: &mut DeviceBuffer, + scales: &DeviceBuffer, + slots: &Slots<'_>, + width: usize, + ) -> Result<(), GpuError> { + let what = "ExpertInput::restore"; + let len = launch_u32(what, "slots", slots.len)?; + let width = launch_u32(what, "width", width)?; + let n_card = launch_u32(what, "n_card", slots.n_card)?; + let prep = self + .module + .prepare_restore(LaunchConfig1D::new(len, 32, 0))?; + self.module.restore( + gpu.stream(), + &prep, + y, + scales, + slots.ids, + slots.counts, + u32::from(slots.routed), + n_card, + len, + width, + )?; + Ok(()) + } +} + +pub(crate) struct Protection { + pub kernels: ExpertInputKernels, + pub scales: DeviceBuffer, +} + +impl Protection { + pub fn new(gpu: &Gpu, slots: usize) -> Result { + Ok(Self { + kernels: ExpertInputKernels::load(gpu)?, + scales: DeviceBuffer::zeroed(gpu.stream(), slots)?, + }) + } + + pub fn bytes(&self) -> usize { + self.scales.num_bytes() + } +} diff --git a/crates/gpu/src/flash_gqa_h128_swa.rs b/crates/gpu/src/flash_gqa_h128_swa.rs index 68379d062..7d1d7c0a5 100644 --- a/crates/gpu/src/flash_gqa_h128_swa.rs +++ b/crates/gpu/src/flash_gqa_h128_swa.rs @@ -67,7 +67,7 @@ fn live_keys(count: u32, ctx: usize) -> usize { clippy::too_many_arguments, reason = "the tile step's state is the caller's registers, passed by reference" )] -unsafe fn fold_tile( +pub(crate) unsafe fn fold_tile( sc: f32, live: bool, mx: &mut f32, diff --git a/crates/gpu/src/gemm/route.rs b/crates/gpu/src/gemm/route.rs index 2abfceb70..49607235d 100644 --- a/crates/gpu/src/gemm/route.rs +++ b/crates/gpu/src/gemm/route.rs @@ -159,6 +159,11 @@ impl GemmRoute { self.n_experts } + /// The unique active-slot list and counters for per-slot expert operations. + pub(crate) fn expert_slots(&self) -> (&DeviceBuffer, &DeviceBuffer) { + (&self.cols, &self.n_tiles) + } + /// Device bytes of the table. #[must_use] pub fn bytes(&self) -> usize { diff --git a/crates/gpu/src/host/leg_tier.rs b/crates/gpu/src/host/leg_tier.rs index c35afc97d..a26de3699 100644 --- a/crates/gpu/src/host/leg_tier.rs +++ b/crates/gpu/src/host/leg_tier.rs @@ -40,7 +40,7 @@ use super::run::{HostRun, HostWidths}; use super::tier::{ TierAct, TierBlock, TierCard, TierExperts, TierInput, TierIo, TierOpen, TierSet, TierShape, }; -use crate::card_leg::{CardLeg, LegBufs, LegRun, LegShape, LegStacks, STACK_ROLES}; +use crate::card_leg::{CardLeg, ExpertInput, LegBufs, LegRun, LegShape, LegStacks, STACK_ROLES}; use crate::hybrid::{Hybrid, SlotMap}; use crate::kquant::Act; use crate::model::CardBytes; @@ -239,12 +239,11 @@ impl LegTier { }; let stacks = LegStacks::of_arms(w, layer.names(), arms, STACK_ROLES)?; stacks.check(k, shape)?; - let (_, form) = kernels::down(stacks.down_ty()).ok_or_else(|| GpuError::Tensor { - what: WHAT, - name: layer.names[2].clone(), - need: "a down stack of a type the kernel table names a down entry for", - })?; - downs = downs.with(form); + let (_, act) = kernels::down(stacks.down_ty()).ok_or(GpuError::state( + WHAT, + "the selected tier down's activation form", + ))?; + downs = downs.with(act); cfg.push(Some(layer)); } Ok(LegTier { @@ -256,8 +255,12 @@ impl LegTier { arms, }) } -} + /// Select the loaded policy shared by stage-card and tier-card experts. + pub fn set_expert_input(&mut self, gpu: &Gpu, policy: ExpertInput) -> Result<(), GpuError> { + self.bufs.set_expert_input(gpu, policy) + } +} /// Layer `layer`'s tier experts of `cfg` (the tier's layers from /// `layers.start`); a layer the tier holds none of is refused by name as /// `what`. @@ -344,7 +347,6 @@ impl TierExperts for LegTier { io: TierIo<'_>, ) -> Result<(), GpuError> { const W: &str = "LegTier::enqueue_layer"; - let arms = self.arms; let tl = layer_of(&self.layers, &self.cfg, layer, W)?; let TierInput::F32(x) = io.act else { return Err(GpuError::Shape { @@ -353,7 +355,7 @@ impl TierExperts for LegTier { .to_string(), }); }; - let stacks = LegStacks::of_arms(weights, tl.names(), arms, STACK_ROLES)?; + let stacks = LegStacks::of_arms(weights, tl.names(), self.arms, STACK_ROLES)?; let run = LegRun { layer, stacks: &stacks, @@ -502,6 +504,11 @@ pub trait LegFamily { /// The file as the family's placement reads it. type Inputs; + /// The loaded down-input policy, shared with the stage card and host. + fn expert_input(_inputs: &Self::Inputs) -> ExpertInput { + ExpertInput::Direct + } + /// `file` read as the family's placement reads it; its refusals are /// the family's. fn describe(file: &Split) -> Result; @@ -592,7 +599,7 @@ fn node_experts( let inputs = node_inputs::(file, open)?; let widths = F::widths(&inputs); let shape = widths.shape(F::CHUNK); - Ok(Box::new(LegTier::new( + let mut tier = LegTier::new( gpu, F::layers(&inputs), &open.set, @@ -600,7 +607,9 @@ fn node_experts( shape, widths.program, F::tier_act, - )?)) + )?; + tier.set_expert_input(gpu, F::expert_input(&inputs))?; + Ok(Box::new(tier)) } fn node_host(file: Arc, open: &NodeOpen) -> Result { @@ -618,8 +627,12 @@ fn node_host(file: Arc, open: &NodeOpen) -> Result(views) }, ) } diff --git a/crates/gpu/src/host/run.rs b/crates/gpu/src/host/run.rs index 405d5aade..01d434628 100644 --- a/crates/gpu/src/host/run.rs +++ b/crates/gpu/src/host/run.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use gguf::Split; use model::moe::{HostLayer, HostScratch, UNION_MAX_COLS, UnionScratch}; -use model::ops::PostNorm; +use model::ops::{ExpertInput, PostNorm}; use model::r8file::{R8Pair, R8Source}; use model::{Tensor2, Tensor2View}; @@ -174,6 +174,9 @@ impl HostRun { scratch.len() - 1 } }; + if view.expert_input() == ExpertInput::AmaxF16 { + scratch[scratch_of[l]].set_expert_input(ExpertInput::AmaxF16); + } layers[l] = Some(view); } Ok(HostRun { @@ -257,13 +260,15 @@ impl HostRun { .scratch .iter() .map(|s| { - if cols <= UNION_MAX_COLS { + let mut union = if cols <= UNION_MAX_COLS { UnionScratch::new_routed(embd, s.ff(), cols, n_used) } else { let groups = cols.div_ceil(UNION_MAX_COLS); let slots = groups * UNION_MAX_COLS * n_used; UnionScratch::new_tail(embd, s.ff(), groups, slots, n_used) - } + }?; + union.set_expert_input(s.expert_input()); + Ok::<_, model::ModelError>(union) }) .collect::>()?; Ok(()) diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index a21243a3b..04e864351 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -29,6 +29,8 @@ use cuda_host::cuda_module; use std::sync::{Arc, Mutex, PoisonError}; pub mod arch; +pub mod attention_h128; +mod attention_h128_ring; pub mod cand; pub mod capsync; pub mod card_experts; @@ -41,6 +43,7 @@ pub mod draft; pub mod draft_taps; pub mod dspark; pub mod elem; +mod expert_input; pub mod fault; pub mod flash; pub mod flash_gqa; diff --git a/crates/gpu/src/ring_gemm.rs b/crates/gpu/src/ring_gemm.rs index 008cedc73..f631a68dc 100644 --- a/crates/gpu/src/ring_gemm.rs +++ b/crates/gpu/src/ring_gemm.rs @@ -8,7 +8,9 @@ use cuda_core::DeviceBuffer; use model::arch::place_hybrid::{RingForms, ring_pass_bytes_in}; +use crate::card_leg::ExpertInput; use crate::card_leg::{RouteBlock, RouteIo}; +use crate::expert_input::Protection; use crate::gemm::{Gemm32Kernels, GemmAct, GemmAct32, GemmKernels, GemmRoute}; use crate::host::xstream::LandBatch; use crate::kquant::Act; @@ -50,6 +52,8 @@ pub(crate) struct RingGemm { h: Rows, route: GemmRoute, cap: usize, + slots: usize, + protection: Option, } impl RingGemm { @@ -72,6 +76,8 @@ impl RingGemm { h: Rows::new(gpu, slots, ff, forms.down32)?, route: GemmRoute::new(stream, slots, half_slots)?, cap, + slots, + protection: None, }; let want = ring_pass_bytes_in(cap, n_used, n, ff, half_slots, forms); if ring.bytes() != want { @@ -87,6 +93,21 @@ impl RingGemm { Ok(ring) } + pub(crate) fn set_expert_input( + &mut self, + gpu: &Gpu, + policy: ExpertInput, + ) -> Result<(), GpuError> { + match policy { + ExpertInput::Direct => self.protection = None, + ExpertInput::AmaxF16 if self.protection.is_none() => { + self.protection = Some(Protection::new(gpu, self.slots)?); + } + ExpertInput::AmaxF16 => {} + } + Ok(()) + } + /// Device bytes of the buffers. pub(crate) fn bytes(&self) -> usize { self.x.bytes() @@ -94,6 +115,7 @@ impl RingGemm { + self.u.num_bytes() + self.h.bytes() + self.route.bytes() + + self.protection.as_ref().map_or(0, Protection::bytes) } /// The slots of the half the buffers were made for. @@ -148,7 +170,13 @@ impl RingGemm { ) -> Result<(), GpuError> { let stream = gpu.stream(); let RingGemm { - x, g, u, h, route, .. + x, + g, + u, + h, + route, + protection, + .. } = self; batch.with_map(gpu.context(), n_expert, |row| { g32.enqueue_route_remap(stream, ids, row, n_slots, &mut *route, fault) @@ -164,7 +192,7 @@ impl RingGemm { Rows::Q128(a) => (Some(&mut **a), None), Rows::Blocks32(a) => (None, Some(&mut **a)), }; - blk.enqueue_rule( + blk.enqueue_input( gpu, (gemm, g32), (&*route, top_k), @@ -181,6 +209,7 @@ impl RingGemm { }, rule, ), + protection.as_mut(), fault, ) } diff --git a/crates/gpu/src/route_body.rs b/crates/gpu/src/route_body.rs index 0bf4cd545..253d668d7 100644 --- a/crates/gpu/src/route_body.rs +++ b/crates/gpu/src/route_body.rs @@ -27,7 +27,7 @@ //! ones. use crate::q8f32::f32_lane_partial_1col_w32; -use crate::route_core::{Score, renorm_divisor, router_score, take}; +use crate::route_core::{Score, floor_divisor, renorm_divisor, router_score, take}; use crate::{FaultSink, FaultSite}; use cuda_device::atomic::{AtomicOrdering, DeviceAtomicU32}; use cuda_device::{DisjointSlice, thread, threadfence, warp}; @@ -44,20 +44,20 @@ const _: () = assert!(ROUTER_THREADS_U32 as usize == ROUTER_THREADS); // The entries' `launch_bounds` and `launch_contract` block are literals. const _: () = assert!(ROUTER_THREADS == 128); -/// V4.1's `publish` over `E` experts: thread `tid` parks `score[e] = probs[at + -/// e]` (a volatile load) and `sel_v[e] = score[e] + bias[e]` for `e = tid + -/// ROUTER_THREADS · i`, raising [`FaultSite::Router`] on a selection value -/// that is not finite. +/// Publish `E` scores: thread `tid` parks `score[e] = probs[at + e]` +/// (a volatile load) and the same value plus any supplied selection bias in +/// `sel_v[e]`, for `e = tid + ROUTER_THREADS · i`. A nonfinite selection +/// value raises [`FaultSite::Router`]. /// /// # Safety /// -/// `probs` addresses at least `at + E` values, `bias` holds `E`, +/// `probs` addresses at least `at + E` values, any `bias` holds `E`, /// `score` and `sel_v` are the block's shared arrays of `E`, and the caller's /// barrier follows before any read of them. #[inline(always)] pub unsafe fn publish_n( probs: *const f32, - bias: &[f32], + bias: Option<&[f32]>, at: usize, score: *mut f32, sel_v: *mut f32, @@ -68,8 +68,13 @@ pub unsafe fn publish_n( while e < E { // SAFETY: at + e < at + E, inside probs by this fn's contract. let p = unsafe { core::ptr::read_volatile(probs.add(at + e)) }; - // SAFETY: e < E <= bias.len() by this fn's contract. - let v = p + unsafe { *bias.get_unchecked(e) }; + let v = match bias { + Some(bias) => { + // SAFETY: e < E and any supplied bias holds E entries. + p + unsafe { *bias.get_unchecked(e) } + } + None => p, + }; if !v.is_finite() { fault.raise(FaultSite::Router); } @@ -86,12 +91,12 @@ pub unsafe fn publish_n( /// V4.1's `select` over `E` experts and `U` slots: `U` rounds of the butterfly /// argmax under [`take`]`::`, lane `L` scanning experts `L + 32 j < E` /// (`j < ⌈E / 32⌉`) no earlier round took; lane 0 writes each round's id to -/// `ids[at + s]`, then sums the `U` scores in f64 in slot order, takes -/// [`renorm_divisor`] of the sum narrowed to f32, and writes each weight -/// `score / sum · scale` to `weights[at + s]`, raising [`FaultSite::Router`] -/// on one that is not finite. With `REFUSE_NAN`, a non-finite selection -/// value writes NaN weights instead: [`take`] never selects a NaN, so the -/// ids stay the finite rank order and no slot's weight is a defined pick. +/// `ids[at + s]`, then sums the `U` scores in f64 in slot order. The divisor +/// is [`floor_divisor`] with `FLOOR`, [`renorm_divisor`] otherwise, of the +/// sum narrowed to f32. Each weight is `score / divisor · scale`, raising +/// [`FaultSite::Router`] on one that is not finite. With `REFUSE_NAN`, any +/// nonfinite selection value writes NaN weights instead, so the ids are +/// not a defined pick. /// /// # Safety /// @@ -104,7 +109,12 @@ pub unsafe fn publish_n( reason = "the selecting block's shared arrays, its lane and scale, the fault and the outputs" )] #[inline(always)] -pub unsafe fn select_n( +pub unsafe fn select_n< + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( score: *const f32, sel_v: *const f32, slot_p: *mut f32, @@ -196,7 +206,11 @@ pub unsafe fn select_n( sum += f64::from(unsafe { *slot_p.add(s) }); s += 1; } - let sum = renorm_divisor(sum as f32); + let sum = if const { FLOOR } { + floor_divisor(sum as f32) + } else { + renorm_divisor(sum as f32) + }; let mut s = 0usize; while s < U { // SAFETY: s < U, this lane's own write. @@ -222,7 +236,7 @@ pub unsafe fn select_n( /// /// the grid is `E / ROWS_PER_BLOCK` blocks of [`ROUTER_THREADS`] /// (the launcher's), every thread arrives converged; `n_expert == E`, `w` -/// holds `n_expert · k` values, `x` `k`, `bias`, `logits` and `probs` +/// holds `n_expert · k` values, `x` `k`, any `bias`, `logits` and `probs` /// `n_expert`, `ids` and `weights` `U`, `done` one, zero before the launch; /// `k` is a positive multiple of 32; `last`, `score`, `sel_v` and `slot_p` /// are this block's own shared arrays of 1, `E`, `E` and `U` entries. @@ -231,10 +245,16 @@ pub unsafe fn select_n( reason = "a kernel entry's arguments and the shared arrays it declares (rust-quality R8)" )] #[inline(always)] -pub unsafe fn router_body( +pub unsafe fn router_body< + S: Score, + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( w: &[f32], x: &[f32], - bias: &[f32], + bias: Option<&[f32]>, n_expert: u32, k: u32, scale: f32, @@ -297,7 +317,9 @@ pub unsafe fn router_body(score, sel_v, slot_p, lane, scale, fault, ids, weights, 0) + select_n::( + score, sel_v, slot_p, lane, scale, fault, ids, weights, 0, + ) }; if lane == 0 { // SAFETY: `ticket` is done[0], inside `done` by this fn's @@ -316,7 +338,7 @@ pub unsafe fn router_body( +pub unsafe fn pick_body< + const E: usize, + const U: usize, + const REFUSE_NAN: bool, + const FLOOR: bool, +>( probs: &[f32], - bias: &[f32], + bias: Option<&[f32]>, n_expert: u32, t: u32, scale: f32, @@ -352,7 +379,7 @@ pub unsafe fn pick_body( // ids and weights hold U entries from U · token by this fn's // contract. unsafe { - select_n::( + select_n::( score, sel_v, slot_p, @@ -538,10 +565,10 @@ macro_rules! sigmoid_router_instance { // contract's, the launcher passes k a positive multiple of 32, // and `done[0]` is zero before the launch. unsafe { - router_body::( + router_body::( w, x, - bias, + Some(bias), n_expert, k, scale, @@ -599,9 +626,9 @@ macro_rules! sigmoid_router_instance { // reaches each `static mut` without a reference); the block is // 128 threads, converged; the lengths are the launch contract's. unsafe { - pick_body::( + pick_body::( probs, - bias, + Some(bias), n_expert, t, scale, @@ -851,7 +878,11 @@ macro_rules! sigmoid_router_instance { r: &$crate::routed_block::RouteIn<'_>, out: &mut RouterOut, ) -> Result<(), GpuError> { - self.enqueue_router(stream, r.w, x, r.bias, r.scale, out, r.fault) + let bias = r.bias.ok_or($crate::GpuError::State { + what: concat!(stringify!($m), "::route"), + missing: "the required selection bias", + })?; + self.enqueue_router(stream, r.w, x, bias, r.scale, out, r.fault) } fn route_rows( @@ -862,13 +893,17 @@ macro_rules! sigmoid_router_instance { t: usize, rows: $crate::routed_block::RoutedRows<'_>, ) -> Result<(), GpuError> { + let bias = r.bias.ok_or($crate::GpuError::State { + what: concat!(stringify!($m), "::route_rows"), + missing: "the required selection bias", + })?; self.enqueue_router_rows( stream, gpu, wide, r.w, x, - r.bias, + bias, r.scale, t, rows.probs, @@ -946,3 +981,445 @@ crate::sigmoid_router_instance! { refuse_nan: true, } } + +/// A fixed selected-sum-floor sigmoid binding over 256 experts and six +/// slots, with an actual optional selection-only bias. Shares the score, +/// publication, selection and sum order with the legacy instances above. +pub mod sigmoid_floor_256_6 { + use super::{ROUTER_THREADS_U32, ROWS_PER_BLOCK, pick_body, router_body}; + use crate::gemm::Gemm32Kernels; + use crate::route_core::Sigmoid; + use crate::routed_block::{RouteIn, RoutedRows, RouterEntry}; + use crate::{DeviceTensor, FaultSink, Gpu, GpuError, launch_u32}; + use cuda_core::{CudaContext, CudaStream, DeviceBuffer, LaunchConfig1D}; + use cuda_device::{DisjointSlice, SharedArray, kernel, launch_bounds, launch_contract}; + use cuda_host::cuda_module; + use model::arch::models::shape::{self, RouterBody, RouterInst, rules}; + use std::sync::Arc; + + pub const N_EXPERT: usize = 256; + pub const N_USED: usize = 6; + /// Both metadata bindings name the same optional-bias device entries. + pub const ROW: RouterInst = + shape::router_row_rule(RouterBody::SigmoidFloor, rules::SIGMOID_FLOOR, 256, 6); + pub const BIASED_ROW: RouterInst = shape::router_row_rule( + RouterBody::SigmoidFloor, + rules::BIASED_SIGMOID_FLOOR, + 256, + 6, + ); + const _: () = assert!(shape::router_row_is( + ROW, + rules::SIGMOID_FLOOR, + false, + (6, 6) + )); + const _: () = assert!(shape::router_row_is( + BIASED_ROW, + rules::BIASED_SIGMOID_FLOOR, + false, + (6, 6) + )); + const _: () = assert!(N_EXPERT.is_multiple_of(ROWS_PER_BLOCK)); + const _: () = assert!(N_EXPERT.div_ceil(32) <= 32); + + #[allow( + dead_code, + reason = "#[cuda_module] derives unchecked launchers that no caller uses" + )] + #[cuda_module] + mod sigmoid_floor_256_6_router_kernels { + use super::*; + + /// One token, with `has_bias` distinguishing an unused borrowed + /// input slice from a real selection bias. No zero-bias storage. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + w.len() >= 256 * k, + x.len() >= k, + bias.len() >= 256, + logits.len() >= 256, + probs.len() >= 256, + ids.len() >= 6, + weights.len() >= 6, + done.len() >= 1 + ) + )] + pub fn sigmoid_floor_256_6_router( + w: &[f32], + x: &[f32], + bias: &[f32], + has_bias: u32, + k: u32, + scale: f32, + mut logits: DisjointSlice, + mut probs: DisjointSlice, + mut ids: DisjointSlice, + mut weights: DisjointSlice, + mut done: DisjointSlice, + fault: FaultSink, + ) { + static mut SCORE: SharedArray = SharedArray::UNINIT; + static mut SEL_V: SharedArray = SharedArray::UNINIT; + static mut LAST: SharedArray = SharedArray::UNINIT; + static mut SLOT_P: SharedArray = SharedArray::UNINIT; + // SAFETY: the launch contract supplies every input/output length; + // the launcher uses k a positive multiple of 32, 64 blocks of 128 + // converged threads, and done is zero before each ordered launch. + // Shared arrays belong to this block and have the body's sizes. + unsafe { + router_body::( + w, + x, + if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, + k, + scale, + &mut logits, + &mut probs, + &mut ids, + &mut weights, + &mut done, + fault, + SharedArray::as_raw_mut_ptr(&raw mut LAST), + SharedArray::as_raw_mut_ptr(&raw mut SCORE), + SharedArray::as_raw_mut_ptr(&raw mut SEL_V), + SharedArray::as_raw_mut_ptr(&raw mut SLOT_P), + ); + } + } + + /// Each block selects one token's scores, with the same optional + /// bias and floor as the one-token entry's final block. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(128)] + #[launch_contract( + domain = 1, + block = (128, 1, 1), + requires = ( + probs.len() >= 256 * t, + bias.len() >= 256, + ids.len() >= 6 * t, + weights.len() >= 6 * t + ) + )] + pub fn sigmoid_floor_256_6_router_pick( + probs: &[f32], + bias: &[f32], + has_bias: u32, + t: u32, + scale: f32, + mut ids: DisjointSlice, + mut weights: DisjointSlice, + fault: FaultSink, + ) { + static mut SCORE: SharedArray = SharedArray::UNINIT; + static mut SEL_V: SharedArray = SharedArray::UNINIT; + static mut SLOT_P: SharedArray = SharedArray::UNINIT; + // SAFETY: launch-contract lengths and 128 converged threads per + // block; shared arrays belong to this block at the body's sizes. + unsafe { + pick_body::( + probs, + if has_bias != 0 { Some(bias) } else { None }, + N_EXPERT as u32, + t, + scale, + &mut ids, + &mut weights, + fault, + SharedArray::as_raw_mut_ptr(&raw mut SCORE), + SharedArray::as_raw_mut_ptr(&raw mut SEL_V), + SharedArray::as_raw_mut_ptr(&raw mut SLOT_P), + ); + } + } + } + + /// One token's expert logits/scores and six rank-ordered ids/weights. + /// Launches sharing this output must be ordered on one stream. + pub struct RouterOut { + pub logits: DeviceBuffer, + pub probs: DeviceBuffer, + pub ids: DeviceBuffer, + pub weights: DeviceBuffer, + done: DeviceBuffer, + } + + impl RouterOut { + /// Allocate result storage at load time, including the zero ticket. + pub fn new(stream: &CudaStream) -> Result { + Ok(Self { + logits: DeviceBuffer::zeroed(stream, N_EXPERT)?, + probs: DeviceBuffer::zeroed(stream, N_EXPERT)?, + ids: DeviceBuffer::zeroed(stream, N_USED)?, + weights: DeviceBuffer::zeroed(stream, N_USED)?, + done: DeviceBuffer::zeroed(stream, 1)?, + }) + } + + /// The ticket is zero between ordered launches, including replay. + pub fn tickets(&self, stream: &CudaStream) -> Result { + Ok(self.done.to_host_vec(stream)?[0]) + } + } + + /// Loaded floor router entries; all enqueue paths are allocation-free + /// and capturable, and borrow the optional bias rather than materialize it. + pub struct RouterKernels { + module: sigmoid_floor_256_6_router_kernels::LoadedModule, + } + + fn weight_k(what: &'static str, w: &DeviceTensor) -> Result { + let k = w.cols(); + if w.rows() != N_EXPERT || k == 0 || !k.is_multiple_of(32) { + return Err(GpuError::shape( + what, + format!( + "router weight is {} x {k}, want 256 rows of a positive multiple of 32", + w.rows() + ), + )); + } + Ok(k) + } + + fn check_bias(what: &'static str, bias: Option<&DeviceBuffer>) -> Result<(), GpuError> { + if let Some(bias) = bias + && bias.len() < N_EXPERT + { + return Err(GpuError::shape( + what, + format!( + "selection bias has {} entries, want at least 256", + bias.len() + ), + )); + } + Ok(()) + } + + impl RouterKernels { + /// Load the embedded module once, before capture. + pub fn load(ctx: &Arc) -> Result { + // SAFETY: this crate owns the module's embedded device bundle. + let module = unsafe { crate::shared_module!(sigmoid_floor_256_6_router_kernels, ctx)? }; + Ok(Self { module }) + } + + /// Six largest sigmoid scores, larger id on ties. Bias changes + /// selection only; unbiased selected scores are summed in f64 slot + /// order, narrowed to f32, divided by max(sum, 2^-14), then scaled. + /// Nonfinite logits, bias-derived keys or weights raise Router and + /// an invalid key poisons every slot weight instead of routing it. + #[allow( + clippy::too_many_arguments, + reason = "the launcher's inputs are the kernel's (rust-quality R8)" + )] + pub fn enqueue_router( + &self, + stream: &CudaStream, + w: &DeviceTensor, + x: &DeviceBuffer, + bias: Option<&DeviceBuffer>, + scale: f32, + out: &mut RouterOut, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "sigmoid_floor_256_6::enqueue_router"; + let k = weight_k(what, w)?; + check_bias(what, bias)?; + if x.len() < k { + return Err(GpuError::shape( + what, + format!("x.len() {} < k {k}", x.len()), + )); + } + let k = launch_u32(what, "k", k)?; + let prep = self + .module + .prepare_sigmoid_floor_256_6_router(LaunchConfig1D::new( + (N_EXPERT / ROWS_PER_BLOCK) as u32, + ROUTER_THREADS_U32, + 0, + ))?; + let RouterOut { + logits, + probs, + ids, + weights, + done, + } = out; + // With no bias the weight slice is only an ABI borrow: the + // kernel takes None and never loads it as a selection bias. + self.module.sigmoid_floor_256_6_router( + stream, + &prep, + w.buf(), + x, + bias.unwrap_or(w.buf()), + u32::from(bias.is_some()), + k, + scale, + logits, + probs, + ids, + weights, + done, + fault, + )?; + Ok(()) + } + + /// Token-major batch equivalent to ordered single-token launches. + /// The common F32 tile requires k a positive multiple of 64 and + /// aligned weight/input buffers. All buffers are caller-provided; + /// no optional-bias scratch is needed. + #[allow( + clippy::too_many_arguments, + reason = "router inputs, token count and the batch's output buffers (rust-quality R8)" + )] + pub fn enqueue_router_rows( + &self, + stream: &CudaStream, + gpu: &Gpu, + wide: &Gemm32Kernels, + w: &DeviceTensor, + x: &DeviceBuffer, + bias: Option<&DeviceBuffer>, + scale: f32, + t: usize, + probs: &mut DeviceBuffer, + ids: &mut DeviceBuffer, + weights: &mut DeviceBuffer, + fault: FaultSink, + ) -> Result<(), GpuError> { + let what = "sigmoid_floor_256_6::enqueue_router_rows"; + let k = weight_k(what, w)?; + check_bias(what, bias)?; + let tt = launch_u32(what, "t", t)?; + let input = k + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "input size overflow"))?; + let scores = N_EXPERT + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "score size overflow"))?; + let slots = N_USED + .checked_mul(t) + .ok_or_else(|| GpuError::shape(what, "slot size overflow"))?; + if t == 0 + || x.len() < input + || probs.len() < scores + || ids.len() < slots + || weights.len() < slots + { + return Err(GpuError::shape( + what, + format!( + "{t} tokens of {k} (at least one): x {}, probs {}, ids {}, weights {}", + x.len(), + probs.len(), + ids.len(), + weights.len() + ), + )); + } + // The tile GEMM uses the one-token row walk and warp sum tree. + wide.enqueue_f32_tile(stream, w, x, t, probs)?; + gpu.elem().enqueue_sigmoid_rows(stream, probs, scores)?; + let prep = self + .module + .prepare_sigmoid_floor_256_6_router_pick(LaunchConfig1D::new( + tt, + ROUTER_THREADS_U32, + 0, + ))?; + self.module.sigmoid_floor_256_6_router_pick( + stream, + &prep, + probs, + bias.unwrap_or(w.buf()), + u32::from(bias.is_some()), + tt, + scale, + ids, + weights, + fault, + )?; + Ok(()) + } + } + + impl RouterEntry for RouterKernels { + const N_EXPERT: usize = N_EXPERT; + const N_USED: usize = N_USED; + const OPTIONAL_BIAS: bool = true; + const NORM: model::arch::models::RouterNorm = + model::arch::models::RouterNorm::SelectedSumFloor; + type Out = RouterOut; + + fn load(ctx: &Arc) -> Result { + Self::load(ctx) + } + fn new_out(stream: &CudaStream) -> Result { + RouterOut::new(stream) + } + fn out_ids(out: &RouterOut) -> &DeviceBuffer { + &out.ids + } + fn out_weights(out: &RouterOut) -> &DeviceBuffer { + &out.weights + } + fn out_bytes(out: &RouterOut) -> usize { + out.logits.num_bytes() + + out.probs.num_bytes() + + out.ids.num_bytes() + + out.weights.num_bytes() + + out.done.num_bytes() + } + fn route( + &self, + stream: &CudaStream, + x: &DeviceBuffer, + r: &RouteIn<'_>, + out: &mut RouterOut, + ) -> Result<(), GpuError> { + self.enqueue_router(stream, r.w, x, r.bias, r.scale, out, r.fault) + } + fn route_rows( + &self, + (stream, gpu, wide): (&CudaStream, &Gpu, &Gemm32Kernels), + x: &DeviceBuffer, + r: &RouteIn<'_>, + t: usize, + rows: RoutedRows<'_>, + ) -> Result<(), GpuError> { + self.enqueue_router_rows( + stream, + gpu, + wide, + r.w, + x, + r.bias, + r.scale, + t, + rows.probs, + rows.ids, + rows.weights, + r.fault, + ) + } + } +} diff --git a/crates/gpu/src/route_core.rs b/crates/gpu/src/route_core.rs index 4c00bfe9b..b32595b86 100644 --- a/crates/gpu/src/route_core.rs +++ b/crates/gpu/src/route_core.rs @@ -18,9 +18,9 @@ //! [`Sigmoid`]). What a router stores is [`router_score`] of the logit, never //! the score alone. //! -//! A weight stage that renormalizes its selected scores divides by -//! [`renorm_divisor`] of their sum: the one guard every such stage and its -//! host simulation share. +//! A weight stage uses either [`renorm_divisor`] (the legacy epsilon-added +//! policy) or [`floor_divisor`] (Flint's fixed selected-sum floor). The +//! compiled instance chooses the policy, never a per-token launch flag. use crate::elem::argmax_take; @@ -58,6 +58,12 @@ pub fn renorm_divisor(sum: f32) -> f32 { sum + 1e-20 } +/// Flint's selected-score divisor: the f32 sum floored at exactly `2^-14`. +#[inline(always)] +pub fn floor_divisor(sum: f32) -> f32 { + sum.max(1.0 / 16384.0) +} + /// The score a router stores for `logit`: `S::score(logit)`, or NaN for a /// logit that is not finite. A logit overflows to ±infinity when a row's /// finite products sum past f32, and the scores give that a finite value diff --git a/crates/gpu/src/routed_block.rs b/crates/gpu/src/routed_block.rs index b965c0b79..146707c34 100644 --- a/crates/gpu/src/routed_block.rs +++ b/crates/gpu/src/routed_block.rs @@ -48,11 +48,11 @@ use crate::weights::Weights; use crate::{DeviceTensor, FaultSink, Gpu, GpuError}; /// What a router launch reads besides its activations: the layer's router -/// weight and selection bias, the file's `expert_weights_scale` and the -/// layer's fault sink. +/// weight and optional selection bias, the file's `expert_weights_scale` +/// and the layer's fault sink. pub struct RouteIn<'a> { pub w: &'a DeviceTensor, - pub bias: &'a DeviceBuffer, + pub bias: Option<&'a DeviceBuffer>, pub scale: f32, pub fault: FaultSink, } @@ -72,6 +72,10 @@ pub trait RouterEntry: Sized { const N_EXPERT: usize; /// Experts each token routes to. const N_USED: usize; + /// Whether an absent selection bias is a native binding, not a zero buffer. + const OPTIONAL_BIAS: bool = false; + /// The selected-score normalization implemented by this compiled binding. + const NORM: model::arch::models::RouterNorm = model::arch::models::RouterNorm::SelectedSum; /// Where one token's launch leaves its results. type Out; @@ -111,7 +115,7 @@ pub struct RouteLayer { pub norm: String, /// The router weight. pub router: String, - /// The selection bias; `None` routes with a zero bias. + /// The selection bias; `None` routes without a correction bias. pub bias: Option, /// The layer's row in the slot map's card copy, as a word offset /// (`SlotMap::row_offset`). @@ -141,7 +145,7 @@ pub struct RouteBlock { handoff: HandoffKernels, /// The wide F32 tile GEMM the rows router's scores run on. g32: Gemm32Kernels, - /// The zero selection bias of a layer the file gives none. + /// Legacy bindings' zero selection bias for a layer the file gives none. no_bias: Option>, } @@ -270,7 +274,7 @@ impl RouteBlock { Some(name) => { w.f32_buf(what, name)?; } - None => zero_bias = true, + None => zero_bias |= !R::OPTIONAL_BIAS, } } let ctx = gpu.context(); @@ -335,7 +339,11 @@ impl RouteBlock { gpu: &Gpu, ) -> Result, GpuError> { let layer = self.layer(l)?; - let bias = self.bias_of(w, l)?; + let bias = if layer.bias.is_none() && R::OPTIONAL_BIAS { + None + } else { + Some(self.bias_of(w, l)?) + }; Ok(RouteIn { w: w.f32_tensor(self.what, &layer.router)?, bias, diff --git a/crates/model/src/arch/coverage.rs b/crates/model/src/arch/coverage.rs index 169f31a22..2f46ead56 100644 --- a/crates/model/src/arch/coverage.rs +++ b/crates/model/src/arch/coverage.rs @@ -46,6 +46,66 @@ pub struct Available { /// Everything the programs run that a [`Need`] names. A need no row covers /// is an item of [`check`]; a row removed makes its needs items. pub const AVAILABLE: &[Available] = &[ + Available { + programs: &[Program::FlintBody], + at: "gpu/src/attention_h128.rs Binding::enqueue and attention_h128_ring.rs", + runs: |n| { + matches!( + n, + Need::Gqa { + head: 128, + value: 128, + group: 8, + window: None | Some(513), + sinks: true, + scaled: false, + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu/src/attention_h128_ring.rs (head128 RMS with optional full NeoX)", + runs: |n| { + matches!( + n, + Need::QkRope { + qk_norm: true, + head: 128, + mode: RopeMode::None, + dims: 0, + } | Need::QkRope { + qk_norm: true, + head: 128, + mode: RopeMode::Neox, + dims: 128, + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu/src/route_body.rs sigmoid_floor_256_6", + runs: |n| routes(n, &[RouterBody::SigmoidFloor]), + }, + Available { + programs: &[Program::FlintBody], + at: "gpu-deepseek41/src/shared_routed.rs (ungated shared expert)", + runs: |n| { + matches!( + n, + Need::Shared { + sigmoid_gate: false, + .. + } + ) + }, + }, + Available { + programs: &[Program::FlintBody], + at: "gpu-deepseek41/src/shared_routed.rs dense-leading gated blocks", + runs: |n| matches!(n, Need::DenseFfn { ff } if ff.is_multiple_of(256)), + }, Available { programs: &[Program::Deepseek41Chain], at: "gpu-deepseek41/src/attn.rs LATENT; chain/attn.rs rope dims", @@ -1552,6 +1612,46 @@ struct RowPin { /// and per-layer input projection (the host's rows), each `what` naming the /// types its row takes or the table row it follows. const ROW_PINS: &[RowPin] = &[ + RowPin { + program: Program::FlintBody, + role: Role::Attention, + matrices: true, + names: None, + what: "head128 attention projections (common dense sites)", + row: h128_projection, + }, + RowPin { + program: Program::FlintBody, + role: Role::DenseFfn, + matrices: true, + names: None, + what: "dense-leading gated blocks", + row: gated_block_site, + }, + RowPin { + program: Program::FlintBody, + role: Role::SharedExpert, + matrices: true, + names: None, + what: "shared gated blocks", + row: gated_block_site, + }, + RowPin { + program: Program::FlintBody, + role: Role::TokenEmbedding, + matrices: true, + names: None, + what: "common token embedding", + row: |ty| embed_of_widened(ty).is_some(), + }, + RowPin { + program: Program::FlintBody, + role: Role::Head, + matrices: true, + names: None, + what: "common output head", + row: |ty| head_of_widened(ty).is_some(), + }, // PIN(2026-10-22): Qwen38 dense pins admit MXFP8/FP8_B128 and Q8_0 indexers via BY_NAME; the old MXFP8 refusal fails the new coverage test (PR FAIL-first). // PIN(2026-10-26): the qwen38 body's dense sites (the delta rule's and the selecting attention's, the shared // expert's, the PLE site's) read the common dense entries by name (`site::ByName`) in every type but Q5_0, so the @@ -1773,6 +1873,35 @@ const ROW_PINS: &[RowPin] = &[ }, ]; +fn h128_projection(ty: GgmlType) -> bool { + matches!( + ty, + GgmlType::Q3_K + | GgmlType::Q4_K + | GgmlType::Q5_K + | GgmlType::Q6_K + | GgmlType::IQ4_XS + | GgmlType::Q5_0 + | GgmlType::Q8_0 + | GgmlType::F32 + | GgmlType::MXFP8 + | GgmlType::FP8_B128 + | GgmlType::BF16 + ) +} + +fn gated_block_site(ty: GgmlType) -> bool { + matches!( + ty, + GgmlType::Q8_0 + | GgmlType::Q4_K + | GgmlType::Q5_K + | GgmlType::Q6_K + | GgmlType::IQ4_XS + | GgmlType::MXFP8 + ) +} + /// `Body35`'s projection types (`PROJ`): every attention, delta-rule and /// dense FFN matrix. const Q35_PROJ: &[GgmlType] = &[ @@ -1992,7 +2121,7 @@ pub fn check_with( // stack's row width ([`routed_unrun`], the one owner). A layer // with a stack the card experts do not read keeps all its // experts on the host. - Some(p @ (Program::Glm5nextBody | Program::MiniMaxM3Body)) => { + Some(p @ (Program::Glm5nextBody | Program::MiniMaxM3Body | Program::FlintBody)) => { if routed_unrun(t, |ty| leg_card_routed(p, ty)) { at(t.layer, Need::RoutedFormat(t.ty)); } @@ -3627,21 +3756,13 @@ mod tests { GgmlType::IQ4_NL, GgmlType::Q4_0, ]; - for (name, role, what) in [ - ("attn_q.weight", Role::Attention, "attention matrices"), - ("ssm_out.weight", Role::Attention, "attention matrices"), - ( - "indexer.q_proj.weight", - Role::Attention, - "attention matrices", - ), - ( - "indexer.k_proj.weight", - Role::Attention, - "attention matrices", - ), - ("ffn_gate_shexp.weight", Role::SharedExpert, "shared expert"), - ("ffn_up_shexp.weight", Role::SharedExpert, "shared expert"), + for (name, role) in [ + ("attn_q.weight", Role::Attention), + ("ssm_out.weight", Role::Attention), + ("indexer.q_proj.weight", Role::Attention), + ("indexer.k_proj.weight", Role::Attention), + ("ffn_gate_shexp.weight", Role::SharedExpert), + ("ffn_up_shexp.weight", Role::SharedExpert), ] { for ty in q8_1.into_iter().chain(f32_rows) { let got = items(Program::Qwen38Body, &matrix(name, role, ty)); @@ -3650,7 +3771,6 @@ mod tests { for ty in refused { let got = items(Program::Qwen38Body, &matrix(name, role, ty)); assert_eq!(got.len(), 1, "{name} {ty}: {got:?}"); - assert!(got[0].starts_with(&format!("{ty} {what}")), "{got:?}"); } } // The shared expert's down is K = 640: the q8_1 types' rows are whole @@ -3664,10 +3784,6 @@ mod tests { for ty in q8_1 { let got = items(Program::Qwen38Body, &matrix(down, Role::SharedExpert, ty)); assert_eq!(got.len(), 1, "{down} {ty}: {got:?}"); - assert!( - got[0].starts_with(&format!("{ty} shared expert down (K = 640")), - "{got:?}" - ); } for ty in refused { let got = items(Program::Qwen38Body, &matrix(down, Role::SharedExpert, ty)); diff --git a/crates/model/src/arch/flint/host.rs b/crates/model/src/arch/flint/host.rs new file mode 100644 index 000000000..714fd4eb2 --- /dev/null +++ b/crates/model/src/arch/flint/host.rs @@ -0,0 +1,83 @@ +//! Flint's validated routed block as the common host and NVMe expert tiers serve it. + +use models::{Act, Ffn, ModelSpec}; + +use crate::ModelError; +use crate::arch::names; +use crate::moe::{HostLayer, HostLayerSpec}; +use crate::ops::ExpertInput; +use crate::r8file::R8Source; + +/// Build the requested layer's routed stacks with Flint's unclamped SwiGLU and activated-value +/// scaling. Dense layers, layers outside the description and non-SwiGLU routed blocks are refused +/// before a tensor is opened. Load-time only. +pub fn layer(src: R8Source<'_>, spec: &ModelSpec, l: usize) -> Result { + let refuse = |detail: String| ModelError::Metadata { + key: format!("layer {l} routed block"), + detail, + }; + let block = spec + .layers + .get(l) + .ok_or_else(|| refuse(format!("outside the file's {} layers", spec.layers.len())))?; + let Ffn::Moe(moe) = &block.ffn else { + return Err(refuse("is dense, not a routed block".to_string())); + }; + if moe.act != (Act::SwiGlu { limit: None }) { + return Err(refuse(format!( + "activation {:?}; Flint's host experts require unclamped SwiGLU", + moe.act + ))); + } + let [gate, up, down] = names::routed_stacks(l); + let mut layer = HostLayer::build( + src, + &HostLayerSpec { + gate: &gate, + up: &up, + down: &down, + n_expert: moe.experts as usize, + embd: spec.hidden as usize, + ff: moe.expert_ff as usize, + swiglu_limit: 0.0, + }, + )?; + layer.set_expert_input(ExpertInput::AmaxF16); + Ok(layer) +} + +#[cfg(test)] +mod tests { + use super::super::tests::{keys, tensors}; + use super::*; + use crate::arch::synthetic::{V, header_shaped}; + + #[test] + fn refuses_dense_outside_and_non_swiglu_before_opening_experts() { + let path = header_shaped( + "flint-host-layer", + "flint", + &keys(), + &[("tokenizer.ggml.pre", V::Str("flint"))], + &tensors(), + ); + let split = gguf::Split::open(&path).unwrap(); + let mut spec = super::super::spec::read(&split).unwrap().spec; + let src = R8Source::rows(&split); + for (l, fragment) in [(0, "is dense"), (4, "outside the file's 4 layers")] { + let error = layer(src, &spec, l).err().expect("typed refusal"); + assert!(matches!(error, ModelError::Metadata { ref key, ref detail } + if key == &format!("layer {l} routed block") && detail.contains(fragment))); + } + let Ffn::Moe(moe) = &mut spec.layers[1].ffn else { + panic!("routed"); + }; + moe.act = Act::ReluSqr; + let error = layer(src, &spec, 1) + .err() + .expect("non-gated activation refused"); + assert!(matches!(error, ModelError::Metadata { ref detail, .. } + if detail.contains("require unclamped SwiGLU"))); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/model/src/arch/flint/mod.rs b/crates/model/src/arch/flint/mod.rs index 5297cb5f4..afee36297 100644 --- a/crates/model/src/arch/flint/mod.rs +++ b/crates/model/src/arch/flint/mod.rs @@ -1,12 +1,11 @@ -//! Flint's header description and exact tensor roles. Full and sliding attention -//! independently select NoPE or NeoX rotation; every layer has head-local q/k -//! norms and attention sinks. Routed blocks carry one ungated shared MLP whose -//! width is explicit, not a count of physical modules. -//! -//! This is a reader, not an executable program. Coverage retains `Need::Program` -//! and the missing numerical/site bindings; no fixture or serve seat is registered. +//! Flint's validated description, placement inputs and routed host adapter. Full and sliding +//! attention independently select NoPE or NeoX rotation; every layer has head-local q/k norms and +//! attention sinks. Routed blocks carry one ungated shared MLP whose width is explicit, not a +//! count of physical modules. Executable admission belongs to the common program/site registry. +pub mod host; mod hparams; +pub mod place; mod roles; pub mod spec; diff --git a/crates/model/src/arch/flint/place.rs b/crates/model/src/arch/flint/place.rs new file mode 100644 index 000000000..aba005733 --- /dev/null +++ b/crates/model/src/arch/flint/place.rs @@ -0,0 +1,477 @@ +//! Flint's validated description attached to the common hybrid placement and buffer owners. + +use gguf::Split; +use models::{Ffn, Mixer, ModelSpec}; + +use super::spec; +use crate::arch::{names, place_hybrid}; +use crate::placement::accepts::{Program, leg_down_forms}; +use crate::placement::kernels::attention::{Forms, Head128Scratch}; +use crate::placement::kernels::ffn::{PromptWalk, RoutedBatch, WideBlock, WideForms}; +use crate::placement::workstation::{TierBatchBytes, TierLegScratch}; +use crate::placement::{ModelTensors, PlacementError, kernels, workstation}; + +pub use place_hybrid::{KvLayout, PlaceError}; + +/// No second hyperparameter reader: the validated description owns Flint's facts. +pub type PlanInputs = place_hybrid::PlanInputs<()>; + +/// The common host union's prompt capacity, also the window ring's batch allowance. +pub const PROMPT_ROWS: usize = crate::moe::UNION_MAX_COLS; +/// Tokens served by one run of the common expert tier leg. +pub const TIER_CHUNK: u64 = 8; + +fn overflow() -> PlacementError { + PlacementError::Metadata { + key: "flint prompt reserve".into(), + detail: "buffer bytes overflow u64".into(), + } +} + +fn reads(model: &ModelTensors, name: &str) -> Result { + let tensor = model + .tensors + .iter() + .find(|t| t.name == name) + .ok_or_else(|| PlacementError::Tensor { + name: name.into(), + detail: "is not in the validated description".into(), + })?; + kernels::dense(tensor.ty) + .map(|site| site.reads()) + .ok_or_else(|| PlacementError::Tensor { + name: name.into(), + detail: format!("type {:?} has no dense projection site", tensor.ty), + }) +} + +/// The actual Q/K/V and output forms, widest rows and maximum partial segments of the shared +/// head128 attention scratch. NoPE changes no buffer width; windowing changes stores, not the +/// maximum flash launch scratch a full layer needs. +pub fn attention_scratch( + spec: &ModelSpec, + model: &ModelTensors, +) -> Result { + let mut scratch = Head128Scratch { + hidden: u64::from(spec.hidden), + query: 0, + key_value: 0, + heads: 0, + segments: 0, + input: Forms::default(), + output: Forms::default(), + }; + for (l, layer) in spec.layers.iter().enumerate() { + let Mixer::Gqa(g) = &layer.mixer else { + return Err(PlacementError::Metadata { + key: format!("layer {l} attention"), + detail: "Flint requires GQA".into(), + }); + }; + scratch.query = scratch + .query + .max(u64::from(g.heads) * u64::from(g.head_dim)); + scratch.key_value = scratch + .key_value + .max(u64::from(g.kv_heads) * u64::from(g.head_dim)); + scratch.heads = scratch.heads.max(u64::from(g.heads)); + scratch.segments = scratch + .segments + .max(g.window.map_or(80, |w| u64::from(w).div_ceil(64))); + for name in [names::attn_q(l), names::attn_k(l), names::attn_v(l)] { + scratch.input = scratch.input.with(reads(model, &name)?); + } + scratch.output = scratch.output.with(reads(model, &names::attn_output(l))?); + } + Ok(scratch) +} + +fn block_forms( + model: &ModelTensors, + names: [String; 3], + block: &mut WideBlock, +) -> Result<(), PlacementError> { + block.input = block + .input + .with(reads(model, &names[0])?) + .with(reads(model, &names[1])?); + block.down = block.down.with(reads(model, &names[2])?); + Ok(()) +} + +/// The common routed/shared batch with selected dense/shared forms. `card` prices the allocation +/// made when either resident card serves experts; a plan reserves that candidate before assigning +/// expert ids. `tiered` adds only the prompt join's three slot words. +pub fn routed_batch( + spec: &ModelSpec, + model: &ModelTensors, + rows: u64, + card: bool, + tiered: bool, +) -> Result { + let empty = WideBlock { + hidden: u64::from(spec.hidden), + ff: 0, + input: WideForms::default(), + down: WideForms::default(), + }; + let mut shared = empty; + let mut dense = empty; + let mut experts = 0; + let mut used = 0; + let mut expert_ff = 0; + for (l, layer) in spec.layers.iter().enumerate() { + match &layer.ffn { + Ffn::Dense { ff, .. } => { + dense.ff = dense.ff.max(u64::from(*ff)); + block_forms( + model, + [names::ffn_gate(l), names::ffn_up(l), names::ffn_down(l)], + &mut dense, + )?; + } + Ffn::Moe(moe) => { + experts = experts.max(u64::from(moe.experts)); + used = used.max(u64::from(moe.top_k)); + expert_ff = expert_ff.max(u64::from(moe.expert_ff)); + let sh = moe + .shared + .as_ref() + .ok_or_else(|| PlacementError::Metadata { + key: format!("layer {l} shared block"), + detail: "Flint requires one ungated shared MLP".into(), + })?; + shared.ff = shared.ff.max(u64::from(sh.ff)); + block_forms( + model, + [ + names::ffn_gate_shexp(l), + names::ffn_up_shexp(l), + names::ffn_down_shexp(l), + ], + &mut shared, + )?; + } + } + } + Ok(RoutedBatch { + hidden: u64::from(spec.hidden), + experts, + used, + shared, + dense: (dense.ff > 0).then_some(dense), + card: card.then_some(TierLegScratch { + n_embd: u64::from(spec.hidden), + ff: expert_ff, + n_used: used, + chunk: rows, + downs: leg_down_forms(Program::FlintBody, model), + input_scales: true, + }), + tiered, + }) +} + +/// Prompt reserve from the same descriptors the attention, FFN and walk allocate by. It includes +/// the candidate card batch, not a quantization padding estimate or a per-layer copy of scratch. +pub fn prompt_reserve_bytes( + spec: &ModelSpec, + model: &ModelTensors, + rows: u64, +) -> Result { + let attention = attention_scratch(spec, model)? + .bytes(rows) + .ok_or_else(overflow)?; + let walk = PromptWalk { + hidden: u64::from(spec.hidden), + sets: 1, + } + .bytes(rows) + .ok_or_else(overflow)?; + let ffn = routed_batch(spec, model, rows, true, false)? + .bytes(rows) + .ok_or_else(overflow)?; + attention + .checked_add(walk) + .and_then(|bytes| bytes.checked_add(ffn)) + .ok_or_else(overflow) +} + +/// Price the selected prompt group before borrowing the inputs for a plan. Reconfiguration is +/// idempotent: only unit rows multiply; attention, FFN and the host-sum upload stay shared. +pub fn configure_group(inputs: &mut PlanInputs, group: usize) -> Result<(), PlacementError> { + runtime::prompt::check_group(group, bloomery_levers::PREFILL_GROUP_MAX as usize).map_err( + |error| PlacementError::Metadata { + key: runtime::prompt::GROUP_LEVER.into(), + detail: error.to_string(), + }, + )?; + let rows = PROMPT_ROWS as u64; + let hidden = u64::from(inputs.spec.hidden); + let one = PromptWalk { hidden, sets: 1 } + .bytes(rows) + .ok_or_else(overflow)?; + let selected = PromptWalk { + hidden, + sets: runtime::prompt::group_sets(group) as u64, + } + .bytes(rows) + .ok_or_else(overflow)?; + let base = prompt_reserve_bytes(&inputs.spec, &inputs.model, rows)?; + inputs.prompt_reserve = base + .checked_sub(one) + .and_then(|bytes| bytes.checked_add(selected)) + .ok_or_else(overflow)?; + Ok(()) +} + +/// The real expert tier's staging, protected leg scratch and host exchanges for one prompt unit. +pub fn tier_batch(inputs: &PlanInputs) -> Result { + let batch = routed_batch(&inputs.spec, &inputs.model, TIER_CHUNK, true, false)?; + let leg = batch.card.ok_or_else(overflow)?; + workstation::try_tier_leg_batch_bytes_of( + leg.n_embd, + leg.ff, + leg.n_used, + (TIER_CHUNK, PROMPT_ROWS as u64), + leg.downs, + true, + ) +} + +impl PlanInputs { + /// Describe once, then require the genuine program and its numerical/site bindings. + pub fn read(split: &Split) -> Result { + Self::describe(split)?.runnable() + } + + /// The validated file, common windowed stores, actual prompt reserve and tier loader contract. + /// The common host-run loader serves NVMe through the same residency arena that placement split. + pub fn describe(split: &Split) -> Result { + let read = spec::read(split)?; + let reserve = prompt_reserve_bytes(&read.spec, &read.tensors, PROMPT_ROWS as u64)?; + let batch = routed_batch(&read.spec, &read.tensors, PROMPT_ROWS as u64, false, true)?; + let tier = + kernels::ffn::tier_places_bytes(PROMPT_ROWS as u64, batch.used).ok_or_else(overflow)?; + Ok(Self::new("Flint", (), read.tensors, read.spec)? + .with_prompt_reserve(reserve) + .with_tier_reserve(tier) + .with_nvme_tier()) + } +} + +#[cfg(test)] +mod tests { + use super::super::tests::{check, keys, tensors}; + use super::*; + use crate::arch::synthetic::{V, header_shaped}; + + #[test] + fn description_uses_real_prompt_and_tier_terms_without_admitting_unsupported_shapes() { + let path = header_shaped( + "flint-place-description", + "flint", + &keys(), + &[("tokenizer.ggml.pre", V::Str("flint"))], + &tensors(), + ); + let split = Split::open(&path).unwrap(); + let mut inputs = PlanInputs::describe(&split).unwrap(); + assert_eq!( + inputs.prompt_reserve, + attention_scratch(&inputs.spec, &inputs.model) + .unwrap() + .bytes(PROMPT_ROWS as u64) + .unwrap() + + PromptWalk { + hidden: 16, + sets: 1 + } + .bytes(PROMPT_ROWS as u64) + .unwrap() + + routed_batch(&inputs.spec, &inputs.model, PROMPT_ROWS as u64, true, false) + .unwrap() + .bytes(PROMPT_ROWS as u64) + .unwrap() + ); + assert_eq!(inputs.tier_reserve, Some(12 * PROMPT_ROWS as u64 * 2)); + let tier = tier_batch(&inputs).unwrap(); + let leg = tier.leg.unwrap(); + assert!(leg.input_scales); + assert_eq!(leg.chunk, TIER_CHUNK); + assert_eq!(tier.scratch, leg.bytes(leg.downs).unwrap()); + assert!(matches!( + PlanInputs::read(&split), + Err(PlacementError::Unimplemented(_)) + )); + let one = inputs.prompt_reserve; + configure_group(&mut inputs, 2).unwrap(); + let grouped = inputs.prompt_reserve; + assert_eq!(grouped - one, 2 * PROMPT_ROWS as u64 * (8 * 16 + 8)); + configure_group(&mut inputs, 2).unwrap(); + assert_eq!(inputs.prompt_reserve, grouped); + configure_group(&mut inputs, 1).unwrap(); + assert_eq!(inputs.prompt_reserve, one); + assert!(configure_group(&mut inputs, 0).is_err()); + assert_eq!(inputs.prompt_reserve, one); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn released_window_geometry_counts_each_resident_sequence() { + let read = check("place-window-slots", &keys(), &tensors()).unwrap(); + let mut full = read.spec.layers[0].mixer.clone(); + let Mixer::Gqa(g) = &mut full else { + panic!("GQA"); + }; + g.heads = 32; + g.kv_heads = 4; + g.head_dim = 128; + g.value_dim = 128; + g.window = None; + let mut window = full.clone(); + let Mixer::Gqa(g) = &mut window else { + panic!("GQA"); + }; + g.window = Some(513); + let mixers: Vec<_> = (0..49) + .map(|l| if l < 14 { full.clone() } else { window.clone() }) + .collect(); + let kv = KvLayout::of(&mixers).unwrap(); + for ctx in [1, 512, 513, 1023, 1024, 1025, 4096] { + let expected = 2048 * (14 * ctx + 35 * ctx.min(1024)); + assert_eq!(kv.bytes(0..49, ctx), expected); + assert_eq!(kv.rows_at(14, ctx), ctx.min(1024)); + assert_eq!(kv.rows_at(0, ctx), ctx); + for slots in [1, 3] { + assert_eq!(kv.seq_terms().bytes(ctx, slots), slots * expected); + } + } + } +} + +#[cfg(test)] +mod planning_tests { + use super::super::tests::{check, keys, tensors}; + use super::*; + use crate::arch::synthetic::V; + use crate::placement::{PlanLevers, Role}; + + #[test] + fn selected_projection_forms_are_a_union_not_blanket_padding() { + let mut read = check("place-selected-forms", &keys(), &tensors()).unwrap(); + let plain = attention_scratch(&read.spec, &read.tensors).unwrap(); + assert_eq!(plain.input, Forms::default()); + assert_eq!(plain.output, Forms::default()); + for (name, ty) in [ + (names::attn_q(0), gguf::GgmlType::Q4_K), + (names::attn_k(1), gguf::GgmlType::Q8_0), + (names::attn_output(2), gguf::GgmlType::Q6_K), + ] { + read.tensors + .tensors + .iter_mut() + .find(|t| t.name == name) + .unwrap() + .ty = ty; + } + let mixed = attention_scratch(&read.spec, &read.tensors).unwrap(); + assert_eq!( + mixed.input, + Forms { + q128: true, + q32: true + } + ); + assert_eq!( + mixed.output, + Forms { + q128: true, + q32: false + } + ); + let rows = PROMPT_ROWS as u64; + assert_eq!( + mixed.bytes(rows).unwrap() - plain.bytes(rows).unwrap(), + mixed.input.bytes(rows, mixed.hidden).unwrap() + + mixed.output.bytes(rows, mixed.query).unwrap() + ); + } + + #[test] + fn nvme_loader_opt_in_preserves_unserved_loader_refusal_and_slot_kv() { + let mut keys = keys(); + for (key, value) in &mut keys { + *value = match *key { + "block_count" => V::U32(8), + "expert_count" => V::U32(256), + "expert_used_count" => V::U32(6), + "attention.sliding_window_pattern" => { + V::Bools((0..8).map(|l| l % 2 == 1).collect()) + } + "attention.rope_pattern" => V::Bools((0..8).map(|l| l % 2 == 0).collect()), + _ => continue, + }; + } + let mut ts = tensors(); + let template: Vec<_> = ts + .iter() + .filter(|(n, _)| n.starts_with("blk.1.")) + .cloned() + .collect(); + for l in 4..8 { + ts.extend( + template + .iter() + .map(|(n, dims)| (n.replacen("blk.1.", &format!("blk.{l}."), 1), dims.clone())), + ); + } + for (name, dims) in &mut ts { + if name.contains("_exps.weight") { + dims[2] = 256; + } else if name.ends_with("ffn_gate_inp.weight") { + dims[1] = 256; + } + } + let read = check("place-paging", &keys, &ts).unwrap(); + let mut inputs = PlanInputs::new("Flint", (), read.tensors, read.spec).unwrap(); + for tensor in &mut inputs.model.tensors { + if tensor.role == Role::RoutedExperts { + tensor.dims[1] *= 4096; + tensor.file_bytes *= 4096; + } + } + inputs.prompt_reserve = + prompt_reserve_bytes(&inputs.spec, &inputs.model, PROMPT_ROWS as u64).unwrap(); + inputs.room = (u64::MAX, workstation::HostRead::Given); + let machine = workstation::plan_gate(8); + let levers = PlanLevers::default(); + let (need, held) = { + let ample = inputs.plan(&machine, 2048, &levers).unwrap(); + ( + workstation::HostNeed::of(&le, 0).bytes(), + ample.host.expert_bytes, + ) + }; + inputs.room.0 = need - held / 8; + assert!(matches!(inputs.plan(&machine, 2048, &levers), + Err(PlaceError::NoNvmeTier { paged, .. }) if paged > 0)); + let inputs = inputs.with_nvme_tier(); + for slots in [1, 3] { + let plan = inputs + .plan_with_slots( + &machine, + 2048, + &levers, + std::num::NonZeroUsize::new(slots).unwrap(), + ) + .unwrap(); + assert!(plan.host.nvme_expert_bytes > 0); + assert_eq!(plan.host.expert_bytes + plan.host.nvme_expert_bytes, held); + assert_eq!( + plan.cards.iter().map(|c| c.kv_bytes).sum::(), + slots as u64 * inputs.kv.bytes(0..8, 2048) + ); + } + } +} diff --git a/crates/model/src/arch/flint/tests.rs b/crates/model/src/arch/flint/tests.rs index ac2bca5dc..0ec55ee4c 100644 --- a/crates/model/src/arch/flint/tests.rs +++ b/crates/model/src/arch/flint/tests.rs @@ -3,7 +3,7 @@ use crate::arch::synthetic::{V, header_shaped}; use crate::placement::PlacementError; use models::{Ffn, Mixer, Need, RopeMode}; -fn keys() -> Vec<(&'static str, V)> { +pub(super) fn keys() -> Vec<(&'static str, V)> { vec![ ("block_count", V::U32(4)), ("embedding_length", V::U32(16)), @@ -36,7 +36,7 @@ fn keys() -> Vec<(&'static str, V)> { ] } -fn tensors() -> Vec<(String, Vec)> { +pub(super) fn tensors() -> Vec<(String, Vec)> { let mut out = vec![ ("token_embd.weight".into(), vec![16, 2]), ("output.weight".into(), vec![16, 2]), @@ -76,7 +76,7 @@ fn tensors() -> Vec<(String, Vec)> { out } -fn check( +pub(super) fn check( tag: &str, keys: &[(&str, V)], tensors: &[(String, Vec)], @@ -162,12 +162,6 @@ fn masks_are_independent_and_nope_still_requires_head_norm() { .count(), 4 ); - assert!(crate::arch::coverage::program_of(r.spec.arch).is_none()); - assert!( - crate::arch::coverage::check(&r.spec, &r.tensors) - .iter() - .any(|item| item.feature == Need::Program("flint").to_string()) - ); } #[test] @@ -283,10 +277,15 @@ fn normalization_policy_is_architecture_specific_and_bias_is_selection_only() { let shape = models::shape::MoeShape::of(m); assert_eq!(shape.rule.norm, expected); assert_eq!(shape.rule.bias, bias); - assert_eq!( - models::shape::select_router(shape).unwrap_err().why, - models::shape::Refusal::NoRouterRule - ); + let refusal = models::shape::select_router(shape).unwrap_err().why; + if norm { + assert!(matches!( + refusal, + models::shape::Refusal::RouterWidth { ref served } if served == &[256] + )); + } else { + assert_eq!(refusal, models::shape::Refusal::NoRouterRule); + } assert!( uncovered .iter() diff --git a/crates/model/src/arch/glm5next/place.rs b/crates/model/src/arch/glm5next/place.rs index f4baa58f6..48d5ad2b5 100644 --- a/crates/model/src/arch/glm5next/place.rs +++ b/crates/model/src/arch/glm5next/place.rs @@ -162,6 +162,7 @@ pub fn tier_batch(inputs: &PlanInputs) -> Result hp.n_used as u64, (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), crate::placement::accepts::leg_down_forms(Program::Glm5nextBody, &inputs.model), + false, ) } diff --git a/crates/model/src/arch/minimaxm3/place.rs b/crates/model/src/arch/minimaxm3/place.rs index e8528fb48..c794a365e 100644 --- a/crates/model/src/arch/minimaxm3/place.rs +++ b/crates/model/src/arch/minimaxm3/place.rs @@ -142,6 +142,7 @@ pub fn tier_batch(inputs: &PlanInputs, cols: u64) -> Result String { blk(l, "attn_norm.weight") } +/// `blk.{l}.attn_sinks.weight` — one learned sink logit per query head. +#[must_use] +pub fn attn_sinks(l: usize) -> String { + blk(l, "attn_sinks.weight") +} + /// `blk.{l}.attn_q.weight` — the query projection, every head. #[must_use] pub fn attn_q(l: usize) -> String { diff --git a/crates/model/src/arch/place_hybrid.rs b/crates/model/src/arch/place_hybrid.rs index b7076fb73..bfb5eb858 100644 --- a/crates/model/src/arch/place_hybrid.rs +++ b/crates/model/src/arch/place_hybrid.rs @@ -9,9 +9,9 @@ //! forces: the keys it reads, the layers it refuses, the engine's name in the refusals that name it. //! //! The plan refuses by name a machine that hangs an expert tier card where the family's body serves -//! none ([`PlanInputs::tier_reserve`]), a context of no position and a host room that cannot hold -//! the host experts: the bodies this plans serve no NVMe expert tier, so a plan that pages one -//! ([`PlaceError::NoNvmeTier`]) would load experts nothing serves. +//! none ([`PlanInputs::tier_reserve`]), a context of no position and, unless the loader explicitly +//! serves the common NVMe tier ([`PlanInputs::with_nvme_tier`]), a host room that pages any expert +//! ([`PlaceError::NoNvmeTier`]). //! //! A body that makes buffers beside the plan's terms on its card (the prompt batch's rows) states //! their bytes in [`PlanInputs::prompt_reserve`]: the expert rule spreads within the card's budget @@ -72,7 +72,10 @@ pub const fn wide_bytes(cols: usize, k: usize) -> usize { /// (a [`GEMM_TILE_COLS`] column tile each, and the expert's), the two counts and the all-zero ids. #[must_use] pub const fn route_bytes(cols: usize) -> usize { - 4 * (cols + 2 * (cols / GEMM_TILE_COLS + 1) + 2 + cols) + match kernels::ffn::dense_route_bytes(cols as u64) { + Some(bytes) => bytes as usize, + None => panic!("route_bytes: route table bytes overflow u64"), + } } /// The forms a ring pass's route GEMMs read their rows in, by the stacks' file types: the gate's and @@ -196,7 +199,10 @@ pub fn stream_ring_bytes( /// the copy of the slot's route weight, three four-byte words. #[must_use] pub const fn tier_places_bytes(rows: usize, n_used: usize) -> u64 { - 3 * 4 * (rows * n_used) as u64 + match kernels::ffn::tier_places_bytes(rows as u64, n_used as u64) { + Some(bytes) => bytes, + None => panic!("tier_places_bytes: slot bytes overflow u64"), + } } /// What a plan of a hybrid file is made from, read from its headers. `H` is the family's own @@ -213,9 +219,9 @@ pub struct PlanInputs { pub kv: KvLayout, /// The host's available bytes when the inputs were read, with the reading that decided /// ([`workstation::host_available_read`]): the routed experts the room cannot hold follow it to - /// the NVMe tier ([`placement::expert_nvme_tier`]), which these bodies do not serve, so such a - /// plan is refused ([`PlaceError::NoNvmeTier`]). A gate that plans an arm by a room of its own - /// sets it, read [`HostRead::Given`]. + /// the NVMe tier ([`placement::expert_nvme_tier`]). A body without that loader contract refuses + /// a plan that pages experts ([`PlaceError::NoNvmeTier`]). A gate that plans an arm by a room of + /// its own sets it, read [`HostRead::Given`]. /// /// The effective room is [`Plan::plan_at`]'s: the reading's plan room, /// capped by the machine's host room and unified pool. @@ -233,6 +239,9 @@ pub struct PlanInputs { /// keeps `bytes` more than [`PlanInputs::prompt_reserve`] on a machine that names a tier card, /// the buffers only a tiered load makes. pub tier_reserve: Option, + /// Whether the loader serves the common host-run NVMe arena. Opt-in only: a plan cannot + /// promise paging merely because the placement can split the expert stacks. + nvme_tier: bool, /// The engine the refusals that name one name. engine: &'static str, } @@ -357,6 +366,7 @@ impl PlanInputs { arena, prompt_reserve: 0, tier_reserve: None, + nvme_tier: false, engine, }) } @@ -391,6 +401,16 @@ impl PlanInputs { } } + /// `self` for a loader that serves the common host-run NVMe arena, with the same residency + /// identity and split as the plan. Existing loaders remain non-paging unless they opt in. + #[must_use] + pub fn with_nvme_tier(self) -> PlanInputs { + PlanInputs { + nvme_tier: true, + ..self + } + } + /// `self` with each delta-rule layer's state `lanes` lanes ([`KvLayout::with_lanes`]): the /// stores of a load that verifies `lanes` rows a pass. #[must_use] @@ -457,8 +477,8 @@ impl PlanInputs { /// /// The host experts are held to the room the inputs name ([`PlanInputs::room`]): a room under /// the host need runs the common NVMe tier split ([`placement::expert_nvme_tier`], which refuses - /// a room under the tier's floor by name), and a plan that comes out paging any routed expert is - /// refused ([`PlaceError::NoNvmeTier`]), for the body serves no such tier. + /// a room under the tier's floor by name). A plan paging any routed expert is refused + /// ([`PlaceError::NoNvmeTier`]) unless the loader serves it ([`PlanInputs::with_nvme_tier`]). pub fn plan_with_slots<'a>( &'a self, machine: &'a Machine, @@ -485,7 +505,7 @@ impl PlanInputs { let need = HostNeed::of(&plan, 0).bytes(); let room = plan.plan_at(&self.room); placement::expert_nvme_tier(&mut plan, room, self.arena)?; - if plan.host.nvme_expert_bytes > 0 { + if plan.host.nvme_expert_bytes > 0 && !self.nvme_tier { return Err(PlaceError::NoNvmeTier { paged: plan.host.nvme_expert_bytes, need, diff --git a/crates/model/src/arch/qwen35moe/place.rs b/crates/model/src/arch/qwen35moe/place.rs index c85d9bbb0..9b8ccbff2 100644 --- a/crates/model/src/arch/qwen35moe/place.rs +++ b/crates/model/src/arch/qwen35moe/place.rs @@ -1511,6 +1511,7 @@ pub fn leg_tier_batch(inputs: &PlanInputs) -> Result Result hp.experts.n_used as u64, (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), leg_down_forms(PROGRAM, &inputs.model), + false, ) } diff --git a/crates/model/src/arch/solar2/place.rs b/crates/model/src/arch/solar2/place.rs index 849e38fba..a594d727a 100644 --- a/crates/model/src/arch/solar2/place.rs +++ b/crates/model/src/arch/solar2/place.rs @@ -53,6 +53,7 @@ pub fn tier_batch(inputs: &PlanInputs) -> Result hp.experts.n_used as u64, (TIER_CHUNK, crate::moe::UNION_MAX_COLS as u64), leg_down_forms(Program::Solar2Body, &inputs.model), + false, ) } @@ -136,6 +137,7 @@ pub fn prompt_reserve_bytes( n_used: experts.n_used as u64, chunk: r as u64, downs, + input_scales: false, } .bytes(downs)?; ((4 * f32s + acts + wide + 2 * route_bytes(r)) as u64) diff --git a/crates/model/src/moe.rs b/crates/model/src/moe.rs index 44725a667..312edf96b 100644 --- a/crates/model/src/moe.rs +++ b/crates/model/src/moe.rs @@ -22,7 +22,7 @@ //! the weight type's and nothing else. `crate::ops::matmul_q` is that single owner. use std::cell::RefCell; -use std::sync::atomic::{AtomicBool, AtomicU8, Ordering::Relaxed}; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, Ordering::Relaxed}; use std::sync::{Mutex, PoisonError}; use std::time::Instant; @@ -30,9 +30,9 @@ use gguf::{GgmlType, Gguf, Split, TensorInfo}; use crate::ModelError; use crate::ops::{ - self, DEFER_MAX_COLS, ExpertForm, ExpertStack, GroupClaims, GroupInput, Lanes, PostNorm, - R8Stack, RowLayout, ShardTensor, StackTier, Tensor2, Tensor2View, UnionCall, UnionPlanView, - UnionSlabs, UnionStack, WarmSpan, Weight, defers, matmul_q, matmul_q_group, + self, DEFER_MAX_COLS, ExpertForm, ExpertInput, ExpertStack, GroupClaims, GroupInput, Lanes, + PostNorm, R8Stack, RowLayout, ShardTensor, StackTier, Tensor2, Tensor2View, UnionCall, + UnionPlanView, UnionSlabs, UnionStack, WarmSpan, Weight, defers, matmul_q, matmul_q_group, matmul_q_group_into_on, matmul_q_group_swiglu, matmul_q_group_swiglu_into_on, }; use crate::profile; @@ -875,6 +875,8 @@ pub struct HostScratch { pars: Vec, /// The down outputs, `{embd, 1}` each. downs: Vec, + /// Per-expert activation scales, restored before weighting the down outputs. + scales: Vec, /// The call's gate and up matrices, interleaved; empty between calls. gu_w: Vec>, /// The call's down matrices; empty between calls. @@ -910,6 +912,7 @@ impl HostScratch { gate_up: blocks(2 * n_used, ff), pars: blocks(n_used, ff), downs: blocks(n_used, embd), + scales: Vec::new(), gu_w: Vec::with_capacity(2 * n_used), down_w: Vec::with_capacity(n_used), xs: Vec::with_capacity(2 * n_used), @@ -918,6 +921,23 @@ impl HostScratch { }) } + /// Allocate protection scales at load; direct quantization needs none. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + match policy { + ExpertInput::Direct => self.scales = Vec::new(), + ExpertInput::AmaxF16 => self.scales.resize(self.n_used, 1.0), + } + } + + /// The policy this scratch has load-allocated scale storage for. + pub fn expert_input(&self) -> ExpertInput { + if self.scales.is_empty() { + ExpertInput::Direct + } else { + ExpertInput::AmaxF16 + } + } + fn fits(&self, embd: usize, ff: usize) -> bool { (self.embd, self.ff) == (embd, ff) } @@ -961,6 +981,7 @@ impl HostScratch { #[derive(Clone, Copy)] struct LegRun<'n> { form: ExpertForm, + input: ExpertInput, lanes: Lanes, post: Option<&'n PostNorm>, } @@ -971,6 +992,7 @@ impl LegRun<'_> { fn flat() -> LegRun<'static> { LegRun { form: ExpertForm::Gated { limit: 0.0 }, + input: ExpertInput::Direct, lanes: Lanes::Flat, post: None, } @@ -1042,11 +1064,18 @@ fn serve_resolved( s: &mut HostScratch, ) -> Result<(), ModelError> { let n = experts.len(); + if run.input == ExpertInput::AmaxF16 && s.scales.len() < n { + return Err(ModelError::Metadata { + key: "host expert input".to_string(), + detail: "AmaxF16 requires HostScratch::set_expert_input at load".to_string(), + }); + } let lvl = profile::level(); let HostScratch { gate_up, pars, downs, + scales, xs, srcs, .. @@ -1064,33 +1093,55 @@ fn serve_resolved( *xs = relist(xl); let gt = gt?; let mut sl: Vec> = relist(std::mem::take(srcs)); - match run.form { - ExpertForm::Gated { limit } => { - for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { - sl.push(GroupInput::SwigluClamp(gate, up, limit)); - } + let dt = if run.input == ExpertInput::AmaxF16 { + for i in 0..n { + let (gate, up) = match run.form { + ExpertForm::Gated { .. } | ExpertForm::GatedOai { .. } => { + (gate_up[2 * i].col(0), gate_up[2 * i + 1].col(0)) + } + ExpertForm::Relu2 => (&[][..], gate_up[i].col(0)), + }; + run.form.combine_into(gate, up, pars[i].col_mut(0)); + scales[i] = run.input.protect(pars[i].col_mut(0)); } - ExpertForm::GatedOai { alpha, limit } => { - for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { - sl.push(GroupInput::SwigluOai(gate, up, alpha, limit)); + sl.extend(pars[..n].iter().map(GroupInput::Ready)); + matmul_q_group_swiglu_into_on(run.lanes, "host_down", down, &sl, &mut downs[..n], &mut []) + } else { + match run.form { + ExpertForm::Gated { limit } => { + for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { + sl.push(GroupInput::SwigluClamp(gate, up, limit)); + } } - } - ExpertForm::Relu2 => { - for up in &gate_up[..n] { - sl.push(GroupInput::Relu2(up)); + ExpertForm::GatedOai { alpha, limit } => { + for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { + sl.push(GroupInput::SwigluOai(gate, up, alpha, limit)); + } + } + ExpertForm::Relu2 => { + for up in &gate_up[..n] { + sl.push(GroupInput::Relu2(up)); + } } } - } - let dt = matmul_q_group_swiglu_into_on( - run.lanes, - "host_down", - down, - &sl, - &mut downs[..n], - &mut pars[..n], - ); + matmul_q_group_swiglu_into_on( + run.lanes, + "host_down", + down, + &sl, + &mut downs[..n], + &mut pars[..n], + ) + }; *srcs = relist(sl); let dt = dt?; + if run.input == ExpertInput::AmaxF16 { + for (d, &scale) in downs[..n].iter_mut().zip(&scales[..n]) { + for v in d.col_mut(0) { + *v *= scale; + } + } + } if let ExpertForm::Relu2 = run.form { // Back to back from the top: expert `i`'s up goes to slot `2 * i + 1`, // where `HostScratch::up` reads it, and no block moves twice. @@ -1430,6 +1481,8 @@ pub struct UnionScratch { qc: Vec, /// Slot `q`'s down output at `store[q·embd..(q + 1)·embd]`. store: Vec, + /// Per-slot activation scales; claims publish them before the down reads. + scales: Vec, plan: UnionPlan, /// A narrow call's claim state per distinct expert: /// `min(max_cols, DEFER_MAX_COLS) · per_list`, every expert such a call @@ -1550,6 +1603,7 @@ impl UnionScratch { gu: vec![0.0; 2 * slots * ff], qc: vec![0; slots * c_col], store: vec![0.0; slots * embd], + scales: Vec::new(), plan: UnionPlan::with_room(cols, per_list), claims: (0..cols.min(DEFER_MAX_COLS) * per_list) .map(|_| AtomicU8::new(0)) @@ -1559,6 +1613,16 @@ impl UnionScratch { } } + /// Allocate protection scales at load; direct quantization needs none. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + self.scales = match policy { + ExpertInput::Direct => Vec::new(), + ExpertInput::AmaxF16 => (0..self.max_slots) + .map(|_| AtomicU32::new(1.0f32.to_bits())) + .collect(), + }; + } + /// The most columns a call through this scratch takes. pub fn max_cols(&self) -> usize { self.max_cols @@ -1866,6 +1930,7 @@ fn serve_batched<'w>( gu, qc, store, + scales, passes, batches, .. @@ -1886,7 +1951,8 @@ fn serve_batched<'w>( }; let whole = UnionCall::new(plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let mut xqs = whole.quantize_batched(xq, xq_up, passes); slot.clear(); slot.resize(plan.n(), 0); @@ -1927,7 +1993,8 @@ fn serve_batched<'w>( } let batch_call = UnionCall::new(sub.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; batch_call.run_batch(&mut xqs, gu, qc, &mut store[base * embd..], None, passes)?; base += sub.cols.len(); drop(pin); @@ -1982,6 +2049,7 @@ fn serve_step_split<'w>( gu, qc, store, + scales, passes, batches, claims, @@ -1998,7 +2066,8 @@ fn serve_step_split<'w>( let (order, ids) = (&*order, &*ids); let whole = UnionCall::new(plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let deferred = defers(x.ne1()); let mut xqs = if deferred { whole.x_rooms(xq, xq_up) @@ -2032,7 +2101,8 @@ fn serve_step_split<'w>( } let call = UnionCall::new(sub.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, scales)?; let claim = deferred.then(|| GroupClaims { x: g == 0, states: &mut claims[..], @@ -2069,7 +2139,8 @@ fn serve_planned( } let call = UnionCall::new(s.plan.view(), stacks, x, run.form)? .spread_over(run.lanes) - .post_normed(run.post); + .post_normed(run.post) + .expert_input(run.input, &s.scales)?; let slabs = UnionSlabs { xq: &mut s.xq, xq_up: &mut s.xq_up, @@ -2637,6 +2708,7 @@ pub struct HostLayer { /// made for it ([`HostScratch::new`]). ff: usize, form: ExpertForm, + input: ExpertInput, tier: Option, /// The norm each expert's down row takes before its weight /// ([`HostLayer::set_post_norm`]); `None`, the plain weighted sum. @@ -2798,11 +2870,23 @@ impl HostLayer { n_expert, ff, form, + input: ExpertInput::Direct, tier: None, post: None, }) } + /// Select the activated expert input policy for every host serving path. + /// This is a load-time setting; the default preserves direct quantization. + pub fn set_expert_input(&mut self, policy: ExpertInput) { + self.input = policy; + } + + /// The loaded activated-expert input policy. + pub fn expert_input(&self) -> ExpertInput { + self.input + } + /// Give the layer the norm each routed expert's down row takes before /// its router weight ([`PostNorm`]): from then on every call that serves /// the layer — the one-column leg, the union and the step union, from the @@ -3091,6 +3175,7 @@ impl HostLayer { }; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3158,6 +3243,7 @@ impl HostLayer { tier.lend(&mut stacks); let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3213,6 +3299,7 @@ impl HostLayer { )?; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; @@ -3284,6 +3371,7 @@ impl HostLayer { let mut stacks = self.stacks_of(split)?; let run = LegRun { form: self.form, + input: self.input, lanes, post: self.post.as_ref(), }; diff --git a/crates/model/src/ops.rs b/crates/model/src/ops.rs index 298985a72..7fbd55856 100644 --- a/crates/model/src/ops.rs +++ b/crates/model/src/ops.rs @@ -5116,6 +5116,52 @@ pub enum ExpertForm { Relu2, } +/// How an activated routed expert enters its down projection. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ExpertInput { + /// Quantize the activation unchanged. + #[default] + Direct, + /// Rescale each expert/token activation for F16 activation-block sums, + /// then restore the down output before applying its router weight. + AmaxF16, +} + +impl ExpertInput { + /// Divide one activated expert/token column by the oracle's scale. + /// The returned factor restores its down output. + pub(crate) fn protect(self, h: &mut [f32]) -> f32 { + match self { + ExpertInput::Direct => 1.0, + ExpertInput::AmaxF16 => { + let mut amax = 0.0f32; + for &v in h.iter() { + assert!(v.is_finite(), "expert input: non-finite activation {v}"); + amax = amax.max(v.abs()); + } + let exponent = ((amax / 1024.0).ln() * std::f32::consts::LOG2_E) + .ceil() + .clamp(0.0, 127.0); + let scale = (exponent * std::f32::consts::LN_2).exp(); + for v in h { + *v /= scale; + } + scale + } + } + } +} + +impl ExpertForm { + pub(crate) fn combine_into(self, gate: &[f32], up: &[f32], out: &mut [f32]) { + match self { + ExpertForm::Gated { limit } => swiglu_into(gate, up, Some(limit), out), + ExpertForm::GatedOai { alpha, limit } => swiglu_oai_into(gate, up, alpha, limit, out), + ExpertForm::Relu2 => relu2_into(up, out), + } + } +} + /// A layer's norm of each routed expert's down row before the row's router /// weight applies: the row scaled by `gain`, then divided by the root of its /// mean square plus `eps` (the card's `gain_rms_norm`). An architecture whose @@ -5375,6 +5421,7 @@ struct Combines<'c> { gu: &'c [f32], ff: usize, form: ExpertForm, + scales: Option<&'c [std::sync::atomic::AtomicU32]>, ty: GgmlType, qc: SharedQuantCols, } @@ -5413,6 +5460,10 @@ impl Combines<'_> { relu2_into(up, par); } } + if let Some(scales) = self.scales { + let scale = ExpertInput::AmaxF16.protect(par); + scales[q].store(scale.to_bits(), std::sync::atomic::Ordering::Relaxed); + } // SAFETY: the caller owns slot `q` of `qc`. unsafe { self.qc.quantize_into(self.ty, self.ff, par, q) }; }); @@ -5824,6 +5875,7 @@ pub(crate) struct UnionCall<'c, 'w> { /// The norm each down takes before its weight in the sums, when the /// layer has one. post: Option<&'c PostNorm>, + scales: Option<&'c [std::sync::atomic::AtomicU32]>, } impl<'c, 'w> UnionCall<'c, 'w> { @@ -5861,6 +5913,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { down, spread: Lanes::Flat, post: None, + scales: None, }) } @@ -5877,6 +5930,28 @@ impl<'c, 'w> UnionCall<'c, 'w> { UnionCall { post, ..self } } + /// Apply the loaded activation policy, using load-allocated slot scales. + pub(crate) fn expert_input( + self, + input: ExpertInput, + scales: &'c [std::sync::atomic::AtomicU32], + ) -> Result { + let scales = match input { + ExpertInput::Direct => None, + ExpertInput::AmaxF16 => { + if scales.len() < self.plan.slots() { + return Err(crate::ModelError::Metadata { + key: "host union expert input".to_string(), + detail: "AmaxF16 requires UnionScratch::set_expert_input at load" + .to_string(), + }); + } + Some(scales) + } + }; + Ok(UnionCall { scales, ..self }) + } + /// The cells the gate/up pass writes: two columns a slot gated — the /// gate's and the up's — one a slot for a relu² layer, the up's alone. fn gu_cells(&self) -> usize { @@ -6215,6 +6290,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { cb: self.down.cb, ptr: qc.as_mut_ptr(), }, + scales: self.scales, } } @@ -6299,6 +6375,14 @@ impl<'c, 'w> UnionCall<'c, 'w> { let mut pacc = profile::CallAcc::new(); let t0 = if lvl > 0 { Some(Instant::now()) } else { None }; run_row_pool(&set, lvl, &mut pacc, true, self.spread)?; + if let Some(scales) = self.scales { + for (row, scale) in store.chunks_exact_mut(self.down.n).zip(scales) { + let scale = f32::from_bits(scale.load(std::sync::atomic::Ordering::Relaxed)); + for v in row { + *v *= scale; + } + } + } let claim_ns = claims.claim_ns.into_inner(); if let Some(t0) = t0 { let wall = t0.elapsed().as_nanos() as u64; @@ -6757,6 +6841,35 @@ mod tests { }; use gguf::{GgmlType, Gguf}; + #[test] + fn expert_range_protection_preserves_values_and_bounds_large_activated_inputs() { + use super::ExpertInput; + for peak in [0.0f32, 1024.0, 1024.125, 2048.0, 2048.25, 1.0e8] { + let original = [peak, -peak * 0.5, peak * 0.25]; + let mut direct = original; + assert_eq!(ExpertInput::Direct.protect(&mut direct), 1.0); + assert_eq!(direct, original); + let mut protected = original; + let scale = ExpertInput::AmaxF16.protect(&mut protected); + assert!( + protected + .iter() + .all(|v| v.abs() <= 1024.0 * (1.0 + 4.0 * f32::EPSILON)) + ); + if peak <= 1024.0 { + assert_eq!(scale, 1.0); + assert_eq!(protected, original); + } else { + assert!(scale > 1.0); + for (actual, expected) in protected.into_iter().zip(original) { + assert!( + (actual * scale - expected).abs() <= expected.abs() * 2.0 * f32::EPSILON + ); + } + } + } + } + /// A GGUF v3 file holding one `[k, 2]` tensor `w` of type `ty` over zero bytes — /// the least the strict reader opens. fn one_tensor_file(ty: GgmlType, k: usize) -> std::path::PathBuf { diff --git a/crates/model/tests/flint_meta.rs b/crates/model/tests/flint_meta.rs index d6121431c..04887c059 100644 --- a/crates/model/tests/flint_meta.rs +++ b/crates/model/tests/flint_meta.rs @@ -106,32 +106,46 @@ fn hw_flint_header() { .filter(|(n, _)| matches!(n, Need::Gqa { sinks: true, .. })) .count(); assert_eq!((norms, rotary, sinks), (49, 35, 49)); - let coverage = model::arch::coverage::check(s, &r.tensors); - for (need, layer) in &needs { + for (need, _) in &needs { if let Need::Router(shape) = need { + let binding = model::arch::models::shape::select_router(*shape).unwrap(); assert_eq!( - model::arch::models::shape::select_router(*shape) - .unwrap_err() - .why, - model::arch::models::shape::Refusal::NoRouterRule + binding.body, + model::arch::models::shape::RouterBody::SigmoidFloor + ); + assert_eq!(binding.rule, shape.rule); + assert_eq!( + (binding.experts(), binding.top_k_min, binding.top_k_max), + (256, 6, 6) ); - assert!(coverage.iter().any(|u| { - u.layer == layer.map(|l| l as usize) && u.feature == need.to_string() - })); } } - assert!( - coverage - .iter() - .any(|u| u.feature == Need::Program("flint").to_string()) - ); - assert!(model::arch::coverage::program_of(s.arch).is_none()); let bytes: u64 = r.tensors.tensors.iter().map(|t| t.file_bytes).sum(); assert_eq!(bytes, 25_049_218_176); + let mut inputs = model::arch::flint::place::PlanInputs::read(&split) + .expect("the validated header has genuine program and site bindings"); + inputs.room = (u64::MAX, model::placement::workstation::HostRead::Given); + let machine = model::placement::workstation::plan_gate(s.layers.len()); + let levers = model::placement::PlanLevers::default(); + for ctx in [512, 1024, 4096] { + for slots in [1, 3] { + let plan = inputs + .plan_with_slots( + &machine, + ctx, + &levers, + std::num::NonZeroUsize::new(slots).unwrap(), + ) + .expect("the real mixed-quant header plans with resident slots"); + let cache = slots as u64 * 2048 * (14 * ctx + 35 * ctx.min(1024)); + assert_eq!(plan.cards.iter().map(|c| c.kv_bytes).sum::(), cache); + assert!(inputs.prompt_batch_bytes(&machine).unwrap() > 0); + } + } + let tier = model::arch::flint::place::tier_batch(&inputs).unwrap(); + assert!(tier.leg.unwrap().input_scales); + assert!(tier.card().unwrap() > 0); println!( "flint header_sha={HEADER_SHA} tensors=779 layers=49 dense=2 routed=47 norms={norms} rotary={rotary} sinks={sinks} payload_bytes={bytes}" ); - for item in coverage { - println!("unimplemented layer={:?} {}", item.layer, item.feature); - } } diff --git a/crates/models/src/shape.rs b/crates/models/src/shape.rs index 45f79bf5b..a49117c8d 100644 --- a/crates/models/src/shape.rs +++ b/crates/models/src/shape.rs @@ -83,6 +83,9 @@ pub enum RouterBody { /// `mimo2_router*`, `sigmoid_320_8_router*`, `sigmoid_128_4_router*`; /// `gpu/src/arch/nemotron_h_moe/router.rs` `nemotron_moe_router_*`. BiasedSigmoid, + /// `gpu/src/route_body.rs` `sigmoid_floor_256_6_router*`: sigmoid, + /// optional selection-only bias and a fixed `2^-14` selected-sum floor. + SigmoidFloor, } /// One compiled router instance. @@ -160,6 +163,16 @@ const BIASED_SIGMOID: RouterRule = RouterRule { norm: RouterNorm::SelectedSum, gated: false, }; +const SIGMOID_FLOOR: RouterRule = RouterRule { + score: Score::Sigmoid, + bias: false, + norm: RouterNorm::SelectedSumFloor, + gated: false, +}; +const BIASED_SIGMOID_FLOOR: RouterRule = RouterRule { + bias: true, + ..SIGMOID_FLOOR +}; const BIASED_SQRT_SOFTPLUS: RouterRule = RouterRule { score: Score::SqrtSoftplus, bias: true, @@ -326,6 +339,24 @@ pub const ROUTERS: &[RouterInst] = &[ top_k_max: 6, at: "gpu/src/route_body.rs sigmoid_256_6_router*", }, + RouterInst { + body: RouterBody::SigmoidFloor, + rule: SIGMOID_FLOOR, + norm_arg: false, + experts: 256, + top_k_min: 6, + top_k_max: 6, + at: "gpu/src/route_body.rs sigmoid_floor_256_6_router*", + }, + RouterInst { + body: RouterBody::SigmoidFloor, + rule: BIASED_SIGMOID_FLOOR, + norm_arg: false, + experts: 256, + top_k_min: 6, + top_k_max: 6, + at: "gpu/src/route_body.rs sigmoid_floor_256_6_router*", + }, // Muse Spark's fixture: 16 experts, the real file's top 8. RouterInst { body: RouterBody::BiasedSigmoid, @@ -531,6 +562,31 @@ pub const fn router_row_k(body: RouterBody, experts: u32, top_k: u32) -> RouterI panic!("no router row for this body at this width and pick count") } +/// A compiled binding whose body, rule, expert count and pick count all +/// agree. Unlike [`router_row_k`], distinguishes optional-bias bindings. +#[must_use] +pub const fn router_row_rule( + body: RouterBody, + rule: RouterRule, + experts: u32, + top_k: u32, +) -> RouterInst { + let mut i = 0; + while i < ROUTERS.len() { + let r = ROUTERS[i]; + if body_eq(r.body, body) + && r.runs(rule) + && r.experts == experts + && r.top_k_min <= top_k + && top_k <= r.top_k_max + { + return r; + } + i += 1; + } + panic!("no router row for this body, rule, width and pick count") +} + /// Whether row `r` is body `body` at `per_lane`, with rule `rule` and the /// pick range `min..=max`: the whole of what a device crate compiled in. #[must_use] @@ -564,6 +620,11 @@ pub mod rules { /// Sigmoid scores, a selection bias steering the choice only, the kept /// unbiased scores renormalized: GLM-5.3's and MiMo-V2's noaux_tc router. pub const BIASED_SIGMOID: RouterRule = super::BIASED_SIGMOID; + /// Sigmoid, no selection bias, selected scores divided by + /// `max(sum_selected, 2^-14)`. + pub const SIGMOID_FLOOR: RouterRule = super::SIGMOID_FLOOR; + /// [`SIGMOID_FLOOR`] with a bias steering selection only. + pub const BIASED_SIGMOID_FLOOR: RouterRule = super::BIASED_SIGMOID_FLOOR; } const fn norm_eq(a: RouterNorm, b: RouterNorm) -> bool { @@ -890,8 +951,8 @@ pub const fn gqa_row(head: u32, group: u32) -> Option { #[cfg(test)] mod tests { use super::{ - AttnShape, GQA, GroupRule, LANE_PICKS, MoeShape, ROUTERS, Refusal, RouterInst, RouterRule, - Shape, gqa_row_v, router_row, router_row_is, router_row_k, rules, select_gqa, + AttnShape, GQA, GroupRule, LANE_PICKS, MoeShape, ROUTERS, Refusal, RouterBody, RouterInst, + RouterRule, Shape, gqa_row_v, router_row, router_row_is, router_row_k, rules, select_gqa, select_router, }; use crate::{Act, Moe, Router, RouterNorm, Score, Shared}; @@ -954,66 +1015,74 @@ mod tests { assert!(matches!(e.why, Refusal::RouterWidth { ref served } if served == &[128])); } + /// Legacy rows retain their epsilon-added or unnormalized arithmetic; + /// only the real floor binding accepts the floor policy. #[test] - fn a_selected_sum_floor_is_not_a_legacy_device_policy() { + fn floor_bindings_are_distinct_from_legacy_policies() { for row in ROUTERS { - let rule = RouterRule { - norm: RouterNorm::SelectedSumFloor, - ..row.rule - }; assert_eq!( select_router(moe(row.rule, row.experts(), row.top_k_min)).unwrap(), *row ); - assert!(!row.runs(rule)); - assert!(!router_row_is( - *row, - rule, - row.norm_arg, - (row.top_k_min, row.top_k_max) - )); - let shape = moe(rule, row.experts(), row.top_k_min); - let refusal = select_router(shape).expect_err("a legacy body bound the floor policy"); - assert_eq!(refusal.shape, Shape::Moe(shape)); - assert_eq!(refusal.why, Refusal::NoRouterRule); + if row.body == RouterBody::SigmoidFloor { + assert_eq!(row.rule.norm, RouterNorm::SelectedSumFloor); + assert!(!row.norm_arg); + } else { + assert!(!row.runs(RouterRule { + norm: RouterNorm::SelectedSumFloor, + ..row.rule + })); + } } } - /// Flint's sigmoid top-six is not the biased top-eight binding, and no - /// compiled body runs its rule: the 256-expert top-6 instance runs the - /// biased sigmoid with the epsilon-added `SelectedSum`, so every other - /// pair of bias and policy, Flint's `SelectedSumFloor` with or without a - /// selection bias among them, is refused by name. - // PIN(2026-10-25): the biased `SelectedSum` pair was refused as - // `TopK { min: 8, max: 8 }` while only MiMo's top-8 row held 256 experts; - // the `sigmoid_256_6` instance's row now serves exactly that pair - // (`sigmoid_256_6_router_is_its_row_beside_mimos_top_8` pins its identity). - // The other five pairs keep `NoRouterRule`. #[test] - fn flints_sigmoid_top_six_is_not_the_biased_top_eight_binding() { + fn floor_top_six_serves_both_bias_modes_and_only_its_shape() { for bias in [false, true] { - for norm in [ - RouterNorm::None, - RouterNorm::SelectedSum, - RouterNorm::SelectedSumFloor, - ] { - let rule = RouterRule { - bias, - norm, - ..rules::BIASED_SIGMOID + let rule = RouterRule { + bias, + ..rules::SIGMOID_FLOOR + }; + let row = select_router(moe(rule, 256, 6)).unwrap(); + assert_eq!(row.body, RouterBody::SigmoidFloor); + assert_eq!(row.rule, rule); + for top_k in [0, 5, 7, 8, 257] { + assert_eq!( + select_router(moe(rule, 256, top_k)).unwrap_err().why, + Refusal::TopK { min: 6, max: 6 } + ); + } + assert!(matches!( + select_router(moe(rule, 128, 6)).unwrap_err().why, + Refusal::RouterWidth { ref served } if served == &[256] + )); + for (score, gated) in [(Score::Softmax, false), (Score::Sigmoid, true)] { + let wrong = RouterRule { + score, + gated, + ..rule }; - let selected = select_router(moe(rule, 256, 6)); - if bias && norm == RouterNorm::SelectedSum { - assert!( - selected.is_ok(), - "the 256/6 instance's own rule: {selected:?}" - ); - continue; - } - let refusal = selected.expect_err("a sigmoid top-six binding was fabricated"); - assert_eq!(refusal.why, Refusal::NoRouterRule); + assert_eq!( + select_router(moe(wrong, 256, 6)).unwrap_err().why, + Refusal::NoRouterRule + ); } } + let legacy = select_router(moe(rules::BIASED_SIGMOID, 256, 6)).unwrap(); + assert_eq!(legacy.body, RouterBody::BiasedSigmoid); + assert_eq!( + select_router(moe( + RouterRule { + norm: RouterNorm::SelectedSum, + ..rules::SIGMOID_FLOOR + }, + 256, + 6 + )) + .unwrap_err() + .why, + Refusal::NoRouterRule + ); } /// GLM-5.3-Flash (model/tests/glm5next_meta.rs:83, "router: sigmoid, 288 @@ -1275,9 +1344,7 @@ mod tests { assert_eq!(MoeShape::of(&m).rule, rules::SOFTMAX_NORM); } - /// Rows are distinct instances: no two rows run one rule at one width - /// and one pick count, and one body holds two rows at a width only by - /// disjoint pick ranges. + /// No two rows select the same rule, expert width and pick count. #[test] fn no_two_router_rows_overlap() { for (i, a) in ROUTERS.iter().enumerate() { @@ -1285,12 +1352,6 @@ mod tests { let picks = a.top_k_min <= b.top_k_max && b.top_k_min <= a.top_k_max; let overlap = a.experts == b.experts && (a.runs(b.rule) || b.runs(a.rule)) && picks; assert!(!overlap, "{} and {}", a.at, b.at); - assert!( - !((a.body, a.experts) == (b.body, b.experts) && picks), - "{} and {}", - a.at, - b.at - ); } assert!(a.top_k_min >= 1 && a.top_k_min <= a.top_k_max, "{}", a.at); assert!(a.per_lane() >= 1 && a.per_lane() <= 32, "{}", a.at); diff --git a/crates/placement/src/accepts.rs b/crates/placement/src/accepts.rs index 617421833..3c14b0665 100644 --- a/crates/placement/src/accepts.rs +++ b/crates/placement/src/accepts.rs @@ -67,6 +67,8 @@ pub enum Program { /// gate·up and down `_sel` entries ([`MUSESPARK_LEG`]) and the rest on the /// host tier. MuseSparkBody, + /// The head128 sink/SWA body with floor-normalized top-six experts. + FlintBody, } /// The program that runs `arch`'s layers; `None` when none does. @@ -85,7 +87,7 @@ pub fn program_of(arch: Arch) -> Option { Arch::NemotronH => Some(Program::NemotronHBody), Arch::NemotronHMoe => Some(Program::NemotronHMoeBody), Arch::GemmaEmbedding | Arch::GemmaEmbedding2 => Some(Program::GemmaEmbedding2Body), - Arch::Flint => None, + Arch::Flint => Some(Program::FlintBody), Arch::MuseSpark => Some(Program::MuseSparkBody), } } @@ -110,6 +112,7 @@ impl Program { Self::GemmaEmbedding2Body => "gemma-embedding2", Self::Raon1Body => "raon1", Self::MuseSparkBody => "musespark", + Self::FlintBody => "flint", } } } @@ -481,7 +484,7 @@ pub struct LegArms { #[must_use] pub fn leg_arms(p: Program) -> Option { match p { - Program::Qwen3moeBody | Program::Qwen35Body => Some(LegArms { + Program::Qwen3moeBody | Program::Qwen35Body | Program::FlintBody => Some(LegArms { gate_up: prompt_leg_gate_up, down: prompt_leg_down, }), @@ -588,7 +591,7 @@ pub fn leg_down_forms(p: Program, model: &crate::placement::ModelTensors) -> ker #[must_use] pub fn routed_rule(p: Program) -> RoutedRule { match p { - Program::Qwen3moeBody | Program::Qwen35Body => RoutedRule::Leg { + Program::Qwen3moeBody | Program::Qwen35Body | Program::FlintBody => RoutedRule::Leg { gate_up: |ty| prompt_leg_gate_up(ty).then(|| leg_card_format(ty)), down: |ty| prompt_leg_down(ty).then(|| leg_card_format(ty)), pair: leg_pair(p), @@ -669,7 +672,7 @@ mod tests { use super::*; use crate::placement::{ModelTensor, ModelTensors, Role}; - const PROGRAMS: [Program; 12] = [ + const PROGRAMS: [Program; 13] = [ Program::Deepseek41Chain, Program::Qwen3moeBody, Program::Qwen35Body, @@ -682,6 +685,7 @@ mod tests { Program::NemotronHMoeBody, Program::Raon1Body, Program::MuseSparkBody, + Program::FlintBody, ]; /// Every type the file format names. diff --git a/crates/placement/src/kernels.rs b/crates/placement/src/kernels.rs index c9b912b2e..57cd42469 100644 --- a/crates/placement/src/kernels.rs +++ b/crates/placement/src/kernels.rs @@ -16,6 +16,9 @@ pub use gguf::card::{ }; use models::Role; +pub mod attention; +pub mod ffn; + /// The common card gate·up `_sel` entry that runs a stack of type `ty`. #[must_use] pub const fn gate_up(ty: GgmlType) -> Option { @@ -984,7 +987,6 @@ mod tests { assert_eq!(held(f), [ty], "{f:?}"); } } - /// A card entry reads a layout the card loader can build: every type a /// row names has a [`CardFormat`] of its own, or is one of the file-block /// types a program's routed stack keeps as `KQuant`. diff --git a/crates/placement/src/kernels/attention.rs b/crates/placement/src/kernels/attention.rs new file mode 100644 index 000000000..c896272fd --- /dev/null +++ b/crates/placement/src/kernels/attention.rs @@ -0,0 +1,79 @@ +//! The buffers of the common head128 projection/attention scratch. + +use super::{SiteAct, try_q8_act_bytes}; + +/// Quantized wide forms beside the caller-owned f32 input rows. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Forms { + pub q128: bool, + pub q32: bool, +} + +impl Forms { + /// Include the activation form a selected projection reads. + #[must_use] + pub const fn with(self, form: SiteAct) -> Self { + Self { + q128: self.q128 || matches!(form, SiteAct::Q8x128), + q32: self.q32 || matches!(form, SiteAct::Q8x32 | SiteAct::Q8Blocks32), + } + } + + /// Bytes of these forms over `rows` columns of `width` values. + #[must_use] + pub fn bytes(self, rows: u64, width: u64) -> Option { + let q128 = if self.q128 { + try_q8_act_bytes(rows, width)? + } else { + 0 + }; + let q32 = if self.q32 { + wide32_bytes(rows, width)? + } else { + 0 + }; + q128.checked_add(q32) + } +} + +/// Bytes of wide q8 blocks of 32: twenty words per padded 64-value step. +#[must_use] +pub const fn wide32_bytes(rows: u64, width: u64) -> Option { + match rows.checked_mul(width.div_ceil(64)) { + Some(steps) => steps.checked_mul(80), + None => None, + } +} + +/// The loaded projection widths, partial segments and actual selected forms. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Head128Scratch { + pub hidden: u64, + pub query: u64, + pub key_value: u64, + pub heads: u64, + pub segments: u64, + pub input: Forms, + pub output: Forms, +} + +impl Head128Scratch { + /// Checked bytes for Q/K/V, normalized/residual rows, attention/partials, + /// selected wide forms and the one-matrix route table. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let partial = self.heads.checked_mul(self.segments)?.checked_mul(130)?; + let values = self + .hidden + .checked_mul(2)? + .checked_add(self.query.checked_mul(2)?)? + .checked_add(self.key_value.checked_mul(2)?)? + .checked_add(partial)?; + let floats = values.checked_mul(rows)?.checked_mul(4)?; + let route = super::ffn::dense_route_bytes(rows)?; + floats + .checked_add(self.input.bytes(rows, self.hidden)?)? + .checked_add(self.output.bytes(rows, self.query)?)? + .checked_add(route) + } +} diff --git a/crates/placement/src/kernels/ffn.rs b/crates/placement/src/kernels/ffn.rs new file mode 100644 index 000000000..0087d0343 --- /dev/null +++ b/crates/placement/src/kernels/ffn.rs @@ -0,0 +1,234 @@ +//! Buffer geometry of the common routed/shared FFN and prompt walk. + +use super::SiteAct; +use super::attention::Forms; +use crate::placement::workstation::TierLegScratch; + +/// The first unit size whose shared and dense blocks use the wide path. +pub const GATED_FROM: u64 = 9; + +/// Wide quantized forms and an owned f32 row, when a selected down reads one. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct WideForms { + pub quant: Forms, + pub f32: bool, +} + +impl WideForms { + /// Include a selected site's activation form. + #[must_use] + pub const fn with(self, reads: SiteAct) -> Self { + Self { + quant: self.quant.with(reads), + f32: self.f32 || matches!(reads, SiteAct::F32), + } + } + + /// Input f32 rows are borrowed from the caller. + #[must_use] + pub const fn without_f32(self) -> Self { + Self { f32: false, ..self } + } + + /// Checked allocation bytes of these forms. + #[must_use] + pub fn bytes(self, rows: u64, width: u64) -> Option { + let floats = if self.f32 { + rows.checked_mul(width)?.checked_mul(4)? + } else { + 0 + }; + self.quant.bytes(rows, width)?.checked_add(floats) + } +} + +/// One shared or dense block's wide scratch, with the actual selected read forms. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WideBlock { + pub hidden: u64, + pub ff: u64, + pub input: WideForms, + pub down: WideForms, +} + +impl WideBlock { + /// Route table, borrowed-input forms, owned activation forms, and gate/up outputs. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + dense_route_bytes(rows)? + .checked_add(self.input.without_f32().bytes(rows, self.hidden)?)? + .checked_add(self.down.bytes(rows, self.ff)?)? + .checked_add(rows.checked_mul(self.ff)?.checked_mul(8)?) + } +} + +/// A single-expert wide route table: slot ids, tiles, counts and zero ids. +#[must_use] +pub const fn dense_route_bytes(rows: u64) -> Option { + let bytes = (2 * rows as u128 + 2 * (rows as u128 / 64 + 1) + 2) * 4; + if bytes > u64::MAX as u128 { + None + } else { + Some(bytes as u64) + } +} + +/// The common card batch beside its leg: accumulation/base rows and selected places. +#[must_use] +pub fn card_batch_bytes(leg: TierLegScratch) -> Option { + leg.bytes(leg.downs) + .ok()? + .checked_add(leg.chunk.checked_mul(leg.n_embd)?.checked_mul(8)?)? + .checked_add(leg.chunk.checked_mul(leg.n_used)?.checked_mul(4)?) +} + +/// Places, join places and join weights of a prompt unit on an expert-tier load. +#[must_use] +pub const fn tier_places_bytes(rows: u64, used: u64) -> Option { + match rows.checked_mul(used) { + Some(slots) => slots.checked_mul(12), + None => None, + } +} + +/// Exact buffers shared by all units of the routed/shared prompt walk. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RoutedBatch { + pub hidden: u64, + pub experts: u64, + pub used: u64, + pub shared: WideBlock, + pub dense: Option, + pub card: Option, + pub tiered: bool, +} + +impl RoutedBatch { + /// Norm/router/shared/base rows, narrow activation rows, optional dense/card/wide/tier buffers. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let dense_ff = self.dense.map_or(0, |d| d.ff); + let mut values = self + .hidden + .checked_mul(3)? + .checked_add(self.experts)? + .checked_add(self.used.checked_mul(2)?)? + .checked_add(self.shared.ff.max(dense_ff))?; + if self.dense.is_some() { + values = values.checked_add(self.hidden.checked_mul(2)?)?; + } + let mut bytes = rows.checked_mul(values)?.checked_mul(4)?; + if rows >= GATED_FROM { + bytes = bytes.checked_add(self.shared.bytes(rows)?)?; + if let Some(dense) = self.dense { + bytes = bytes.checked_add(dense.bytes(rows)?)?; + } + } + if let Some(card) = self.card { + if card.chunk != rows { + return None; + } + bytes = bytes.checked_add(card_batch_bytes(card)?)?; + } + if self.tiered { + bytes = bytes.checked_add(tier_places_bytes(rows, self.used)?)?; + } + Some(bytes) + } +} + +/// Prompt walk units and the shared host-sum upload buffer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PromptWalk { + pub hidden: u64, + pub sets: u64, +} + +impl PromptWalk { + /// Each unit owns x/x1 plus positions/key counts; all units share one host-sum row set. + #[must_use] + pub fn bytes(self, rows: u64) -> Option { + let unit = self.hidden.checked_mul(8)?.checked_add(8)?; + self.sets + .checked_mul(unit)? + .checked_add(self.hidden.checked_mul(4)?)? + .checked_mul(rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kernels::{DownAct, DownForms}; + + #[test] + fn wide_threshold_selected_forms_and_protection_are_independent() { + let wide = WideBlock { + hidden: 256, + ff: 256, + input: WideForms::default().with(SiteAct::F32), + down: WideForms::default().with(SiteAct::F32), + }; + let batch = RoutedBatch { + hidden: 256, + experts: 8, + used: 2, + shared: wide, + dense: None, + card: None, + tiered: false, + }; + assert_eq!(batch.bytes(8), Some(8 * 4 * (3 * 256 + 8 + 4 + 256))); + assert_eq!( + batch.bytes(9), + Some(9 * 4 * (3 * 256 + 8 + 4 + 256) + dense_route_bytes(9).unwrap() + 9 * 256 * 12) + ); + let leg = TierLegScratch { + n_embd: 256, + ff: 256, + n_used: 2, + chunk: 9, + downs: DownForms::of(DownAct::Q8Act), + input_scales: false, + }; + let direct = card_batch_bytes(leg).unwrap(); + let scaled = card_batch_bytes(TierLegScratch { + input_scales: true, + ..leg + }) + .unwrap(); + assert_eq!(scaled - direct, 4 * 9 * 2); + let q32 = WideBlock { + input: WideForms::default().with(SiteAct::Q8x32), + ..wide + }; + assert_eq!(q32.bytes(9).unwrap() - wide.bytes(9).unwrap(), 9 * 4 * 80); + assert_eq!(batch.bytes(u64::MAX), None); + } + + #[test] + fn walk_groups_only_duplicate_unit_rows() { + let one = PromptWalk { + hidden: 256, + sets: 1, + } + .bytes(512) + .unwrap(); + let two = PromptWalk { + hidden: 256, + sets: 2, + } + .bytes(512) + .unwrap(); + assert_eq!(one, 512 * (12 * 256 + 8)); + assert_eq!(two - one, 512 * (8 * 256 + 8)); + assert_eq!( + PromptWalk { + hidden: u64::MAX, + sets: 1 + } + .bytes(512), + None + ); + } +} diff --git a/crates/placement/src/placement.rs b/crates/placement/src/placement.rs index f470d2617..69267188f 100644 --- a/crates/placement/src/placement.rs +++ b/crates/placement/src/placement.rs @@ -250,13 +250,10 @@ pub struct Card { /// draft model's resident bytes, say): taken from the budget with the /// context and the scratch ([`Card::set_aside_bytes`]). pub reserves: Vec<(String, u64)>, + /// A tier card holding only reserves, not experts. + pub reserve_only: bool, /// A tier leg's scratch, jointly selected with its resident experts. pub tier_leg: Option, - /// A tier card with an expert cap of 0: it holds its reserves and no - /// expert, and serves no tier. The plan refuses one with no reserve - /// ([`PlacementError::IdleTier`]); a body hangs a tier only under the - /// tier cards that are not reserve-only ([`Machine::expert_tiers`]). - pub reserve_only: bool, } impl Card { @@ -1240,6 +1237,139 @@ pub struct Plan<'a> { pub budget_from_pool: bool, } +/// The one-stage reference of a one-expert-tier plan: the stage holds the union +/// of its routed ids and the tier's, and the host segments are unchanged. +/// `flat` is the same stage machine with the tier removed. The reference uses +/// the stage's actual capacity, not the artificial budget that split the sets. +pub fn union_plan<'a>(plan: &Plan<'a>, flat: &'a Machine) -> Result, PlacementError> { + if plan.machine.cards.len() != 1 + || plan.machine.expert_tiers().count() != 1 + || flat.cards.len() != 1 + || !flat.tiers.is_empty() + || !plan.machine.cards[0].same_device(&flat.cards[0]) + { + return Err(PlacementError::Stages( + "union reference needs one stage, one expert tier and the same stage without tiers" + .into(), + )); + } + let model = plan.model; + let tier = Device::Card(plan.machine.cards.len()); + let mut out = plan.clone(); + for row in &mut out.rows { + let t = &model.tensors[row.tensor]; + if t.role != Role::RoutedExperts { + continue; + } + let ids = |d: Device| -> Vec { + row.segments + .iter() + .filter(|s| s.device == d) + .filter_map(|s| s.experts.as_ref()) + .flat_map(|e| e.ids().iter().copied()) + .collect() + }; + let on_tier = ids(tier); + if on_tier.is_empty() { + continue; + } + let mut union = ids(Device::Card(0)); + union.extend(on_tier); + let n = union.len() as u64; + let mut format = None; + for segment in row + .segments + .iter() + .filter(|s| s.device == Device::Card(0) || s.device == tier) + { + let Format::Card(selected) = segment.format else { + return Err(PlacementError::tensor( + t, + "union reference needs resident card expert segments", + )); + }; + if format.is_some_and(|f| f != selected) { + return Err(PlacementError::tensor( + t, + "union reference cannot join different selected card formats", + )); + } + format = Some(selected); + } + let card = routed_row_on_as( + row.tensor, + t, + 0, + vec![(0, ExpertList::new(union, model.experts_of(t))?)], + model, + format, + )?; + let segments = card + .segments + .into_iter() + .filter(|s| s.device == Device::Card(0)) + .chain( + row.segments + .iter() + .filter(|s| !matches!(s.device, Device::Card(_))) + .cloned(), + ) + .collect(); + *row = Row { + segments, + ..row.clone() + }; + if let Some(l) = t.layer.and_then(|l| out.n_l.get_mut(l)) { + *l = n; + } + } + out.machine = flat; + out.cards.truncate(1); + out.tier_n_l.clear(); + out.card_budget = card_budget::CardBudget::default(); + out.budget_from_pool = false; + let (heaps, _) = card_heaps(model, flat, &out.rows); + let card = &flat.cards[0]; + let total = &mut out.cards[0]; + total.expert_bytes = out + .rows + .iter() + .flat_map(|r| { + r.segments.iter().filter_map(|s| { + (model.tensors[r.tensor].role == Role::RoutedExperts && s.device == Device::Card(0)) + .then_some(s.resident_bytes) + }) + }) + .sum(); + total.experts = out.n_l.iter().sum(); + total.rounding_bytes = heaps[0].taken - total.dense_bytes - total.expert_bytes; + total.reserve_bytes = card.selected_reserve_bytes(model, &out.n_l)?; + let used = card + .floor_with_reserves( + heaps[0].taken, + total.kv_bytes, + total.stage_rows_bytes, + total.reserve_bytes, + ) + .ok_or_else(|| card.bytes_past_u64())?; + total.headroom_bytes = i128::from(capped(card, None)) - i128::from(used); + let limit = capped(card, None).saturating_sub(card.margin_bytes); + if used > limit { + return Err(PlacementError::Metadata { + key: "union reference".into(), + detail: Violation::CardOver { + card: card.name.clone(), + total: used, + limit, + } + .to_string(), + }); + } + out.host.headroom_bytes = workstation::HostNeed::of_totals(flat.host(), &out.host, false, 0) + .headroom(workstation::HostNeed::machine_room(flat.host())); + Ok(out) +} + /// Why a placement could not be built; each variant names what it refused. #[derive(Debug, thiserror::Error)] pub enum PlacementError { @@ -6467,6 +6597,147 @@ mod tests { assert_eq!(plan.n_l, vec![8]); } + #[test] + fn a_union_recounts_uploads_selected_forms_and_capacity() { + use crate::kernels::{DownAct, DownForms}; + use workstation::{TIER_BATCH_RESERVE, try_tier_leg_batch_bytes_of}; + + let mut model = legged(&[ + [GgmlType::IQ3_S, GgmlType::IQ3_S, GgmlType::Q5_K], + [GgmlType::IQ4_XS, GgmlType::IQ4_XS, GgmlType::Q8_0], + ]); + model.experts_used = 6; + let forms = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + let batch = try_tier_leg_batch_bytes_of(256, 256, 6, (2, 3), forms, false).unwrap(); + let mut stage = bytes_card("stage", 1 << 30, 0..2); + stage.tier_leg = batch.leg; + stage + .reserves + .push((TIER_BATCH_RESERVE.into(), batch.card().unwrap())); + let flat = Machine { + cards: vec![stage], + tiers: Vec::new(), + nodes: vec![Node { + host: host(), + unified: None, + }], + }; + let machine = Machine { + tiers: vec![bytes_card("tier", 1 << 30, 0..0)], + ..flat.clone() + }; + let rule = crate::accepts::routed_rule(crate::accepts::Program::FlintBody); + let (index, iq3) = model + .tensors + .iter() + .enumerate() + .find(|(_, t)| t.role == Role::RoutedExperts && t.ty == GgmlType::IQ3_S) + .unwrap(); + assert!(matches!( + routed_row( + index, + iq3, + 0, + ExpertList::new(vec![0], model.experts).unwrap(), + &model + ), + Err(PlacementError::NoCardFormat { .. }) + )); + let mut rows = plan_routed(&model, &flat, 1, &NoKv, &PlanLevers::default(), rule) + .unwrap() + .rows; + for row in &mut rows { + let tensor = &model.tensors[row.tensor]; + if tensor.role == Role::RoutedExperts { + let device = usize::from(tensor.layer == Some(1)); + *row = routed_row_on_as( + row.tensor, + tensor, + 0, + vec![(device, ExpertList::new(vec![0], model.experts).unwrap())], + &model, + rule.format(tensor).unwrap(), + ) + .unwrap(); + } + } + let plan = totals( + &model, + &machine, + 1, + rows, + &[(0, 0, 0); 2], + (vec![1, 0], vec![vec![0, 1]]), + card_budget::CardBudget::default(), + ) + .unwrap(); + let joined = union_plan(&plan, &flat).unwrap(); + assert_eq!(joined.n_l, [1, 1]); + assert!(joined.tier_n_l.is_empty()); + assert_eq!( + joined.cards[0].expert_bytes, + plan.cards[0].expert_bytes + plan.cards[1].expert_bytes + ); + assert_eq!(joined.cards[0].experts, 2); + let heap = card_heaps(&model, &flat, &joined.rows).0.remove(0); + assert_eq!( + joined.cards[0].rounding_bytes, + heap.taken - joined.cards[0].dense_bytes - joined.cards[0].expert_bytes + ); + let reserve = batch.staging + batch.leg.unwrap().bytes(forms).unwrap(); + assert_eq!(joined.cards[0].reserve_bytes, reserve); + assert_eq!( + joined.card_reserve(0, TIER_BATCH_RESERVE).unwrap(), + Some(reserve) + ); + assert!(reserve > plan.cards[0].reserve_bytes); + assert_eq!(joined.cards[0].kv_bytes, plan.cards[0].kv_bytes); + for (before, after) in plan.rows.iter().zip(&joined.rows) { + let host = |row: &Row| { + row.segments + .iter() + .filter(|s| !matches!(s.device, Device::Card(_))) + .map(|s| { + ( + s.device, + s.resident_bytes, + s.experts.as_ref().map(|e| e.ids().to_vec()), + ) + }) + .collect::>() + }; + assert_eq!(host(before), host(after)); + } + let mut conflicting = plan.clone(); + let row = conflicting + .rows + .iter_mut() + .find(|r| r.segments.iter().any(|s| s.device == Device::Card(1))) + .unwrap(); + let mut other = row + .segments + .iter() + .find(|s| s.device == Device::Card(1)) + .unwrap() + .clone(); + other.device = Device::Card(0); + other.format = Format::Card(CardFormat::F32); + row.segments.push(other); + assert!(matches!( + union_plan(&conflicting, &flat), + Err(PlacementError::Tensor { .. }) + )); + let mut short = flat.clone(); + short.cards[0].usable_bytes = joined.cards[0].bytes() - 1; + assert!(matches!(union_plan(&plan, &short), + Err(PlacementError::Metadata { key, .. }) if key == "union reference")); + let mut wrong = flat.clone(); + wrong.cards.clear(); + assert!(matches!( + union_plan(&plan, &wrong), + Err(PlacementError::Stages(_)) + )); + } /// Scratch enters each expert candidate's footprint monotonically. /// A boundary that admits one down form cannot pay for an unused second /// form, and adding the first expert of that form pays its exact bytes. @@ -6478,7 +6749,8 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, try_tier_leg_batch_bytes_of}; let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); - let mixed = try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both).expect("tier bytes"); + let mixed = + try_tier_leg_batch_bytes_of(256, 256, 1, (2, 3), both, false).expect("tier bytes"); let rule = routed_rule(Program::Qwen35Body); for (downs, first, batch) in [ ([Q5_K, Q8_0], DownAct::Q8Act, mixed), @@ -6559,7 +6831,8 @@ mod tests { use workstation::{TIER_BATCH_RESERVE, TierLegScratch, try_tier_leg_batch_bytes_of}; let forms = DownForms::of(DownAct::Q8Act); - let batch = try_tier_leg_batch_bytes_of(256, 256, 1, (1, 1), forms).expect("valid batch"); + let batch = + try_tier_leg_batch_bytes_of(256, 256, 1, (1, 1), forms, false).expect("valid batch"); let good = batch.leg.expect("a leg descriptor"); let mut model = legged(&[[GgmlType::Q5_K; 3]]); model.experts_used = 1; @@ -6619,14 +6892,14 @@ mod tests { assert!( matches!( try_tier_leg_batch_bytes_of( - leg.n_embd, leg.ff, leg.n_used, (leg.chunk, 1), leg.downs, + leg.n_embd, leg.ff, leg.n_used, (leg.chunk, 1), leg.downs, leg.input_scales, ), Err(PlacementError::Metadata { key, .. }) if key == "tier leg scratch" ), "{leg:?}" ); } - assert!(try_tier_leg_batch_bytes_of(256, 256, 8, (1, 1 << 61), forms).is_err()); + assert!(try_tier_leg_batch_bytes_of(256, 256, 8, (1, 1 << 61), forms, false).is_err()); let inconsistent = workstation::TierBatchBytes { scratch: 0, ..batch diff --git a/crates/placement/src/placement/workstation.rs b/crates/placement/src/placement/workstation.rs index 5ab150537..a3f9a6fa5 100644 --- a/crates/placement/src/placement/workstation.rs +++ b/crates/placement/src/placement/workstation.rs @@ -439,6 +439,8 @@ pub struct TierLegScratch { pub n_used: u64, pub chunk: u64, pub downs: DownForms, + /// One f32 input-restoration scale per slot of the leg's chunk. + pub input_scales: bool, } impl TierLegScratch { @@ -462,7 +464,16 @@ impl TierLegScratch { { return Err(invalid()); } - try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs).ok_or_else(invalid) + let counted = || { + let bytes = try_leg_bytes(self.chunk, self.n_used, self.n_embd, self.ff, downs)?; + let scales = if self.input_scales { + 4u64.checked_mul(self.chunk)?.checked_mul(self.n_used)? + } else { + 0 + }; + bytes.checked_add(scales) + }; + counted().ok_or_else(invalid) } } @@ -617,6 +628,7 @@ pub fn tier_leg_batch_bytes( n_used, (chunk, cols), DownForms::of(DownAct::Q8Act), + false, ) } @@ -628,6 +640,7 @@ pub fn try_tier_leg_batch_bytes_of( n_used: u64, (chunk, cols): (u64, u64), downs: DownForms, + input_scales: bool, ) -> Result { let leg = TierLegScratch { n_embd, @@ -635,6 +648,7 @@ pub fn try_tier_leg_batch_bytes_of( n_used, chunk, downs, + input_scales, }; let scratch = leg.bytes(downs)?; let counted = || -> Option<(u64, u64)> { @@ -1746,6 +1760,41 @@ mod tests { plan_on, plan_room, plan_tiers, spec_of_device, tier_batch_bytes, unified_of, }; + #[test] + fn tier_input_scales_cost_one_f32_per_chunk_slot() { + use crate::kernels::{DownAct, DownForms}; + + let both = DownForms::of(DownAct::Q8Act).with(DownAct::Q8Blocks32); + for chunk in [1, 8, 16] { + for downs in [DownForms::NONE, DownForms::of(DownAct::Q8Act), both] { + let direct = + super::try_tier_leg_batch_bytes_of(256, 256, 6, (chunk, 32), downs, false) + .expect("direct descriptor"); + let scaled = + super::try_tier_leg_batch_bytes_of(256, 256, 6, (chunk, 32), downs, true) + .expect("scaled descriptor"); + assert_eq!(scaled.staging, direct.staging); + assert_eq!(scaled.host, direct.host); + assert_eq!(scaled.scratch, direct.scratch + 4 * chunk * 6); + assert_eq!( + scaled.card().expect("scaled card"), + direct.card().expect("direct card") + 4 * chunk * 6, + ); + let leg = scaled.leg.expect("scaled leg"); + assert!(leg.input_scales); + assert_eq!( + leg.bytes(DownForms::NONE).expect("empty selected union"), + direct + .leg + .expect("direct leg") + .bytes(DownForms::NONE) + .expect("empty union") + + 4 * chunk * 6, + ); + } + } + } + /// One plan room: `tools/ref/plan-room.sh --room`, the room /// `records-refresh` makes V4.1's plans at, prints [`PLAN_ROOM`], and /// the workstation's host leaves a load that room diff --git a/docs/BUILD.md b/docs/BUILD.md index 9a4621005..d875ba3ef 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -85,7 +85,7 @@ This builds the V4.1 CLI, the GPU kernels and the host expert tier into one bina |---|---|---| | `generate_ds41`, `bloomery-chat` | `cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features deepseek41 --release --bin ` | DeepSeek-V4.1-Flash | | `bloomery-serve-ds41`, `bloomery-serve-qwen38` | the same, `--features deepseek41,gemma-embedding2,hf` (`gemma-embedding2` attaches the optional embedding side seat) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next | -| `bloomery-serve` | `… --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve` (`--model ds41\|qwen38\|glm\|qwen3\|mimo2\|solar2\|minimaxm3\|nemotron\|raon1\|decide`; the ds41 and qwen38 seats are the two binaries above; `vision` adds V4.1's `--mmproj`; `checkpoint` opens a Qwen3.8 safetensors checkpoint folder as stored) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next, GLM-5.3-Flash, Qwen3.6-35B-A3B, Qwen3-30B-A3B, MiMo-V2.6-Flash, Solar-Open2-250B, MiniMax-M3, Raon-1-mini-preview, Nemotron 3 Nano 4B, Nemotron 3.5 Bolt 74B-A9B, Nemotron-3.5-Lightning 30B-A3B, Clef-Flash, lev | +| `bloomery-serve` | `… --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve` (`--model ds41\|qwen38\|glm\|qwen3\|mimo2\|solar2\|minimaxm3\|nemotron\|raon1\|decide`; the ds41 and qwen38 seats are the two binaries above; `vision` adds V4.1's `--mmproj`; `checkpoint` opens a Qwen3.8 safetensors checkpoint folder as stored) | DeepSeek-V4.1-Flash, Qwen3.8-Flash-Next, GLM-5.3-Flash, Qwen3.6-35B-A3B, Qwen3-30B-A3B, MiMo-V2.6-Flash, Solar-Open2-250B, MiniMax-M3, Raon-1-mini-preview, Nemotron 3 Nano 4B, Nemotron 3.5 Bolt 74B-A9B, Nemotron-3.5-Lightning 30B-A3B, Clef-Flash, lev | | `generate_qwen3moe` | `… --features gpu --release --bin generate_qwen3moe` | Qwen3.8-Flash-Next, Qwen3.6-35B-A3B, Qwen3-30B-A3B | | `generate_glm5next` | `… --features glm5next-hf --release --bin generate_glm5next` | GLM-5.3-Flash | | `r8conv` | `cargo build --release -p bloomery-model --bin r8conv` | the V4.1 sidecar | @@ -256,10 +256,10 @@ target/release/bloomery-tokenize -m "$M" --decode -p "$(sed -n 's/^tokens //p' g Flags: `-n N` (default 16), `--ctx C` (default 2048, at most 16,384), `--place a|gate` (default `gate`), `--prefill batch|steps` (default `batch`), `--mode graph|eager`, `--plan` (print the plan and exit before the load), `--logits`, `--time [--warm W]`. -The server is `bloomery-serve --model glm` (build it with `--features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf --bin bloomery-serve`). With `--place` unset it plans `bp` when a second card is visible and keeps it when the plan puts at least 526 experts on that card (the break-even, derived in `docs/cards/glmbp-ab.card`), else it runs `a`; a `place unset` line on stderr names the pick, the tier's experts and the break-even. On the development machine's A6000 + 3090 the `bp` plan holds 1,501 tier experts, and one request (MTP draft on, residency `mid-p0-s1`) decodes 13.0 % ± 2.8 faster than under `a` after a 512-token prompt and prefills it 11.1 % ± 0.5 faster (rig-log 10-06#glmbp-ab); two requests run 35.26 tok/s in total after a 512-token prompt (rig-log 10-06#num3090). At `--place a` and `--place bp` (the 3090 as an expert tier) it runs adaptive residency and the MTP draft by default (`BLOOMERY_RESIDENCY=off` and `BLOOMERY_DRAFT=off` turn them off), and `--plan` prints the plan and those choices and exits before any card is opened: +The server is `bloomery-serve --model glm` (build it with `--features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf --bin bloomery-serve`). With `--place` unset it plans `bp` when a second card is visible and keeps it when the plan puts at least 526 experts on that card (the break-even, derived in `docs/cards/glmbp-ab.card`), else it runs `a`; a `place unset` line on stderr names the pick, the tier's experts and the break-even. On the development machine's A6000 + 3090 the `bp` plan holds 1,501 tier experts, and one request (MTP draft on, residency `mid-p0-s1`) decodes 13.0 % ± 2.8 faster than under `a` after a 512-token prompt and prefills it 11.1 % ± 0.5 faster (rig-log 10-06#glmbp-ab); two requests run 35.26 tok/s in total after a 512-token prompt (rig-log 10-06#num3090). At `--place a` and `--place bp` (the 3090 as an expert tier) it runs adaptive residency and the MTP draft by default (`BLOOMERY_RESIDENCY=off` and `BLOOMERY_DRAFT=off` turn them off), and `--plan` prints the plan and those choices and exits before any card is opened: ```sh -cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve +cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf --release --bin bloomery-serve BLOOMERY_REF_MODEL="$M" target/release/bloomery-serve --model glm --place a --ctx 4096 --host 127.0.0.1 --port 8080 ``` @@ -310,7 +310,7 @@ The same binary opens the Qwen3-30B-A3B-Instruct-2507 `Q4_K_M` file (`unsloth/Qw [Clef-Flash](https://huggingface.co/Cloudflare/clef-flash) is Cloudflare's decision model: a Qwen3.5 backbone and a small joint schema head that scores every allowed option of every question in one prompt pass. bartowski's current GGUF upload is llama.cpp's `clef` layout, which carries the head inside the file; an older `qwen35`-layout file takes the release's own head file. ```sh -cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf --release --bin bloomery-serve +cargo oxide build --arch sm_86 -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf --release --bin bloomery-serve # the backbone on one card (see Picking the card), the head (in the file) on the host target/release/bloomery-serve --hf bartowski/Cloudflare_clef-flash-GGUF:Q8_0 --port 8091 diff --git a/docs/cards/flintclip-tape.card b/docs/cards/flintclip-tape.card new file mode 100644 index 000000000..af82f3a23 --- /dev/null +++ b/docs/cards/flintclip-tape.card @@ -0,0 +1,28 @@ +# The Flint 40B-A2.5B UD-Q4_K_XL (unsloth GGUF) toktape, fourth sitting: one arm, `n6000b` (the P4 prompt, N 6000, so the answer can come +# after the thought), greedy, two discarded warm-ups, a fresh load of the 25.05 GB file with both cards visible (RTX A6000 first, RTX 3090 +# at its 250 W cap), bloomery-serve -m at the serving defaults (every BLOOMERY_* lever unset, --place unset, --ctx-size 16384, no +# --parallel). The run waits for a CPU-quiet box first (loadavg1 < 4 and no compiler process for 60 s, polled every 30 s, at most 20 min, +# then records anyway and says so); the lease is held while it waits, so no new box command starts beside it. After the server is up and warmed it also +# waits until io pressure (some avg10) is under 2 for 30 s, at most 5 min, so the tape's start edge reads uncontended. +# Why: the third sitting's tape (n6000) finished the answer but toktape marked it `contended: yes` (io pressure 7.98 > 5 at the start edge, this run's own +# preheat and load); this re-record changes only that start condition. Before that: the first sitting (N 1500) and the second (N 4000, a peer's rustc/LLVM build beside it, loadavg 2 -> 16) never produced a finished answer; +# the thought ran 3986 tokens and the answer had just begun at the 4000 cap. +# Placement [read from the serve print]: `place=a`, A6000 only, 12032 of 12032 experts on the card, host_experts=0, slots=1, graph_nodes=1378. +# Host DRAM, PCIe and NVMe carry nothing per token; the decode step is the card's node latency (inter-token median 12.4 ms in all three +# earlier tapes against a 3.3 ms weight-byte floor), so wall = the card. +# Measured inputs [lease runs, 2026-10-27]: median inter-token 12.36 / 12.45 / 12.37 ms; means 79.7 (N 1500, loadavg 4 -> 12), 75.8 and 76.4 (loadavg +# 2 -> 16 and 10 -> 16); prefill 811 / 824 / 814 tok/s, TTFT 6607 / 6498 / 6575 ms at 5342 prompt tokens; the first 3986 tokens are thinking. +# Predictions [derived]. Greedy, same prompt and sha: reasoning_n 3986 exactly (the n4000 tape's thought ended with `Let's final.`), content 150..350 tokens +# (8 short bullets and 3 settings), finish stop at 4140..4340 tokens, well under 6000. Prefill 810..830 tok/s, TTFT 6.5..6.6 s. Decode on a quiet box: the +# median 12.4 ms = 80.6 tok/s, less the KV growth (14 x 2048 B per extra row, 0.13 ms between depth 6.1K and 9.5K = 1 %), mean 78.5..80.3 tok/s; +# if the box stays loaded the mean falls toward 76 as in the second sitting while the median stays 12.4 ms. +# The third tape's figures repeat: decode 80.2 tok/s, prefill 823 tok/s, TTFT 6506 ms, reasoning 3986, content 277; the same greedy tokens are predicted again, +# decode 79.5..80.5, prefill 815..830, TTFT 6.45..6.6 s, and `contended: no`. The one term the derivation cannot fix: how long the box takes to go quiet and +# how long io pressure takes to fall after the load (a few seconds to minutes); the card pays it as hold minutes. +# Wall [derived]: quiet wait 0..20 min + preheat 5..17 s + load 10..12 s + warm-ups ~4 s + the tape (TTFT 6.5 s + 4140..4340 tokens / 79 tok/s = 59..62 s) +# + io wait 30 s..5 min + stop ~2 s + the publish dry-run ~3 s = about 2.1 min plus the waits: 2..27 minutes, under the script's 1700 s bound. +kind: exclusive +reason: the lead asked for a Flint clip whose answer completes (2026-10-27): the second tape's answer began at token 3987 and the cap cut it; + per-token times are the clip, so the box must be quiet (a build beside it moved the second sitting's means); the file is 25.05 GB, all of it on + the A6000, one fresh load and three requests +minutes: 2..27 diff --git a/docs/contrib/dgx-spark.md b/docs/contrib/dgx-spark.md index 6f3ce6f4f..6fb6aa40e 100644 --- a/docs/contrib/dgx-spark.md +++ b/docs/contrib/dgx-spark.md @@ -56,7 +56,7 @@ the tested toolkit, answer 7) with LLVM 21.1.8's ARM64 release `llc` named by `C for example: ```sh -cargo oxide build --arch sm_121a -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve +cargo oxide build --arch sm_121a -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve ``` Recipes select `sm_121a` on a GB10 when `BLOOMERY_ARCH` is unset. This enables the existing native FP4 routed prompt diff --git a/docs/research/flint-attach.md b/docs/research/flint-attach.md index 837e44015..81260139e 100644 --- a/docs/research/flint-attach.md +++ b/docs/research/flint-attach.md @@ -2,8 +2,9 @@ Design snapshot: **`e53cabebbb9b16249c81b7c26a026da1670f7d9d`**, requested by [#632](https://github.com/midagedev/bloomery/issues/632). The source-gap observations below describe -that snapshot. This component tree has a header-only reader, shared tier-handoff work, the ordinary -IQ4_XS shared site and selected common-leg work, but no Flint body, app session or serve seat. Primitive GPU evidence is not Flint model admission. +that snapshot. The integration adds an actual `gpu-flint` body, application session and shared +HTTP seat, not just architecture admission. Sections 10–11 retain source-specific historical +execution evidence; section 12 records the current existing-PR cutover and remaining acceptance. ## Six-slot handoff closure checks — 2026-10-26 @@ -37,15 +38,15 @@ It reuses the common metadata validators, tensor stems, classification and dimen GemmaEmbedding2's sliding mask: missing keys, scalars, wrong lengths and non-bool items are refused. Present `expert_weights_scale` uses `arch::finite_positive`; only an absent key defaults to 1, with that default recorded. -The common needs walk emits q/k normalization even on NoPE layers. `program_of(Flint)` remains -`None`; missing program, head128 sinks/SWA, NoPE normalization and 256/top6 bindings remain coverage -refusals. `Router.norm` and `RouterRule.norm` distinguish no normalization, the existing selected-sum -rule and Flint's selected-sum floor; the floor description does not supply a device implementation -or activated-value scaling. +The common needs walk emits q/k normalization even on NoPE layers. `program_of(Flint)` selects +`FlintBody`, whose coverage names the operations its loader actually binds: head128/group8 +sink attention, independent NoPE/NeoX and SWA masks, and the real floor router. +`Router.norm` and `RouterRule.norm` keep the old selected-sum rule distinct from Flint's floor. +The activated-value protection and explicit residual association are loaded FFN policies. Fault formatting uses `bloomery_device::fault`'s shared layout/decoder and the -`gpu::arch::step_order` registry. Flint's registry row uses neutral code order; -it does not describe a runnable step. +`gpu::arch::step_order` registry. The body delegates fault checks and captured execution to the +common chain runtime; the architecture registry's neutral order is not numerical evidence. Run `BLOOMERY_BOX_ENV="BLOOMERY_MODEL=/absolute/path/flint-header.gguf" BLOOMERY_BOX_LOCAL=1 just gate-flint-meta`. Set `BLOOMERY_ENV_FILE` to the local toolchain environment file if it is not `~/bloomery-env.sh`. @@ -67,25 +68,44 @@ remaining acceptance rows. This header gate is not an end-to-end model gate. ### Runtime handoff and host policy The design, common template support, tokenizer/four-EOG contract and reader (#676–#680) are merged. -The existing header recipe was exercised on the exact 5,470,528-byte prefix above: four reader tests and -the pinned-header test passed, still reporting the missing program and operation bindings. -No full weights or GPU were needed for that check. - -The maintainer [assigned the model port and offered card runs](https://github.com/midagedev/bloomery/issues/632#issuecomment-6352066886), -and named a forthcoming head128 sink/SWA attention, QK RMS and sigmoid256/top6 train. -Those complete contracts are not in this main snapshot or the inspected public heads of #726 (`1723485602c1`) -and #747 (`f883e84f8d2c`). Existing ring and sink-merge ingredients do not establish the combined attention -operation; `route_body` still lacks 256/top6 and its normalization-floor policy. -Obtain the train's fetchable ref before duplicating those primitives. - -The router description/selection contract now distinguishes `RouterNorm::None`, `SelectedSum` and -`SelectedSumFloor`. Flint requests `max(sum_selected, 2^-14)`; existing model descriptions and arithmetic -keep their rules. Compiled rows, including boolean `norm_arg` bindings, refuse the floor policy rather -than borrow an epsilon-added implementation. The absent-bias, 256/top6 and `Need::Program` refusals stay -until real device bindings are supplied. Reconcile the policy interface with the pending train before device work. -Then close shared IQ4_XS, selected card/tier down forms and six-slot F32 handoff in their existing common owners, -followed by paging/resident slots, reference provenance and the body/app/seat. -Each device slice needs its numerical/fault gates and the maintainer's offered hardware proof before admission. +The pinned-header consumer now also checks actual Program coverage and planning on the complete +local GGUF: 779 tensors, 49 layers, 2 dense/47 routed, 49 head norms, 35 rotations and 49 sinks. +This reads the header, not tensor contents; it does not prove CUDA execution. + +The maintainer [assigned the model port and offered card runs](https://github.com/midagedev/bloomery/issues/632#issuecomment-6352066886). +The later public QK-RMS, sink/SWA and sigmoid train supplies reusable primitives. The new common +`attention_h128` binding combines those with actual ring append/attention kernels; it is not an +identity-RoPE substitute or a padded-head adaptation of another family. + +The common `route_body::sigmoid_floor_256_6` binding now supplies Flint's fixed +`max(sum_selected, 2^-14)` rule. `RouterKernels::enqueue_router` and `enqueue_router_rows` +take `Option<&DeviceBuffer>` for a real optional selection-only bias. The common +`RouterEntry`/`RouteIn` consuming paths carry that option; `RouteBlock` allocates no zero-bias +buffer for this binding. Absent bias borrows the existing router-weight slice only as an unused +device ABI argument; no bias data is fabricated, copied or read. The floor policy is a compile-time +specialization of the same score, publication and top-k bodies, not a per-step normalization flag. + +Both metadata bindings are `RouterBody::SigmoidFloor` at exactly 256/top6: sigmoid, no shared +sigmoid gate, `SelectedSumFloor`, with or without selection bias. Bias changes only the ranking. +The six selected unbiased scores are summed in f64 slot order, narrowed to f32, divided by the +floored sum and then scaled. Equal selection keys choose larger expert ids. Nonfinite logits, +selection keys or resulting weights raise `FaultSite::Router`; an invalid selection key poisons +all six weights. Existing epsilon-added entries, launch ABIs and legacy norm-argument rules remain +unchanged. A legacy `RouterEntry` given no required bias returns a named state error. + +`gate-flint-router` now consumes the actual floor binding through `RouterEntry` for planted and +dense rows, both bias modes, active tiny-sum floors, zero sums, ties, nonfinite logits/bias/scale, +one-token/batch bit equivalence and eager/captured replay. Its independent host weight expression +uses the declared floor and rejects an epsilon-added or biased-weight implementation. +The two entries (`sigmoid_floor_256_6_router` and +`sigmoid_floor_256_6_router_pick`) need same-toolchain add-class PTX/JIT comparisons +and measured resource pins; compiling an entry or running a numerical gate is not +a resource census. No unmeasured pin is supplied. + +The loaded `FlintBody`, app session and serve seat now compose these common owners. +Program coverage is present; it must not be confused with whole-model numerical +acceptance. Paging/resident slots, reference provenance, eager/captured execution, +HTTP behavior and affected common/tier consumers retain their own acceptance gates. Legacy metadata booleans share `RouterNorm::from_legacy_bool`: false keeps scores and true uses the existing epsilon-added selected sum. Flint's distinct floor mapping stays in its reader. @@ -98,6 +118,9 @@ gated block selects a split gate/up and sizes the down's q8_1 scratch from the actual activation form. Its column walk and the shared wide-prompt owner use those same sites. IQ4_XS is not relabeled as a routed expert or requantized. +A compiled public `file_site` consumer on the pinned header refused all three +layer42 shared matrices before this change and accepts their IQ4_XS/Q8x128 +bindings after it. This reads no payload and proves neither loading nor arithmetic. `hw_iq4_shared_loads_and_runs_narrow_columns_and_wide_replay`, owned by `gate-gpu-ds41-lib`, covers real packed-weight loading, unchanged stored words, the ordinary `SharedExpert` layout, unequal matrix widths, one/eight-column block @@ -112,10 +135,14 @@ reconciles its existing Qwen3.8 projected-site consumer: `site::PROJECTED` inclu IQ4_XS, and selected dispatch uses the supplied stream. Projected attention/shared sites are admitted; the K=640 shared down and unsupported types/shapes remain refused. This reuses the existing program and kernel, not a new model implementation. +Flint's implemented Program separately binds the real ordinary dense/shared operations. -The typed GGUF test `a_site_type_comes_from_the_header_and_a_type_with_no_launch_is_refused` -reads an IQ4_XS shared site through `PROJECTED`, not merely `ALL`. It refuses IQ3_S, another -shape and a type outside the allowed list by name. It is metadata-only and creates no CUDA context. +At component source `b0fc05e5349330927ceb33ce6f30e584f25ecf4d`, the typed GGUF +`iq4_sites_keep_shape_and_launch_admission_separate` consumer passed1 case, +0 failed/ignored,430 filtered. Its successful arm uses `PROJECTED`, not merely +`ALL`; this checks actual file admission rather than copying a metadata registry. +The case is metadata-only and creates no CUDA context; oxide owns its linking. +It is neither fresh NVIDIA stream/launch execution nor a run of this integration. Pinned upstream `95a483fc242d00dec7987efe4a05e24b5f651c1f` subsequently adds the split gate/up path that materializes f32 SwiGLU for a down that reads f32. @@ -168,6 +195,11 @@ Host-only projections intentionally have no card totals: fixed reserves remain o Only a dynamic tier-leg descriptor requires the plan's selected card totals. The existing node paging budget regression exercises this distinction without a GPU, including OS reserve and constrained host room. +The current-main cutover keeps reserve-only draft holders, per-card budgets and each +family's activation contract. `LegTier` receives its activation converter at load; +selected `Program` arms replace the redundant family arm hook. Share-budget floors +use the same selected reserve calculation. + `tier_leg_batch_bytes` and the explicit-form `try_tier_leg_batch_bytes_of` return `Result` and construct candidate-selected descriptors. An empty tier selects no down form, matching `LegTier`'s actual allocation instead of @@ -183,6 +215,11 @@ Malformed geometry and overflow are refused rather than panicking; a dense Qwen3 request for an expert tier is a named metadata refusal, not zero expert geometry. Separate tile/route-based fixed-reserve paths remain unchanged. +`TierLegScratch::input_scales` names an optional physical f32 scale per chunk +slot. Its checked byte formula includes `4 * chunk * n_used` when enabled, +including an empty selected-down union; legacy descriptors leave it disabled. +The same descriptor prices the loader's scale allocation without reserve padding. + The planner's selected scratch calculation has one union/byte rule. Fill prepares per-layer forms once for dynamic tier-leg cards, including multiple forms in one layer, and reuses them with its cached footprint instead of rescanning tensors for @@ -194,10 +231,10 @@ prompt reserve. With byte granules (no reusable dense padding), the exact dense- budget leaves no room for expert weights. The real mixed-wrapper hardware case also checks allocation of a configured empty tier. -Historical exact-source proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: +Historical component proof at `3071a9f0d8ed4773692e0c040dece222b90f6bec`: GPU library418 passed plus10 held passed, verbs1 passed, and all1657 canonical entries identical to main15778e2c (4bins/8carriers, non-JIT rows equal). -The repair at `c26025b1fb9fad1af399401d27cf709005736ba0` then executed the extended +The component repair at `c26025b1fb9fad1af399401d27cf709005736ba0` then executed the extended real CUDA mixed/empty-tier consumer:1 passed,0 failed,0 ignored,427 filtered,24.18s. Its fresh4bin/8carrier canonical comparison again found all1657entries identical to main15778e2c, with non-JIT rows equal. Native formatting/Clippy passed (warning3=ratchet3); model557 and @@ -213,32 +250,27 @@ chain23, GGUF67, model557 and placement202 (five model cases ignored,0 filtered) An exact-source7e37ef9a canonical comparison against main15778e2c found all1657 entries identical across4bins/8carriers and non-JIT rows equal; it did not compare driver-JIT resource rows. The older full-library counts above remain historical, -not a new full-library run of the review repairs. +not a new full-library run of the review repairs or a run of this integration. -The component subsequently merges pinned upstream `cb8d92ebddd1ce1a325a15450c06a01f57ecf88c`. -That reconciliation retains upstream's GLM mixer-form consumers alongside this component's -checked tier-byte API; the explicit-form API has no input-scale flag. The GPU results -above belong to their named historical sources, not the merged source. - -At merged source `c38d603ca1d33c6c2bede63aba102199715fb9ef`, native formatting and -all-target Clippy passed, as did chain23, GGUF67, model559 and placement202 +At component merged source `c38d603ca1d33c6c2bede63aba102199715fb9ef`, native formatting +and all-target Clippy passed, as did chain23, GGUF67, model559 and placement202 (five model cases ignored, zero failed/filtered). The actual oxide-linked `a_dense_header_plans_through_qwen35_on` frontend consumer passed1 case, 0 failed/ignored,6 filtered, including the named expert-tier refusal. -This is metadata-only host execution, not fresh NVIDIA execution of the merged source -or a whole-model numerical result. +This is metadata-only host execution, not fresh NVIDIA execution or a run of this integration. + +The host-only planner covers one-byte boundaries before the first resident layer and another +down form, plus descriptor overflow refusal. Neither this nor the component CUDA proof is a +whole-model oracle or every affected model/residency gate; canonical equality does not waive +changed host/allocation consumers. The integration's body/app/seat consume the selected forms. -The host-only public planner consumer covers one-byte boundaries before the first resident layer -and before adding another down form, plus descriptor overflow refusal. These and the CUDA consumer -are not a whole-model oracle or every affected model/residency gate; canonical equality does not -waive changed host/allocation consumers. Flint's `Program` admission and body/app/seat remain absent. ### Six-slot F32 tier handoff implementation `HandoffKernels::enqueue_handoff_tier` now selects a genuine six-slot F32 entry beside the existing four- and eight-slot entries. Both host and tier layouts must have the same supported slot count and activation width; compressed tier images remain refused. The entry uses the existing copy/place/fault helpers and the existing -subsequent go/publication protocol; no Flint router or Program is registered. +subsequent go/publication protocol; the integration's configured routed block consumes it. `hw_f32_tier_handoffs_conserve_consumed_routes`, owned by `gate-gpu-ds41-lib`, consumes the four-, six- and eight-slot handoffs' weights and emitted @@ -259,6 +291,7 @@ observed spill/JIT-local measurements. Offline no-JIT scans cannot supply that c + ## 1. Sources and observed artifact - Official [config](https://huggingface.co/RecursiveSI/flint/raw/2a247be0c0aec895ccb0ac7ca92229ec6df37288/config.json), @@ -280,7 +313,10 @@ alignment padding, at byte **5,470,528**. Bytes after that boundary were ignored `fb0cc4b520f927f0b679193397ee0c36ca252426e7881283c5aa6de3e44a3a2b`. The existing Python header reader and Rust `gguf::inventory_of` agree on **779 tensors** and **25,049,218,176 encoded tensor bytes**. Shape products total **39,785,287,456 parameters** (derived). -No full model was downloaded or executed. +That initial design inspection fetched only the prefix. A later local CPU oracle run used the +complete 25,054,688,704-byte GGUF (SHA-256 +`63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f`). +Mainline CPU completion is reference evidence, not execution of this CUDA body. The official repository says Apache-2.0; this exact GGUF header says `other` / `flint-preview`. The config/header shapes and the byte-identical 2,328-byte chat templates establish those contracts, @@ -351,13 +387,12 @@ source of the gathered weights. The shared SwiGLU output is added without a shar Three numerical obligations must stay explicit: -1. Current [common router normalization][route-core] uses `sum + 1e-20`, not the reference's floor. +1. Legacy [common router normalization][route-core] uses `sum + 1e-20`, not the reference's floor. This is a semantic difference for tiny scores, not a width-instance problem or a measured GPU - failure. The existing `Router`/`RouterRule` contract now carries the explicit policy, and the - reader requests `SelectedSumFloor` for Flint. Selection refuses that policy on every current - router row; `norm_arg` serves only the two legacy policies. The device floor binding is still - missing. Add it in the common operation without changing the global divisor, existing model - arithmetic, or hiding the difference in a Flint-only wrapper. + failure. `Router`/`RouterRule` carries the explicit policy; the reader requests + `SelectedSumFloor` and the common `sigmoid_floor_256_6` binding implements it. Legacy instances + keep their exact arithmetic and `norm_arg` serves only the two legacy policies. The floor + gate checks active tiny sums independently; GPU numerical and resource evidence is still needed. 2. Mainline [rescales Flint's activated expert input before down projection][ref-down] with `build_amax_src1_f16`, then restores its scale before router weighting. This protects F16/Q8 headroom around a maximum magnitude of 1024; it is not an architectural activation clamp. @@ -383,17 +418,17 @@ requires bias and derives width from a shared-count convention. Invalid mask typ norms/sinks/shared matrices, an out-of-range block, impossible KV grouping, zero dimensions, nonfinite epsilon/base, overflow and malformed optional bias all need named refusals. -At this snapshot there is no Flint architecture reader or selected program. A reader-only landing -must keep `program_of=None` and the existing **`Need::Program`** refusal in [coverage][coverage]. -Do not claim runtime support by adding the name to an enum or relaxing a shape/type row. +The original reader-only landing retained `program_of=None` and `Need::Program` refusals. +The integration now has actual operation consumers and Program admission; adding a name or +loosening a type row alone would still not establish runtime support. ### Concrete common prerequisites -| ID | Current limitation | Required common change before an executable attach | +| ID | Original gap (design snapshot above) | Required common change / acceptance contract | |---|---|---| | G1 | NoPE normalization-need generation is present; the executable head128 norm-only binding is missing | Preserve the emitted normalization **or** rotation need; bind head128 normalization with `mode=None` truthfully. No identity-RoPE table. | | G2 | Existing head128/group8 attention lacks the complete sink/SWA contract; MiMo's window pass is K192/V128 | Complete the described head128 operation in `rope_neox` / `flash_gqa`, including genuine NoPE append, sink merge, window/ring positions, prompt and slot paths. Never pad heads to 192 or copy MiMo's body. | -| G3 | The host description/selection policy is explicit; sigmoid256/top6 and its device floor binding are absent, and higher-level `shared_routed` admission requires bias | Add the real instance/floor operation; bind absent bias through `RouteLayer.bias=None`, which the lower `RouteBlock` already supports. Price/load the BF16 router through its supported F32 widening. | +| G3 | Real common sigmoid256/top6 floor operation and both optional-bias metadata bindings are implemented; whole-program binding remains separate | Attach `sigmoid_floor_256_6` through `RouterEntry`, with `RouteLayer.bias=None` retaining genuine absence and no zero buffer. Price/load the BF16 router through its supported F32 widening. Run `gate-flint-router` and obtain genuine new-entry JIT pins on NVIDIA hardware. | | G4 | Layer42 shared IQ4_XS has routed/GEMM operations but no ordinary dense site, and `BlockPlan` rejects it | Extend the **common** dense/gated block's real layout, load, launch, scratch and prompt binding. Keep the role `SharedExpert`; do not relabel it as routed or silently requantize it. | | G5 | The common card wrapper uses Q8Act-only buffers and `LegStacks::of`; Q6_K/Q8_0 down do not both pass that path | Carry selected arms and actual down forms through `CardExperts`, `LegBufs::for_downs`, loader, planner, residency and tier. Existing Q6_K/Q8_0 arithmetic is not the missing piece. | | G6 | Common tier accumulation supports six slots, but F32 `enqueue_handoff_tier` accepts eight | Add the six-slot F32 handoff instance in the existing owner; preserve the eight-slot entry. A DeepSeek Q8_1 handoff is not an interchangeable input contract. | @@ -647,6 +682,277 @@ The Python reader intentionally skips array values. Their recorded values came f `gguf::inventory_of(...).meta`, not `None` values treated as absent GGUF facts. The recipe above revalidates the **whole header hash**, including those arrays, without a second parser. +## 10. Single-card NVIDIA acceptance and remaining numerical boundary + +The runtime at `313b9f0f88c90fcfae2365a4af53cd077762eb7c` was exercised on one RTX 3090, +cc8.6, driver 595.91.07 (CUDA API 13.2), with nightly-2026-08-28, cuda-oxide `87bc9518`, +LLVM 21.1.8 and CUDA toolkit 13.3.0 / ptxas 13.3.33. The complete 25,054,688,704-byte +GGUF was downloaded separately on that runner and its full SHA-256 matched +`63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f`. + +- `just gate-gpu-flint-attn` and `just gate-gpu-flint-router`: exit 0, including loaded + NoPE/NeoX ring consumers, floor/optional-bias routing, capture/replay and named faults. +- `just gate-gpu-flint-lib`: exit 0; the full-file capture/slots/reset/ring-boundary test + ran with **1 passed, 0 failed, 0 ignored**. Its bitwise consistency is not a mainline oracle. +- The real HTTP seat loaded 49 layers and two resident slots, exposed the original template, + generated English and Korean turns to EOG 150003, split reasoning, parsed a required + `get_weather` call with `{"location":"Seoul"}`, and preserved messages under concurrent + requests. The four stop spellings tokenized to the four expected IDs; natural emission of + all four stop IDs was not tested. +- A full `generate_ds41` scan and the pre-pin spill gate JITed all 436 entries without a + resource cache. The only ten violations were the new unpinned entries, each measured + `spill=0, jit_local=0`; the 26 vision entries passed. These observations supply only the + ten `sm_86`/cc8.6 rows. The three cc12.1 tables still need thirty genuine measurements. + +### Independent greedy comparison remains partly red + +The English raw continuation exactly matched pinned mainline CPU +`a37cc46104eb50900d5ab4a9ba02a0abdac039db`. The Korean continuation did not: the first +different generated ID is index 74, after the same 22 prompt IDs and 74 generated IDs +(next absolute position 96). Mainline chooses 26007 (`라`); Bloomery chooses 29263 (` 정`). +Both eventually answer `4`, which does not turn the comparison green. + +A standalone consumer linked to the preserved mainline library replayed that original +prompt-then-single-token trajectory with zero preceding raw-argmax mismatches. Its complete +150,144-logit rows at context 1024 and 2048 were bit-identical. At the divergence: + +| Observation | Margin `logit(26007) - logit(29263)` | Selected ID | +|---|---:|---:| +| Mainline CPU, original trajectory | +0.608115554 | 26007 | +| Bloomery probability diagnostic, original trajectory | -0.018128157 | 29263 | +| Bloomery, same 96 IDs supplied as one prefill | +0.361473739 | 26007 | + +The GPU margins are differences of reported pre-sampling log probabilities; the diagnostic +reproduced the original first 75 generated IDs. Probability collection reads each row and +uses plain per-token stepping, rather than establishing equality of every execution mode. + +This is not an exact tie or a demonstrated context-capacity defect. The common FFN owner +explicitly uses F32-activation Q8_0 GEMV for narrow/decode rows and quantized-activation +GEMM for wide prompt units (`shared_routed.rs`, `GATED_FROM=9`). Mainline also quantizes +the output head's activations and BF16 router operands differently. These are source-backed +numerical differences, not a proven localization of the Korean flip. No precision mode, +compiler pin, tie rule, tolerance or acceptance criterion was changed to force matching text. +Comparable follow-up traces must retain the original prompt/decode trajectory and can use +the existing 49-layer `Body::taps` and mainline `l_out` callbacks. + +Keep the integration Draft. Korean numerical/oracle acceptance, cc12.1 resource rows, +affected common-owner coverage beyond the exercised consumers, two-card operation, dynamic +residency/NVMe, streamed HTTP and spontaneous tool selection are not established by this run. + +## 11. Current-main integration and trajectory instruments + +The first linear integration candidate, `4dadee0195fb652608be59ffecc033a4643ad1eb`, +starts at main `43073091be21e48cda40d79fe6dae1dfcfd643d5`. The follow-up replay +starts at main `af74ea46b7f3e47692c2afae09eb64fdddf71461`. At that stage neither replay +replaced the published #799 head; section 12 describes the later history-preserving cutover. +Fixed-source comparisons keep the original production and candidate images separate, and +receipts name each image. Section 10 is not acceptance of either replay or the later cutover. +The placement integration keeps both the cached upload `Footprint` and candidate-selected +tier scratch/reserves. No compiler pin, numerical band or model admission was widened. + +### The common union must retain selected formats and recount its allocation + +`model::placement::union_plan` now owns the shared one-stage/one-tier reference transform. +It retains the actual selected card format, refuses incompatible formats, preserves host +assignments and KV, and recomputes expert bytes, upload-heap rounding, selected reserves and +actual-stage headroom. The artificial budget that split the experts is not the reference's +physical capacity. The old shared helper is removed and its consumers use the common owner. +The standalone DS41 and Qwen3.8 transforms have different contracts and remain separate. + +A public host-only consumer over the complete pinned GGUF reproduced the former generic +transform's `NoCardFormat` refusal on `blk.2.ffn_gate_exps.weight` (`IQ3_S`). The corrected +transform preserves the program-selected format rather than admitting IQ3_S generically. +With the workstation's 3090 stage/A6000 tier figures, 1024 context and an 8 GiB per-card +planning budget, its union contains 5,197 experts and 9,959,514,112 expert bytes, equal to +the assigned rows, with no plan violations. This is real-file planning, not device loading. +The pure regression also covers mixed selected down forms, heap/reserve recounting, +unchanged generic refusal, conflicting formats and insufficient union capacity. + +The common mixed card/tier hardware consumer stages its fixture through `Weights::load_rows` +using `accepts::routed_rule(FAMILY.program)` and the shared segment buffer arithmetic. +Neither `Weights::load_where` nor the generic `routed_row` admits selected-only IQ3_S: +assigning a routed role alone is insufficient. The fixture keeps all mixed-format +step/pass/batch/tier bit comparisons and byte conservation, without widening generic +admission. The obsolete Solar2 Q6_K-down rejection is removed: the current selected owner +already admits that arm. Mismatched gate/up rejection remains exercised. + +### CPU and GPU use one forced-ID diagnostic artifact layout + +`tools/ref/flint_trace.cpp` and the `bloomery-gpu-flint` example `trace_flint` consume exactly +96 whitespace-separated decimal IDs. The original HTTP `Slot::close_call` in +`crates/serve/src/genloop.rs` prefills the first 21 prompt IDs, then calls `next` for the +22nd ID before the 74 forced continuation IDs. Both consumers reproduce that boundary. +They do not retokenize, sample, or stop at an input EOG. The 75 observations save processed +positions 21 through 95, predicting positions 22 through 96; the prefix itself is not an +observation. `--prompt 22` names the complete prompt, not a 22-ID batched submission. + +The format is `flint-trajectory-v2`, not a reference-set family or accuracy gate. +The former v1 consumer submitted all 22 prompt IDs as one batch: its controls remain +historical evidence but do not reproduce the actual HTTP prompt/step boundary. Do not +relabel v1 arrays or compare them as if the submission contracts matched. + +- `tokens.ids` and `steps.tsv`: exact inputs and each call's processed/predicted position + and raw argmax. +- `logits.f32`: little-endian F32 `[75][150144]` in every arm. +- With taps: `layers.f32` `[75][49][2048]` and `norm.f32` `[75][2048]`. + Untapped arms explicitly mark those tensors absent. +- CPU `tensors.tsv` records actual node types, dimensions, strides and selected rows. + GPU artifacts retain the actual expert assignments, loaded slots, device/plan facts, + graph nodes and completion counters. +- A new output directory is required. `MANIFEST.tsv` is published only after successful + capture; a partial run cannot masquerade as complete. External receipts identify the + model, source/diagnostic overlay, toolchain, libraries and binary hashes. + +Build the CPU consumer against the preserved mainline revision +`a37cc46104eb50900d5ab4a9ba02a0abdac039db`, with shell variables naming its source, +built library directory and the new diagnostic executable: + +```sh +c++ -std=c++17 -O2 -I"$LLAMA_SRC/include" -I"$LLAMA_SRC/ggml/include" \ + tools/ref/flint_trace.cpp -L"$LLAMA_LIB" -Wl,-rpath,"$LLAMA_LIB" \ + -lllama -lggml -lggml-cpu -lggml-base -o "$CPU_TRACE" +CUDA_VISIBLE_DEVICES= "$CPU_TRACE" -m "$MODEL" --ids "$IDS" \ + --prompt 22 --ctx 1024 --out "$CPU_OUT" +``` + +The preserved **v1** CPU callback arm was actually run on the complete model: 75 calls, 3,675 layer vectors, +zero preceding argmax mismatches, and final argmax 26007. Its complete final-logit SHA-256 +is `083c8aa95aab025c0183c2ee3c4788642499533d5a2ee4e280c1125302c23bf9`, bit-identical +to the preserved callback-free original row. The model was rehashed in this round. +The separate untapped control also ran: all 75 complete 150,144-logit rows are +bit-identical to the callback arm, whole-file SHA-256 +`7f2e476bd7fbf33faf6548eedadbb89b1844c50e0a55d7fc8aa25b297fbaff4a`. +These are results of the first candidate's CPU consumer, not new GPU results. + +The corrected **v2** CPU callback and callback-free consumers also ran on the complete +model. Their 75 full-logit rows are bit-identical, SHA-256 +`4ef35ee7390341e640a6d38c39a010643f51a853fe743b29a5da989c7b8357b5`. +At next position96 the argmax is26007 and its margin over29263 is+0.238318443. +V1/v2 CPU arrays are not bit-identical; call16 has a different argmax. This is evidence +that the submission boundary matters, not a GPU fix or permission to waive the oracle. + +The GPU build uses the pinned oxide toolchain. Actual capture runs belong inside an +authorized runner configuration with its normal card locks and bounds; `TRACE` names +the compiled example in that checkout's target directory: + +```sh +cargo oxide build --arch sm_86 -- -p bloomery-gpu-flint --release --example trace_flint +bash tools/gpu-gate.sh --executable "$TRACE" -m "$MODEL" --ids "$IDS" \ + --prompt 22 --ctx 1024 --slots 2 --mode graph --out "$GPU_OUT" +``` + +Use separate new output directories for all four mode/instrumentation combinations: +Eager/untapped, Eager/tapped, Graph/untapped and Graph/tapped (`--taps`). +Check full-logit neutrality within each mode as well as Eager/Graph agreement. +Arming taps forces Eager; the consumer explicitly restores the requested mode afterwards, +and Graph decode must actually capture. The instrument admits only the existing +card-budget lever; do not assume other inherited levers were applied. +Context 1024 is per slot, two slots total 2048, matching the original HTTP setup. +Compare the actual per-layer placement, not just model identity or aggregate counts. +The diagnostic's all-false `HostCfg` matches the original `r8=off` and `HOST_LOCK=0` +settings, but page population/discard differ from the original HTTP defaults. Record that +difference; it is neither an established numerical cause nor permission to skip direct-B +boundary reproduction. `norm.f32` is the weighted RMS output before head activation +quantization, not an RMS-only row or quantized activation bytes. + +Diagnose the original `313b9f0` production source with the separately recorded example +overlay first; then bridge the same controls onto the integration candidate. A new binary +on a larger card is not automatically the original 3090 trajectory. A control mismatch +stops causal attribution. If layer outputs agree but the final norm/head does not, isolate +that operation on matching inputs; intended quantization is neither a demonstrated bug +nor automatic permission to waive the Korean flip. + +### Dedicated real two-card consumer + +`gate-gpu-flint-twocard` runs +`two_card_tests::hw_two_card_full_file_union_capture_publication_fault_reset`. +It requires the complete pinned GGUF in inner `BLOOMERY_MODEL`, two distinct visible +devices, and actual expert assignments to both cards at every routed layer. It compares +batched prompt and changed-input decode logits/layer rows with the same plan's one-card +union, checks ranked six-slot publication and stage/tier capture/completion, then raises +a real quantizer fault on the tier GPU while the stage is clean and checks propagation, +poison, refusal, reset and refeed. No unsafe fault-injection API is added. + +The one-card library recipe keeps its exact original test filter. Both full-file recipes +refuse the fixture tier. The two-card recipe takes the existing `both`/big-load runner +path and has a derived host-resource ceiling, not a measured RSS claim. It does not prove +dynamic residency/NVMe, loss/deadline recovery or a Korean oracle result. + +Require fresh NVIDIA evidence tied to each candidate's actual source, model, placement +and executed hardware case; compiled/ignored tests are not such evidence. Preserve +existing workloads unless their owner authorizes a separate window. Physical cc12.1 +resource values must be measured, never inferred from an sm_86 build or cc8.6 run. + +## 12. Shared-contract integration and same-input head probe — 2026-10-26 + +The integrated contracts started from main +`15778e2cb2dd7af428d2dd14259e6b0ea8ca6b89` and were reconciled with pinned main +`cb8d92ebddd1ce1a325a15450c06a01f57ecf88c`'s projection and stream interfaces. +They retain Rust1.99.1 and cuda-oxide9df9e7ce / repository backend patch31ab6a1d3f58, +not a model-specific compiler override. Older nightly/87bc arrays remain evidence +of their own source and toolchain only. + +Current-main's prequantized leg input and metadata-only `Common` remain separate. +Both ordinary and prequantized entry points retain the loaded `AmaxF16` protection state. +The already-activated card leg normalizes with `rule=None`: its second device ABI operand +is unread, so it borrows the distinct existing down buffer rather than allocating or copying +a fake input. A hardware regression compares ordinary input with prequantized captured replay, +including changed amplitudes, Direct/AmaxF16, full/partial batches and skipped/tail canaries. +Historical integration source `082cf7ffac3bb5b5d574b489062732735985feeb` +passed native all-target Clippy and formatting, with three warning lines against ratchet three. +Its host library run passed chain23, GGUF67, placement206 and model563 (five model tests +ignored), with zero filtered tests. Common-name and unsafe ownership checks also passed. +On RTX3090 cc8.6 / driver595.91.07, its protected-prequantized, mixed/empty-tier and +complete-GGUF consumers each passed one hardware case, zero failed/ignored. Its attention +and router executables also exited0. These are source-specific results, not execution +of later ownership/API repairs or whole-model oracle acceptance. + +The current-main code reconciliation at `49b11b36d4a1de2e140010c5a2d2c9afc3e2671a` +passed native formatting/all-target Clippy (zero warning lines against ratchet three), +model565 and placement206, with five model cases ignored and zero filtered. +The common-name check found1337 lines against the existing1400-line allowance; +all39 unsafe ownership rows matched their pins. These are host checks, not NVIDIA execution. +The merge retains IQ4_XS's selected `_sel` dispatch on the supplied stream and its +q8_1 launch accounting. The Qwen3.8 admission fixture accepts its projected sites while +retaining the K=640 shared-down refusal. IQ4 activation bands were not relaxed, +and the removed source-text wiring guard was not restored. + +Every recipe that builds the shared `bloomery-serve` binary carries its required-feature +union, including `flint`; this also applies to newly merged consumers such as +`gate-gpu-llama-serve`. Recipe validation and actual model-fence opening must agree +before a batch can schedule that consumer. This configuration check is not a GPU serve run. + +### Measured head/upstream separation, not a precision fix + +A throwaway probe used the preserved mainline CPU backend +`a37cc46104eb50900d5ab4a9ba02a0abdac039db`, four threads, and the original complete +Q8_0 output matrix (2048 × 150144). It evaluated both CPU and GPU final-normalization +vectors from the matched-boundary v2 trajectory. Its CPU-input controls reproduce the +original complete 150144-logit rows bit for bit at both calls; this validates the cross-input +comparison rather than a dequantized/reduced-matrix approximation. + +Here `HC`/`HG` are the CPU/GPU head implementations and `xC`/`xG` their saved norm inputs. +Each cell is the first named token's logit minus the second's: + +| Predicted position / token pair | HC(xC) | HC(xG) | HG(xG) | HG(xG) − HC(xG) | HC(xG) − HC(xC) | +|---|---:|---:|---:|---:|---:| +| 38 / 86671 − 55591 | +0.164879680 | −0.166242123 | −0.142640829 | +0.023601294 | −0.331121802 | +| 96 / 26007 − 29263 | +0.238318443 | +0.005065918 | −0.018128157 | −0.023194075 | −0.233252525 | + +At position96, the CPU head on the GPU norm selects26007, whereas the actual GPU head +selects29263. At position38 the CPU head on the GPU norm still selects55591. A head-only +change therefore cannot resolve the whole trajectory. The columns separate head-implementation +and upstream contributions; they do not identify the first upstream operation or attribute all +head difference to one quantization step. No production precision, tolerance, tie rule or oracle +criterion was changed, and no existing GPU arm was replayed for this CPU-only probe. + +These source-specific cross-input results are not a run of a later integration. +Whole-model acceptance still requires exact-source runtime proof, disposition of +the Korean numerical/oracle mismatch, genuine physical cc12.1 resource measurements +and remaining affected common/residency/tier/NVMe coverage. Component proof is not +whole-model admission, ROCm support or a performance result. + + [needs]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/models/src/need.rs#L505-L531 [stems]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/model/src/arch/stems.rs#L61-L125 [coverage]: https://github.com/midagedev/bloomery/blob/e53cabebbb9b16249c81b7c26a026da1670f7d9d/crates/model/src/arch/coverage.rs#L1654-L1770 diff --git a/docs/serving.md b/docs/serving.md index 4e5238d03..753f23dca 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -699,7 +699,7 @@ In progress: serving on N cards for every model; Clef from the IQ and Q2_K files ## EmbeddingGemma 2 images A side seat serves the `gemma-embedding2` encoder beside a generative model. Build `bloomery-serve` with -`--features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,vision` through `cargo oxide`, and add these flags +`--features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,vision` through `cargo oxide`, and add these flags to the chat model's command: ```text diff --git a/justfile b/justfile index 3a86a046d..8f2a95fbc 100644 --- a/justfile +++ b/justfile @@ -32,7 +32,7 @@ default: # 빠른 루프: 타입 검사만, 커널은 안 만든다. 의존을 고친 뒤 cargo가 박스 쪽 Cargo.lock을 고쳐 쓰면 lock-back.sh가 # 그것을 이 트리로 가져온다 — box.sh는 한 방향으로만 싣는다. check: - ./tools/box.sh 'cargo check --workspace --all-targets --features gpu,bloomery-metal/pipeline-scan,bloomery-metal/encode-trace,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' + ./tools/box.sh 'cargo check --workspace --all-targets --features gpu,bloomery-metal/pipeline-scan,bloomery-metal/encode-trace,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/flint,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' ./tools/lock-back.sh # check와 같은 이유로 `--features gpu`. 이 피처를 켠 것이 기준 계기다(R26). V4.1 op 게이트의 피처 @@ -40,14 +40,14 @@ check: # V4.1 커널의 컴파일러 결함은 여기가 아니라 op 게이트 빌드에서 드러난다. # lint. 에러 0이 계약이고 경고 수는 RESULTS/AGENTS에 적힌 기준선과 비교한다. lint: - ./tools/box.sh 'cargo clippy --workspace --all-targets --features gpu,bloomery-metal/pipeline-scan,bloomery-metal/encode-trace,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' + ./tools/box.sh 'cargo clippy --workspace --all-targets --features gpu,bloomery-metal/pipeline-scan,bloomery-metal/encode-trace,bloomery-gpu-gates/deepseek41,bloomery-gpu-gates/vision,bloomery-gpu-gates/glm5next,bloomery-gpu-gates/mimo2,bloomery-gpu-gates/solar2,bloomery-gpu-gates/minimaxm3,bloomery-gpu-gates/flint,bloomery-gpu-gates/gemma-embedding2,bloomery-gpu-gates/gemma4a,bloomery-gpu-gates/clef,bloomery-gpu-gates/hf,bloomery-gpu-gates/glm5next-hf,bloomery-gpu-glm5next/hf,bloomery-decision/safetensors' # The build without `vision` of the bins that refuse an image request by name — serve_seats/ds41.rs:444 and glm.rs:825 (--mmproj), # decide.rs:545 (the Qwen3.5 seat's encoder), gate_glm5next_e2e.rs:5146 (--only media-seat). One bin takes one feature set (tools/recipes.py's # bin check), and every recipe that builds these bins builds them with `vision`, so only this check compiles the # `cfg(not(feature = "vision"))` branches. A cargo check writes no executable, so it replaces no bin another lane runs. mac-combos runs this shape. check-no-vision: - ./tools/box.sh 'cargo check -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41 --bin gate_glm5next_e2e' + ./tools/box.sh 'cargo check -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41 --bin gate_glm5next_e2e' # fmt는 맥에서 돈다. box.sh의 rsync가 단방향이라 박스에서 포맷하면 결과가 돌아오지 # 않고 다음 명령에 덮여 사라진다(2026-09-19에 그렇게 한 번 날렸다). cargo fmt는 컴파일을 @@ -433,7 +433,7 @@ gate-gpu-musespark-prefill: [group('solo')] [group('v41-load')] gate-gpu-musespark-twocard: - BLOOMERY_MODEL=muse-spark BLOOMERY_CARD=both ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_musespark_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_musespark_prefill --two-card && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_musespark_serve --dir $D' + BLOOMERY_MODEL=muse-spark BLOOMERY_CARD=both ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_musespark_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_musespark_prefill --two-card && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_musespark_serve --dir $D' # Muse Spark Open's load gate (gate_musespark_load's doc): the body's placed load of the gate plan proven against its plan — # every expert in one place and the stack bytes the file's, each segment on the card resident at the plan's buffers and nothing @@ -452,7 +452,7 @@ gate-gpu-musespark-load: [group('solo-real')] [group('v41-load')] gate-gpu-musespark-serve: - BLOOMERY_MODEL=muse-spark ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_musespark_serve --dir $D' + BLOOMERY_MODEL=muse-spark ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_musespark_serve && D=target/musespark-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_musespark_serve --dir $D' # Solar-Open2-250B end-to-end gate (gate_solar2_e2e's doc): the 36 KDA layers (beta = 2 sigmoid, softplus decay) and 12 NoPE gated GQA @@ -505,7 +505,7 @@ gate-gpu-solar2-twocard: [group('solo-real')] [group('v41-load')] gate-gpu-solar2-serve: - BLOOMERY_MODEL=solar-open2 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_solar2_serve && D=target/solar2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=600 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_solar2_serve --dir $D' + BLOOMERY_MODEL=solar-open2 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_solar2_serve && D=target/solar2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=600 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_solar2_serve --dir $D' # The nemotron seat of bloomery-serve on Nemotron 3 Nano 4B Q4_K_M (gate_nemotron_serve's doc): `--plan` with no flag names its # default context (the ladder's rung, held by the ctx line and the rung above refused) and exits before any load, `--parallel 2` is @@ -514,14 +514,14 @@ gate-gpu-solar2-serve: # server's template renders), serves that turn on /v1/chat/completions (usage counts its ids), resends an edited prompt as a # fresh run that keeps nothing and an extended one that keeps the held positions. One card. gate-gpu-nemotron-serve: - BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + BLOOMERY_MODEL=nemotron_h ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' # The same seat gate on the moe body (gate_nemotron_serve's doc, every clause one path over both # bodies): the Bolt fixture under the fixture tier (nemotron_h_moe, the fixture's four layers, any # card), the arch word and the layer count read from the file, the session's prompt call the body's # batched `GpuModel::prefill` (several ids; one id is a step). One card. gate-gpu-nemotron-h-moe-serve: - BLOOMERY_TIER=fixture BLOOMERY_MODEL=nemotron_h_moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-h-moe-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' + BLOOMERY_TIER=fixture BLOOMERY_MODEL=nemotron_h_moe ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_nemotron_serve && D=target/nemotron-h-moe-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_nemotron_serve --dir $D' # GLM-5.3-Flash 라우터(288개 중 8개, sigmoid, ×2.5 — `router::glm5next`)를 모델 없이 합성 입력으로 호스트 규칙과 맞춘다. # 규칙마다 절이 하나다: logits·ids·weights·batch는 비트, scores는 sigmoid 밴드, ties·guard·fault·graph는 명시한 값. @@ -1501,7 +1501,7 @@ gate-gpu-qwen35moe-mtp: # target). The card is the A6000: the DSpark draft's load holds the whole target on it, and Qwen3.6's # 29 GB does not fit the 24 GB card (the seat then refuses the draft by name). gate-gpu-qwen3-dspark: - BLOOMERY_MODEL=qwen35moe BLOOMERY_CARD=a6000 ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen3-dspark && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_dspark && D=target/qwen3-dspark-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && MINI=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/minicpmv47.sh) && test -n "$MINI" && Q30=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen3moe.sh) && test -n "$Q30" && bash tools/gpu-gate.sh gate_qwen3_dspark --qwen "$Q36" --qwen-draft /models/Qwen3.6-35B-A3B/dspark-redhat --sage /models/Sage-A1.5/Sage-A1.5-Q4_K_M.gguf --ornith /models/Ornith-1.5-35B-A3B/Ornith-1.5-35B-Q4_K_M.gguf --minicpm "$MINI" --minicpm-draft /models/MiniCPM-V-4.7-35B-A3B/dspark --qwen3moe "$Q30" --dir $D' + BLOOMERY_MODEL=qwen35moe BLOOMERY_CARD=a6000 ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen3-dspark && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_dspark && D=target/qwen3-dspark-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && MINI=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/minicpmv47.sh) && test -n "$MINI" && Q30=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen3moe.sh) && test -n "$Q30" && bash tools/gpu-gate.sh gate_qwen3_dspark --qwen "$Q36" --qwen-draft /models/Qwen3.6-35B-A3B/dspark-redhat --sage /models/Sage-A1.5/Sage-A1.5-Q4_K_M.gguf --ornith /models/Ornith-1.5-35B-A3B/Ornith-1.5-35B-Q4_K_M.gguf --minicpm "$MINI" --minicpm-draft /models/MiniCPM-V-4.7-35B-A3B/dspark --qwen3moe "$Q30" --dir $D' # Qwen3.6-35B-A3B's DSpark draft, RedHatAI's speculator (an HF directory on the box, `--draft DIR` names another): the dense layers on the card against a host f64 computation of the same layers from the file's bf16 # weights (a block of 8 rows over 3 and over 2,100 context rows, the window 2,048), the captured pass replaying to the @@ -1553,7 +1553,7 @@ gate-gpu-llama-dense: # through a scripted engine behind the same template, and answers /completion at temperature 0 with the ids generate_qwen3moe --last-step prints for the # same ids (run once the server has stopped, the file does not fit a card twice). One card; the gate lock. gate-gpu-llama-serve: - BLOOMERY_MODEL=nanbeige43 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_llama_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/llama-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_llama_serve --dir $D' + BLOOMERY_MODEL=nanbeige43 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_llama_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/llama-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_llama_serve --dir $D' # Body35's image rows on Clef-Flash at bartowski's Q8_0 (9.5 GB, either card), against the clefvis sets (set B's tower rows # as the spans' rows; set C's three prompts, their CPU twins and the C'' runs): the unit rows and the text rows a media call @@ -1595,7 +1595,7 @@ gate-gpu-qwen35moe-media: [group('solo')] [group('v41-load')] gate-gpu-qwen38-media: - BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen38-media && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen38_media && D=target/qwen38-media-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=2400 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen38_media --dir $D' + BLOOMERY_MODEL=qwen4exp ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen38-media && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen38_media && D=target/qwen38-media-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=2400 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen38_media --dir $D' # The decide seat of bloomery-serve (gate_clef_serve's doc): Clef-Flash at bartowski's Q5_K_M (6.9 GB, the # `qwen35` file of rev d7f376ea) with the release's head; the six refusals (a qwen35 file with no head, --model @@ -1613,7 +1613,7 @@ gate-gpu-qwen38-media: # adds the clauses that start the seat from the hub with no projector flag (it fetches the repo's own projector and # serves images) and with --no-mmproj (it serves none): a network read. Either card. gate-gpu-clef-serve *ARGS: - BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_clef_serve && D=target/clef-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_clef_serve --model /models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --clef-model /root/models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --image-model /models/clef-flash/Cloudflare_clef-flash-Q8_0.gguf --mmproj /root/models/clef-flash/mmproj-Cloudflare_clef-flash-bf16.gguf --dir $D {{ARGS}}' + BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_clef_serve && D=target/clef-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_clef_serve --model /models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --clef-model /root/models/clef-flash/Cloudflare_clef-flash-Q5_K_M.gguf --image-model /models/clef-flash/Cloudflare_clef-flash-Q8_0.gguf --mmproj /root/models/clef-flash/mmproj-Cloudflare_clef-flash-bf16.gguf --dir $D {{ARGS}}' # The decide seat of bloomery-serve on lev (gate_lev_serve's doc): ggml-org/lev-GGUF's Q4_K_M (3.0 GB, a `qwen35` # file with `qwen35.decision.type = lev`, fetched into /root/models/lev-4b/) with no head flag; five refusals before @@ -1623,7 +1623,7 @@ gate-gpu-clef-serve *ARGS: # prompts' ids; ARGS `--hf ggml-org/lev-GGUF:Q4_K_M` adds the clause that serves the file from the hub, no head flag, and # answers the first request as the `-m` server does (a network read). Either card. gate-gpu-lev-serve *ARGS: - BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_lev_serve && D=target/lev-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_lev_serve --model /root/models/lev-4b/lev-Q4_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --ref $BLOOMERY_DATA/lev/4b/ref --dir $D {{ARGS}}' + BLOOMERY_MODEL=qwen35 ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_lev_serve && D=target/lev-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_lev_serve --model /root/models/lev-4b/lev-Q4_K_M.gguf --head /models/clef-flash/hf/joint_head.safetensors --ref $BLOOMERY_DATA/lev/4b/ref --dir $D {{ARGS}}' # clef_hidden on the box (ARGS as its doc: --model, --ids, --out, ...): a qwen35 file's prompt-only pass, every # position's final-norm hidden state to a file and a `clef hidden` record with the call's functional wall. @@ -1761,7 +1761,7 @@ gate-gpu-minimaxm3-prefill: [group('solo-real')] [group('v41-load')] gate-gpu-minimaxm3-serve: - BLOOMERY_MODEL=minimax-m3 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' + BLOOMERY_MODEL=minimax-m3 ./tools/box.sh 'export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' # MiniMax-M3.1-Flash on plan (b′) (gate_minimaxm3_prefill's doc, its `--arm bp`): the stage on the A6000 and the expert tier on the 3090, # both planned under one card budget the gate sets, against the same plan with every tier segment on the stage card (the A6000 alone, no @@ -1774,7 +1774,7 @@ gate-gpu-minimaxm3-serve: [group('solo')] [group('v41-load')] gate-gpu-minimaxm3-twocard: - BLOOMERY_MODEL=minimax-m3 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-minimaxm3-twocard && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features minimaxm3,checkpoint --release --bin gate_minimaxm3_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_minimaxm3_prefill --arm bp && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' + BLOOMERY_MODEL=minimax-m3 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-minimaxm3-twocard && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features minimaxm3,checkpoint --release --bin gate_minimaxm3_prefill && BLOOMERY_GATE_BOUND=2400 bash tools/gpu-gate.sh gate_minimaxm3_prefill --arm bp && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_minimaxm3_serve && D=target/minimaxm3-serve-twocard && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1800 bash tools/gpu-gate.sh gate_minimaxm3_serve --dir $D' # Nemotron-H MoE (Lightning) with its 32-value experts on the card (Relu2Leg for the step, the 32-value GEMM for the # prompt): host, card (the 3090's room, split) and whole (the A6000's room, box.sh's pick) arms near the host path [derived: 8..12 min, @@ -1799,7 +1799,7 @@ gate-gpu-gemma-embedding-head256: # greedy tokens with embedding calls in flight, and named refusals; the runner holds one card. [group('pool')] gate-gpu-gemma-embedding2-serve *ARGS: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-gemma-embedding2-serve && bash tools/gate.sh -p bloomery-gpu-gates --features hf --test sentence_prompts && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding2-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D" {{ARGS}} && if [ -z "{{ARGS}}" ]; then BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D/audio" --audio; fi' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-gemma-embedding2-serve && bash tools/gate.sh -p bloomery-gpu-gates --features hf --test sentence_prompts && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding2-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D" {{ARGS}} && if [ -z "{{ARGS}}" ]; then BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --dir "$D/audio" --audio; fi' # EmbeddingGemma 2's encoder (gpu-gemma-embedding2) end to end on the Q8_0 file the sets are dumped from # (refset's gemma_embedding2::MODEL, held to its sha256; the sets' `# model` line names the same path): every @@ -1811,7 +1811,7 @@ gate-gpu-gemma-embedding-e2e: BLOOMERY_MODEL=gemma-embedding ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gemma-embedding2,vision,deepseek41 --release --bin gate_gemma_embedding2_e2e && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_e2e --v1' gate-gpu-gemma-embedding-serve: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-gemma-embedding-serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --v1 --dir "$D"' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-gemma-embedding-serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_gemma_embedding2_serve && D=target/embedding-serve-gate-$(date +%s)-$$ && mkdir -p "$D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_gemma_embedding2_serve --v1 --dir "$D"' gate-gpu-gemma-embedding2-e2e *ARGS: ./tools/box.sh 'bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-gemma-embedding2 --lib -- --include-ignored' @@ -1857,10 +1857,20 @@ gate-gpu-mimo2-flash *ARGS: gate-gpu-flint-attn: ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_flint_attn && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_flint_attn' -# Flint's 256-expert top-6 sigmoid router. Model-free: one token, scale 1, zero bias. +# Flint's floor-normalized top-six router: optional selection bias, rows and captured replay. gate-gpu-flint-router: ./tools/box.sh 'cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin gate_flint_router && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_flint_router' +# Full-file eager/captured, resident-slot and ring-retention consumer transitions. +gate-gpu-flint-lib: + ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-flint-lib && bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-flint --lib -- --include-ignored --exact tests::hw_full_file_capture_slots_reset_and_ring_boundary' + +# Real two-card top-six prompt/decode, union reference, capture and tier fault recovery. +[group('solo')] +[group('v41-load')] +gate-gpu-flint-twocard: + BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-flint-twocard && export BLOOMERY_GATE_V41_LOAD=1 && bash tools/gate.sh --gpu --oxide --release -p bloomery-gpu-flint --lib -- --include-ignored --exact two_card_tests::hw_two_card_full_file_union_capture_publication_fault_reset' + # Muse Spark's 256-expert top-8 sigmoid router (`sigmoid_256_8`). Model-free like the Flint one, # plus the bias clause: the selection follows the biased scores, the weights the unbiased ones. gate-gpu-muse-router: @@ -2474,7 +2484,7 @@ gate-q38-serve-rules: # gate lock. [group('host')] gate-serve-draft-rules: - ./tools/box.sh 'bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features glm5next,clef,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41' + ./tools/box.sh 'bash tools/gate.sh --oxide -p bloomery-gpu-gates --release --features glm5next,clef,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --bin bloomery-serve --bin bloomery-serve-ds41' # The V4.1 binaries' `--place` word (`generate::Place`, behind the deepseek41 feature like bind): each alias and its card # list (`a6000+3090` is `bp`) planning the machine its plan function makes, a list word planning its own cards, and every @@ -2970,7 +2980,7 @@ gate-gpu-qwen38-serve: # _decode_*): the qwen3moe file's --mmproj refusal and, on the Qwen3.6 file, the seat with its encoder. [group('pool')] gate-gpu-qwen3-serve: - BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen3-serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/qwen3-serve-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && Q38D=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen38d.sh) && test -n "$Q38D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen3_serve --model "$BLOOMERY_REF_MODEL" --model "$Q36" --model "$Q38D" --dir $D --images' + BLOOMERY_MODEL=qwen3moe ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-qwen3-serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_qwen3_serve && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features gpu --release --bin generate_qwen3moe && D=target/qwen3-serve-gate && rm -rf $D && mkdir -p $D && Q36=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen35moe.sh) && test -n "$Q36" && Q38D=$(sed -n "s/^MODEL=\${BLOOMERY_REF_MODEL:-\(.*\)}$/\1/p" tools/ref/models/qwen38d.sh) && test -n "$Q38D" && BLOOMERY_GATE_CARD=${BLOOMERY_GATE_CARD:-any} bash tools/gpu-gate.sh gate_qwen3_serve --model "$BLOOMERY_REF_MODEL" --model "$Q36" --model "$Q38D" --dir $D --images' # bloomery-serve-qwen38 on the box, for a person to attach a client to (toktape records from it): the A6000 by # default (SERVE_PLACE=gate for the 3090), the port 8080 unless SERVE_PORT, a four-hour gate bound unless @@ -3089,7 +3099,7 @@ depth-gpu-glm5next *ARMS: # BLOOMERY_BOX_ENV=BLOOMERY_DRY=1 nothing is built and no lease is taken: the plan and each binary's state. depth-serve *ARMS: @[ -n "${BLOOMERY_MODEL:-}" ] || { echo "depth-serve: BLOOMERY_MODEL is unset: name the model profile (tools/ref/models/.sh), e.g. BLOOMERY_MODEL=mimo2 just depth-serve :" >&2; exit 64; } - ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && { here=$(basename "$PWD"); ours=; for a in {{ARMS}}; do t=${a#env=*:}; [ "${t%%:*}" != "$here" ] || ours=1; done; if [ -n "$ours" ] && [ -z "${BLOOMERY_DRY:-}" ]; then cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve; fi; } && bash tools/ref/depth-serve.sh {{ARMS}}' + ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && { here=$(basename "$PWD"); ours=; for a in {{ARMS}}; do t=${a#env=*:}; [ "${t%%:*}" != "$here" ] || ours=1; done; if [ -n "$ours" ] && [ -z "${BLOOMERY_DRY:-}" ]; then cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve; fi; } && bash tools/ref/depth-serve.sh {{ARMS}}' # The schemajump against Clef-Flash sitting: the Clef decide seat and a Qwen3.6 seat, both bloomery-serve, one lease on # the A6000, the arms alternating request by request (the design's §4; the cards are docs/cards/sj-*.card). Two windows @@ -3110,7 +3120,7 @@ depth-serve *ARMS: # bloomery-serve-ds41 picks, with SJ_DS41_MODEL; this recipe builds that binary when the windows name it) and SJ_SUITES # other suite files (`tools/ref/clef/suite-hard.jsonl`, scored against its `.gold.jsonl`); the header of the runner has both. sj-vs-clef: - ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && if [ -z "${BLOOMERY_DRY:-}" ]; then case " $(printf %s "${SJ_WINDOWS:-qwen:on qwen:off}" | tr , " ") " in *" qwen:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve;; esac && case " $(printf %s "${SJ_WINDOWS:-}" | tr , " ") " in *" ds41:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features deepseek41,vision,gemma-embedding2,hf --release --bin bloomery-serve-ds41;; esac; fi && bash tools/ref/sj-vs-clef.sh' + ./tools/box.sh "${BLOOMERY_AB_ROUNDS:+export BLOOMERY_AB_ROUNDS=$BLOOMERY_AB_ROUNDS && }"'{{precheck}} && if [ -z "${BLOOMERY_DRY:-}" ]; then case " $(printf %s "${SJ_WINDOWS:-qwen:on qwen:off}" | tr , " ") " in *" qwen:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve;; esac && case " $(printf %s "${SJ_WINDOWS:-}" | tr , " ") " in *" ds41:"*) cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features deepseek41,vision,gemma-embedding2,hf --release --bin bloomery-serve-ds41;; esac; fi && bash tools/ref/sj-vs-clef.sh' # Qwen3-30B-A3B decode-step kernel timeline (nsys, A6000, under the lease, lead-only): generate_qwen3moe's seed form at each # depth (default 6 4096), one prefill pass + BLOOMERY_NSYS_N - 1 replays, the cache height depth-qwen3moe.sh uses. The @@ -3673,7 +3683,7 @@ gate-gpu-glm5next-residency: [group('solo')] [group('v41-load')] gate-gpu-glm5next-serve: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-glm5next-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,hf,glm5next-hf --release --bin generate_glm5next && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-serve-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm plain --dir $D/plain && bash tools/gpu-gate.sh gate_glm5next_serve --arm drafted --dir $D/drafted' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh gate-gpu-glm5next-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,hf,glm5next-hf --release --bin generate_glm5next && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-serve-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm plain --dir $D/plain && bash tools/gpu-gate.sh gate_glm5next_serve --arm drafted --dir $D/drafted' # The GLM seat's image input beside its draft (gate_glm5next_serve --arm media): one server of --parallel 2 under # BLOOMERY_DRAFT=mtp BLOOMERY_RESIDENCY=off with --mmproj, the GLM-5.3 projector, then an image chat on one slot and a @@ -3688,7 +3698,7 @@ gate-gpu-glm5next-serve: [group('solo')] [group('v41-load')] weekly-gpu-glm5next-media-serve: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-glm5next-media-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-media-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm media --dir $D --mmproj /models/mmproj/unsloth--GLM-5.3-Flash-GGUF/mmproj-BF16.gguf' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-glm5next-media-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_glm5next_serve && D=target/glm-media-gate && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_glm5next_serve --arm media --dir $D --mmproj /models/mmproj/unsloth--GLM-5.3-Flash-GGUF/mmproj-BF16.gguf' # The shipped `bloomery-serve` started once per image flag on the GLM-5.3 fixture (gate_serve_start's header has the clauses): (s1) # `-m ` with the serving defaults and one text chat; (s2) `--place a --mmproj `, the tower's @@ -3706,7 +3716,7 @@ weekly-gpu-glm5next-media-serve: # 8.6 GB set and the 1.2 GB projector in threads beside the first two starts: 53 to 98 s warm; the bound is the runner's 900 s. [group('solo')] gate-gpu-serve-start-glm: - BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-glm && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/glm && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family glm --dir $D' + BLOOMERY_MODEL=glm5next BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-glm && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/glm && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family glm --dir $D' # The same gate on the V4.1 fixture (the file's architecture picks the seat, which has no `--plan`, so its plan is read from each # start's own log). The projector is the `# mmproj` line of the V4.1 vision set ($BLOOMERY_DATA/ref-vision/deepseek41v, `just dump-ref-vision`), @@ -3717,7 +3727,7 @@ gate-gpu-serve-start-glm: # beside the first two starts, a warm build: 72 to 132 s warm; the bound is the runner's 900 s. [group('solo')] gate-gpu-serve-start-ds41: - BLOOMERY_MODEL=deepseek41 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-ds41 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/ds41 && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family ds41 --dir $D' + BLOOMERY_MODEL=deepseek41 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/fixture-only.sh gate-gpu-serve-start-ds41 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_serve_start && D=target/serve-start-gate/ds41 && rm -rf $D && mkdir -p $D && bash tools/gpu-gate.sh gate_serve_start --family ds41 --dir $D' # The MiMo seat of `bloomery-serve` (gate_mimo2_serve): the refusals before any load (--plan's default context, the one # slot, the tier card), one server at --ctx 4160 --parallel 1 holding its load and listening records, the three prompts' @@ -3734,7 +3744,7 @@ gate-gpu-serve-start-ds41: [group('solo')] [group('v41-load')] weekly-gpu-mimo2-serve: - BLOOMERY_MODEL=mimo2 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-mimo2-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_mimo2_serve && D=target/mimo2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1680 bash tools/gpu-gate.sh gate_mimo2_serve --dir $D' + BLOOMERY_MODEL=mimo2 BLOOMERY_CARD=both ./tools/box.sh 'bash tools/ref/real-only.sh weekly-gpu-mimo2-serve && export BLOOMERY_GATE_V41_LOAD=1 && cargo oxide build --arch {{arch}} -- -p bloomery-gpu-gates --features glm5next,clef,mimo2,solar2,minimaxm3,flint,vision,gemma-embedding2,hf,checkpoint --release --bin bloomery-serve --bin gate_mimo2_serve && D=target/mimo2-serve-gate && rm -rf $D && mkdir -p $D && BLOOMERY_GATE_BOUND=1680 bash tools/gpu-gate.sh gate_mimo2_serve --dir $D' # glm5next decode CLI, functional run (no timing): generate_glm5next feeds --tokens one step per id, then greedy -n # tokens. The gate placement on the 3090 unless --place a (and BLOOMERY_CARD=a6000). 3090, gate lock. diff --git a/tools/families.tsv b/tools/families.tsv index acca29727..451e5f0a4 100644 --- a/tools/families.tsv +++ b/tools/families.tsv @@ -10,7 +10,7 @@ # fixture yes (the family has a fixture file, tools/fixture-dirs.tsv) or no deepseek2 F crates/model/src/arch/deepseek2/** crates/gpu/src/arch/deepseek2/** 7 no deepseek41 R crates/gpu-deepseek41/** crates/model/src/arch/deepseek41/** crates/refset/src/arch/deepseek41/** crates/refset/src/arch/deepseek41v/** crates/vision/src/arch/deepseek41v/** crates/gpu-gates/src/bin/gate_deepseek41_*.rs crates/gpu-gates/src/bin/gate_ds41_*.rs crates/gpu-gates/src/bin/gate_dspark_*.rs crates/gpu-gates/src/bin/shared/ds41_*.rs crates/gpu-gates/src/bin/shared/dspark_hf.rs crates/gpu-gates/src/bin/shared/serve_seats/ds41.rs crates/gpu-gates/src/bin/shared/routed_legs/deepseek41.rs 7 yes -flint F crates/model/src/arch/flint/** crates/model/tests/flint_meta.rs crates/gpu/src/flash_gqa_h128_swa.rs crates/gpu/src/qk_rms.rs crates/gpu-gates/src/bin/gate_flint_attn.rs crates/gpu-gates/src/bin/gate_flint_router.rs 7 no +flint F crates/model/src/arch/flint/** crates/model/tests/flint_meta.rs crates/gpu-flint/** crates/app/src/arch/flint.rs crates/gpu-gates/src/bin/shared/serve_seats/flint.rs crates/gpu/src/attention_h128.rs crates/gpu/src/attention_h128_ring.rs crates/gpu/src/flash_gqa_h128_swa.rs crates/gpu/src/qk_rms.rs crates/gpu-gates/src/bin/gate_flint_attn.rs crates/gpu-gates/src/bin/gate_flint_router.rs 7 no gemma-embedding F crates/model/src/arch/gemma_embedding/** crates/refset/src/arch/gemma_embedding/** crates/gpu-gates/src/bin/gate_gemma_embedding_*.rs 7 no gemma-embedding2 F crates/gpu-gemma-embedding2/** crates/model/src/arch/gemma_embedding2/** crates/refset/src/arch/gemma_embedding2/** crates/gpu-gates/src/bin/gate_gemma_embedding2_*.rs 7 no gemma4a F crates/gpu-gemma4a/** crates/vision/src/arch/gemma4a.rs crates/gpu-gates/src/bin/gate_gemma4a.rs 7 no diff --git a/tools/gate-batch-resources.tsv b/tools/gate-batch-resources.tsv index e4fdb0f90..de831a023 100644 --- a/tools/gate-batch-resources.tsv +++ b/tools/gate-batch-resources.tsv @@ -36,6 +36,11 @@ # so the e2e's 187e9 bounds its host bytes [derived]. # qwen3moe-twocard: unmeasured ceiling, the qwen38-twocard row's figure (qwen4exp --place gate need, 92,054,154,240 B). # The two loads run one after the other. Not a measurement of the Qwen3-30B or Qwen3.6 plan. +# flint-twocard: [derived] pinned GGUF, 1024 context, one sequence, residency/R8 off: +# HostNeed with every routed file tensor on the host is 24,168,079,360 B; +# four histories of 75 * (49 * 2048 + 150144) f32 bound retained readbacks and temporaries, +# 300,595,200 B. Sequential model loads: total ceiling 24,468,674,560 B, not measured RSS. +gate-gpu-flint-twocard - 24468674560 gate-gpu-ds41-callstream - 237600000000 gate-gpu-ds41-long m 237600000000 gate-gpu-ds41-twocard - 237600000000 diff --git a/tools/gate-class.tsv b/tools/gate-class.tsv index 06daa21cc..4846f6c9a 100644 --- a/tools/gate-class.tsv +++ b/tools/gate-class.tsv @@ -76,6 +76,8 @@ gate-gpu-dspark-kv family deepseek41 draft drafts_dspark gate-gpu-e2e family deepseek2 e2e graph_fault+moe_route+weights_load gate-gpu-flint-attn family flint kernel attn_gqa gate-gpu-flint-router family flint kernel moe_route +gate-gpu-flint-lib family flint e2e graph_fault+attn_gqa+moe_route+tier_legs+placement_sizing +gate-gpu-flint-twocard family flint tier tier_legs+node_link+placement_sizing+prompt_walks+graph_fault gate-gpu-muse-router common kernel moe_route gate-gpu-gates-lib tooling harness - gate-gpu-gemm common kernel quant_gemm diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 401ee37a2..2ce4135c5 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -114,7 +114,7 @@ tools/census/stubs.tsv derive the census manifest (refset::census::MANIFEST name # arch/coverage.rs and arch/qwen35moe/mtp.rs are carved out to their own rows below. crates/model/src/arch/{mod.rs,registry.rs,dspark.rs,classify.rs,stems.rs} derive gate-head gate-r8 gate-ptx-spill the dispatch's own files (Arch and spec, the registry, the shared architecture reads and the DSpark reader): every gate that links one, less the gguf-only kernel gates; the family modules behind the registry's rows are reached from them only by the gates that can open that family (tools/recipes.py, Reach) crates/model/src/arch/deepseek2/** derive gate-head gate-r8 the V2-Lite engine's reader and forward: the gates whose targets name it or call what it calls from another module (qwen35moe/mod.rs embeds through deepseek2::forward), and the gates that open a V2-Lite file (its profile, the names of their targets, the /models paths they name) -crates/model/src/arch/flint/** derive Flint's header-only reader and exact roles: its metadata gate and the common registry's architecture dispatch; no executable program or fixture +crates/model/src/arch/flint/** derive Flint's validated reader, placement and host expert adapter: metadata and the actual head128 body/app/serve consumers; no redistributed fixture crates/model/src/arch/qwen3moe/** derive the Qwen3 reader and plan: the gates whose targets name it, call it from another crate, or open a qwen3moe file (their profile, the names of their targets, the /models paths they name) crates/model/src/arch/glm5next/** derive the GLM reader and plan: the gates whose targets name it, call it from another crate (the GLM body's chains), or open a glm5next file (their profile, the names of their targets, the /models paths they name) crates/model/src/arch/mimo2/** derive weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve MiMo-V2.6-Flash's reader and plan: the gates that open a mimo2 file (its meta test, its flash gate, bloomery-serve's mimo2 seat), and the gates whose code calls it except gate-fixture: gate-fixture walks the registry's rows and runs no mimo2 reader on a file @@ -139,6 +139,7 @@ crates/gpu/src/arch/qwen3moe/tier_leg.rs derive the Qwen3 and Qwen3.6 expert-tie crates/gpu/src/site.rs derive gate-ds41-bind gate-gpu-gemm gate-gpu-nvtier gate-gpu-qwen35moe-moe gate-gpu-qwen38-residency gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp the weight sites' launches by type: read by the Qwen bodies and the Clef hidden gate, the bodies that read a site in more than one type crates/gpu/src/arch/deepseek2/** derive gate-ds41-bind the V2-Lite body: the V2-Lite GPU gates and AnyEngine (gate_e2e) crates/gpu/src/arch/musespark/** derive the muse-spark card body: the end-to-end gate names it, and every bin that links bloomery-gpu reaches it through the lib +crates/gpu-flint/** derive Flint's head128 body, resident slots and actual single/two-card consumers; the two-card gate retains its independent runtime filter # Derived rows: each row's recipes are `python3 tools/recipes.py reach --row ` at the time of writing, the # gates whose targets can run code of the files through another lib (tools/recipes.py, Reach); files grouped only diff --git a/tools/hf-arch-check.py b/tools/hf-arch-check.py index 0a479adae..0d0817121 100755 --- a/tools/hf-arch-check.py +++ b/tools/hf-arch-check.py @@ -112,7 +112,7 @@ def load_by_path(name, *rel): "mimo2": ("mimo2",), } SERVES_VIA_FROM_NAME = {"glm": ("Glm5next",), "solar2": ("SolarOpen2",), "minimaxm3": ("MiniMaxM3",), "nemotron": ("NemotronH", "NemotronHMoe"), - "raon1": ("Raon1",), "musespark": ("MuseSpark",)} + "raon1": ("Raon1",), "musespark": ("MuseSpark",), "flint": ("Flint",)} # crates/gpu-gates/src/qwen_seat.rs:Family::pick — the programs a program-served seat serves (Model::serves asks # Family::of). crates/placement/src/accepts.rs:program_of — the program that runs each crates/models Arch variant, here # the variants of those programs; crates/models/src/lib.rs:Arch::name — the architecture string of each variant. @@ -123,7 +123,7 @@ def load_by_path(name, *rel): MODELS_ARCH_NAME = {"Qwen3Moe": "qwen3moe", "Qwen35Moe": "qwen35moe", "Qwen35": "qwen35", "Llama": "llama"} # crates/gpu-gates/src/bin/bloomery_serve.rs:Model::ALL and Model::word — the generative seats in the order # the refusal of an unserved file lists them; crates/serve/src/decide.rs:WORD is the decide seat's word. -SEAT_ORDER = ("ds41", "qwen38", "glm", "qwen3", "mimo2", "solar2", "minimaxm3", "nemotron", "raon1", "musespark") +SEAT_ORDER = ("ds41", "qwen38", "glm", "qwen3", "mimo2", "solar2", "minimaxm3", "nemotron", "raon1", "musespark", "flint") DECIDE_WORD = "decide" # tools/arch-registry.tsv, the registry's print: FROM_NAME is crates/model/src/arch/mod.rs:Arch::from_name — variant -> the # strings that name it; SPEC is crates/model/src/arch/mod.rs:spec — the architectures the model reader takes (the @@ -2018,7 +2018,7 @@ def row_line(seat, arch, tensors=(), extra=(), vocab=0, quant="Q4_K_M", cards=No line, green, _ = row_line("glm", "glm5nextx", small) check("a glm5nextx file is refused by name", not green and "arch=glm5nextx route=REFUSED: a glm5nextx file, which no seat serves; the seats are " - "--model ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, decide" in line, line) + "--model ds41, qwen38, glm, qwen3, mimo2, solar2, minimaxm3, nemotron, raon1, musespark, flint, decide" in line, line) line, green, _ = row_line("glm", "deepseek41", small) check("a file the router seats elsewhere is refused for the row's seat", not green and "in the ds41 seat, and the README row says glm" in line, line) diff --git a/tools/levers-direct.txt b/tools/levers-direct.txt index 404c9286b..771496572 100644 --- a/tools/levers-direct.txt +++ b/tools/levers-direct.txt @@ -65,6 +65,8 @@ crates/model/tests/deepseek4_meta.rs BLOOMERY_REF_MODEL stays: the model file th crates/model/tests/ds41_meta.rs BLOOMERY_DSPARK_MODEL stays: the DSpark draft file crates/model-hf/tests/fixture.rs BLOOMERY_DSPARK_MODEL stays: the DSpark draft file crates/model/tests/flint_meta.rs BLOOMERY_MODEL stays: the pinned model file or header prefix the harness opens +crates/gpu-flint/src/tests.rs BLOOMERY_MODEL stays: cfg(test) full-file hardware consumer opens this exact local model path, not a production serving lever +crates/gpu-flint/src/two_card_tests.rs BLOOMERY_MODEL stays: cfg(test) two-card full-file input, not a production serving lever crates/model/tests/gemma_embedding_meta.rs BLOOMERY_MODEL stays: the model file the harness opens crates/model/tests/gemma_embedding2_meta.rs BLOOMERY_MODEL stays: the model file the harness opens crates/model/tests/mt.rs BLOOMERY_MT_CHILD_DUMP stays: the gate's handshake with the child it runs diff --git a/tools/model-tax.tsv b/tools/model-tax.tsv index fe9af6618..049aeafbb 100644 --- a/tools/model-tax.tsv +++ b/tools/model-tax.tsv @@ -82,7 +82,7 @@ crates/gpu-gates/src/bin/shared/dspark_hf.rs qwen 1 baseline O 1: own path: an a crates/gpu-gates/src/bin/shared/emb2_image.rs gemma 1 baseline O 1: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu-gates/src/bin/shared/route_trace38.rs qwen 3 baseline O 3: own path: an alias row in tools/model-tokens.tsv or a move into the family directory crates/gpu-gates/src/bin/shared/routed_legs.rs deepseek 8 baseline P 9: MK2 program by body -crates/gpu-gates/src/bin/shared/routed_legs.rs flint 2 PIN(2026-10-23) flint: family_of and skip_of match every models::Arch exhaustively; the header-only reader has no program or routed-legs adapter +crates/gpu-gates/src/bin/shared/routed_legs.rs flint 2 PIN(2026-10-25) flint: family_of and skip_of match every models::Arch exhaustively; no common routed-legs adapter is registered, and the full-file body transitions have their own owning gate crates/gpu-gates/src/bin/shared/routed_legs.rs gemma 4 baseline P 5: MK2 program by body crates/gpu-gates/src/bin/shared/routed_legs.rs glm 7 baseline P 8: MK2 program by body crates/gpu-gates/src/bin/shared/routed_legs.rs mimo 4 baseline P 5: MK2 program by body @@ -165,7 +165,7 @@ crates/models/src/lib.rs mimo 2 baseline A 2: MK1a/MK1b ArchRow registry crates/models/src/lib.rs nemotron 4 baseline A 4: MK1a/MK1b ArchRow registry crates/models/src/lib.rs qwen 9 baseline A 9: MK1a/MK1b ArchRow registry crates/placement/src/accepts.rs deepseek 14 PIN(2026-10-26) deepseek: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet; +4 are V4.1's leg arms (ds41_gate_up, ds41_down, ds41_pair and its routed_rule arm), held in the Program-keyed arm table beside qwen35's and qwen38's until a Program owns its arms -crates/placement/src/accepts.rs flint 1 PIN(2026-10-23) flint: program_of matches every models::Arch; the header-only reader must explicitly return no program until a body is registered +crates/placement/src/accepts.rs flint 5 PIN(2026-10-25) flint: the implemented body registers its Program variant, name, architecture selection, expert arms and routed rule in the existing exhaustive program owners crates/placement/src/accepts.rs gemma 5 PIN(2026-10-21) gemma: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs glm 5 PIN(2026-10-21) glm: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet crates/placement/src/accepts.rs mimo 5 PIN(2026-10-21) mimo: MK2 program by body; +1 is the family's word in the common Program::name owner, which the model's own code does not carry yet @@ -218,6 +218,8 @@ crates/gpu-gates/src/record.rs muse 4 PIN(2026-10-23) muse: MK12 per-bin Kind st crates/model/src/arch/coverage.rs muse 25 PIN(2026-10-23) muse: the coverage rows, the type pins and the expert-stack match are keyed by Program, so each row the musespark body runs lists its variant; the program would own its rows instead of the list naming programs crates/hf/src/vocab.rs minimax 3 PIN(2026-10-23) minimax: the minimax-m2 pre-tokenizer's name, regex and pipeline row; tokenizer programs are not yet described by ModelSpec crates/gpu-gates/src/prompts.rs deepseek 1 PIN(2026-10-23) deepseek: the measured logit σ is named for the one family it was measured on (SIGMA_LOGIT_V41); other families carry it in their gate description until their own is measured +crates/gpu-gates/src/bin/bloomery_serve.rs flint 10 PIN(2026-10-25) flint: MK9 seat by program: the seat word, help/models list, architecture match and dispatch register the implemented shared hybrid seat until the description selects it +crates/model/src/arch/coverage.rs flint 11 PIN(2026-10-25) flint: the implemented Program selects its actual head128 attention, floor router, dense/shared sites, embedding/head and expert-stack capabilities in the existing coverage owners crates/gpu-gates/src/bin/shared/routed_legs.rs llama 2 PIN(2026-10-26) llama: family_of and skip_of match every models::Arch exhaustively; the dense trunk has no routed-legs adapter crates/model/src/arch/mod.rs llama 2 PIN(2026-10-26) llama: the family's module declaration (`pub mod llama;`) and the SentencePiece rule's tokenizer-model word; the module list is not a registry crates/models/src/lib.rs llama 2 PIN(2026-10-26) llama: ModelSpec.arch names the family by an enum variant and its name arm; the variant stays until ModelSpec carries the registry's name instead diff --git a/tools/recipes.py b/tools/recipes.py index dbade4362..610c465e2 100755 --- a/tools/recipes.py +++ b/tools/recipes.py @@ -13072,7 +13072,7 @@ def _st_host_crates(fails: list[str], side: Side) -> None: f"host nightly: the real tree's set is {host_now} — a new member of the set is proven by a nightly run before this list takes it", ) left = [n for n, _ in host_left_out(tree, lock_now)] - expect(left == ["bloomery-gpu", "bloomery-gpu-deepseek41", "bloomery-gpu-gates", "bloomery-gpu-gemma-embedding2", "bloomery-gpu-gemma4a", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-minimaxm3", "bloomery-gpu-solar2", "bloomery-gpu-vision"], f"host nightly: left out {left}") + expect(left == ["bloomery-gpu", "bloomery-gpu-deepseek41", "bloomery-gpu-flint", "bloomery-gpu-gates", "bloomery-gpu-gemma-embedding2", "bloomery-gpu-gemma4a", "bloomery-gpu-glm5next", "bloomery-gpu-mimo2", "bloomery-gpu-minimaxm3", "bloomery-gpu-solar2", "bloomery-gpu-vision"], f"host nightly: left out {left}") expect(not nightly_problems(tree, lock_now), f"host nightly: {nightly_problems(tree, lock_now)}") ran = {u for u, _ in host_units(tree, lock_now)} expect( diff --git a/tools/ref/dense-logits.py b/tools/ref/dense-logits.py index 51c46a851..388baf042 100644 --- a/tools/ref/dense-logits.py +++ b/tools/ref/dense-logits.py @@ -249,7 +249,7 @@ def native_build(args): env = os.environ.copy() env["CARGO_TARGET_DIR"] = str(ROOT / "target") argv = ["cargo", "oxide", "build", "--arch", args.arch, "--", "-p", "bloomery-gpu-gates", - "--features", "glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint", "--release", + "--features", "glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint", "--release", "--bin", "generate_qwen3moe", "--bin", "bloomery-serve"] subprocess.run(argv, cwd=ROOT, env=env, check=True, timeout=1800) if clean_head(ROOT) != head: diff --git a/tools/ref/flint_trace.cpp b/tools/ref/flint_trace.cpp new file mode 100644 index 000000000..36bf894f8 --- /dev/null +++ b/tools/ref/flint_trace.cpp @@ -0,0 +1,359 @@ +// Flint forced-ID diagnostic trajectory, not a reference-set family or accuracy gate. +// Only the last input requests logits: the prompt's last layer is therefore already cropped. +// Node taps change scheduler segmentation; --no-taps is the same-engine logits-only control. +// Model SHA256 and binary/source identity belong in the external execution receipt. +#include "llama.h" +#include "ggml.h" +#include "ggml-backend.h" +#include "ref_io.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void require(bool ok, const std::string & message) { + if (!ok) throw std::runtime_error(message); +} + +static std::ofstream output(const fs::path & path, bool binary = false) { + std::ofstream stream; + stream.exceptions(std::ios::badbit | std::ios::failbit); + stream.open(path, std::ios::out | (binary ? std::ios::binary : std::ios::openmode(0))); + return stream; +} + +static void write_row(std::ofstream & out, const float * row, size_t width) { + out.write(reinterpret_cast(row), static_cast(width * sizeof(float))); +} + +static std::string metadata(llama_model * model, const char * key) { + char value[256]; + const int n = llama_model_meta_val_str(model, key, value, sizeof(value)); + require(n >= 0 && static_cast(n) < sizeof(value), std::string("missing/oversized model metadata: ") + key); + return value; +} + +static std::string actual_path(const std::string & path) { + const std::string result = fs::canonical(path).string(); + require(result.find_first_of("\t\r\n") == std::string::npos, "path contains a TSV separator: " + path); + return result; +} + +struct backend_guard { + backend_guard() { llama_backend_init(); } + ~backend_guard() { llama_backend_free(); } +}; + +struct batch_guard { + llama_batch batch = llama_batch_init(32, 0, 1); + ~batch_guard() { llama_batch_free(batch); } +}; + +struct tap_state { + int width; + int layers; + int call = -1; + int call_tokens = 0; + bool active = false; + std::vector names; + std::vector seen; + std::vector rows; + std::vector staging; + std::ofstream * tensors; + std::string error; + + tap_state(int width_, int layers_, std::ofstream * tensors_) : + width(width_), layers(layers_), seen(layers_ + 1), rows(static_cast(layers_ + 1) * width_), + tensors(tensors_) { + for (int i = 0; i < layers; ++i) names.push_back("l_out-" + std::to_string(i)); + names.push_back("result_norm"); + staging.reserve(static_cast(width) * sizeof(float)); + } + + int selected(const char * name) const { + const auto it = std::find(names.begin(), names.end(), name); + return it == names.end() ? -1 : static_cast(it - names.begin()); + } + + void begin(int call_, int tokens_) { + call = call_; + call_tokens = tokens_; + active = true; + std::fill(seen.begin(), seen.end(), 0); + error.clear(); + } + + void capture(const ggml_tensor * t, int index) { + const std::string label = "call " + std::to_string(call) + " tensor " + t->name; + require(!seen[index], label + ": duplicate capture"); + const int expected_rows = index < layers - 1 ? call_tokens : 1; + require(t->ne[0] == width && t->ne[1] == expected_rows && t->ne[2] == 1 && t->ne[3] == 1, + label + ": invalid shape (expected [" + std::to_string(width) + "," + + std::to_string(expected_rows) + ",1,1])"); + size_t element_bytes; + switch (t->type) { + case GGML_TYPE_F32: element_bytes = sizeof(float); break; + case GGML_TYPE_F16: element_bytes = sizeof(ggml_fp16_t); break; + case GGML_TYPE_BF16: element_bytes = sizeof(ggml_bf16_t); break; + default: throw std::runtime_error(label + ": unsupported type " + ggml_type_name(t->type)); + } + require(t->buffer && t->data, label + ": unallocated tensor"); + require(t->nb[0] >= element_bytes, label + ": invalid element stride"); + const size_t selected_row = static_cast(t->ne[1] - 1); + const size_t bytes = ggml_nbytes(t); + require(selected_row == 0 || t->nb[1] <= bytes / selected_row, label + ": invalid row stride"); + const size_t offset = selected_row * t->nb[1]; + require(offset <= bytes && element_bytes <= bytes - offset, label + ": row outside tensor"); + require(static_cast(width - 1) <= (bytes - offset - element_bytes) / t->nb[0], + label + ": element strides outside tensor"); + const size_t span = static_cast(width - 1) * t->nb[0] + element_bytes; + staging.resize(span); + // Copy while the scheduler still owns this live output, including view strides. + ggml_backend_tensor_get(t, staging.data(), offset, span); + float * dest = rows.data() + static_cast(index) * width; + for (int i = 0; i < width; ++i) { + const uint8_t * src = staging.data() + static_cast(i) * t->nb[0]; + float value; + if (t->type == GGML_TYPE_F32) { + std::memcpy(&value, src, sizeof(value)); + } else if (t->type == GGML_TYPE_F16) { + ggml_fp16_t half; + std::memcpy(&half, src, sizeof(half)); + value = ggml_fp16_to_fp32(half); + } else { + ggml_bf16_t half; + std::memcpy(&half, src, sizeof(half)); + value = ggml_bf16_to_fp32(half); + } + if (!std::isfinite(value)) { + throw std::runtime_error(label + ": nonfinite element " + std::to_string(i)); + } + dest[i] = value; + } + *tensors << call << '\t' << t->name << '\t' << ggml_type_name(t->type); + for (int i = 0; i < 4; ++i) *tensors << '\t' << t->ne[i]; + for (int i = 0; i < 4; ++i) *tensors << '\t' << t->nb[i]; + *tensors << '\t' << bytes << '\t' << selected_row << '\t' << ggml_op_name(t->op) + << '\t' << (ggml_is_contiguous(t) ? 1 : 0) << '\n'; + seen[index] = 1; + } + + void finish() { + active = false; + require(error.empty(), error); + for (int i = 0; i <= layers; ++i) { + require(seen[i], "call " + std::to_string(call) + " tensor " + names[i] + ": missing capture"); + } + } +}; + +static bool on_node(ggml_tensor * tensor, bool ask, void * user) { + auto & state = *static_cast(user); + if (!state.active) return !ask; + const int index = state.selected(tensor->name); + if (ask) return index >= 0; + // Never unwind a C callback or cancel a partially computed graph as if it succeeded. + if (index >= 0 && state.error.empty()) { + try { + state.capture(tensor, index); + } catch (const std::exception & e) { + state.error = e.what(); + } + } + return true; +} + +int main(int argc, char ** argv) { + try { + const char * usage = "usage: flint_trace -m MODEL --ids FILE --prompt 22 --out NEWDIR --ctx 1024 [--no-taps]\n"; + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "--help") == 0 || std::strcmp(argv[i], "-h") == 0) { + std::fputs(usage, stdout); + std::fputs("Read exactly 96 whitespace-separated decimal token IDs; never retokenize or stop on EOG.\n" + "Matches HTTP: 21-ID prefix prefill, last prompt ID as next, then 74 forced steps.\n" + "Default: CPU layer/norm node taps and full logits. --no-taps: full logits only.\n" + "NEWDIR must not exist; MANIFEST.tsv is installed only after complete capture.\n", stdout); + return 0; + } + } + std::string model_arg, ids_arg, out_arg; + int prompt = 22, capacity = 1024; + bool taps = true; + for (int i = 1; i < argc; ++i) { + const bool more = i + 1 < argc; + if (std::strcmp(argv[i], "-m") == 0 && more) model_arg = argv[++i]; + else if (std::strcmp(argv[i], "--ids") == 0 && more) ids_arg = argv[++i]; + else if (std::strcmp(argv[i], "--out") == 0 && more) out_arg = argv[++i]; + else if (std::strcmp(argv[i], "--prompt") == 0 && more) { + if (!parse_i32("flint_trace", "--prompt", argv[++i], 1, prompt)) return 2; + } else if (std::strcmp(argv[i], "--ctx") == 0 && more) { + if (!parse_i32("flint_trace", "--ctx", argv[++i], 1, capacity)) return 2; + } else if (std::strcmp(argv[i], "--no-taps") == 0) taps = false; + else throw std::runtime_error(std::string("unknown/incomplete argument: ") + argv[i]); + } + require(!model_arg.empty() && !ids_arg.empty() && !out_arg.empty(), usage); + require(prompt == 22 && capacity == 1024, "flint-trajectory-v2 requires --prompt 22 --ctx 1024"); + const uint32_t endian = 1; + require(*reinterpret_cast(&endian) == 1 && sizeof(float) == 4 && + std::numeric_limits::is_iec559, "requires little-endian IEEE754 f32 host"); + const std::string model_path = actual_path(model_arg); + const std::string ids_path = actual_path(ids_arg); + require(out_arg.find_first_of("\t\r\n") == std::string::npos, "output path contains a TSV separator"); + struct stat model_stat {}; + require(stat(model_path.c_str(), &model_stat) == 0 && S_ISREG(model_stat.st_mode) && + model_stat.st_size == 25054688704LL, "model file is not the complete Flint UD-Q4_K_XL release size"); + + std::vector ids; + ids.reserve(96); + std::ifstream source(ids_path); + require(bool(source), "cannot read IDs"); + std::string record; + while (source >> record) { + int32_t id; + require(!record.empty() && record[0] >= '0' && record[0] <= '9' && parse_token_id(record, id), + "invalid decimal token ID at position " + std::to_string(ids.size()) + ": " + record); + require(ids.size() < 96, "ID file must contain exactly 96 IDs: extra token at position 96"); + ids.push_back(id); + } + require(source.eof() && !source.bad(), "cannot finish reading IDs"); + require(ids.size() == 96, "ID file must contain exactly 96 IDs, found " + std::to_string(ids.size())); + + backend_guard backend; + auto mp = llama_model_default_params(); + mp.n_gpu_layers = 0; + mp.load_mode = LLAMA_LOAD_MODE_MMAP; + mp.use_extra_bufts = false; + std::unique_ptr model( + llama_model_load_from_file(model_path.c_str(), mp), llama_model_free); + require(bool(model), "cannot load model"); + const int width = llama_model_n_embd(model.get()); + const int layers = llama_model_n_layer(model.get()); + const int vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get())); + const std::string arch = metadata(model.get(), "general.architecture"); + const std::string model_name = metadata(model.get(), "general.name"); + require(arch == "flint" && model_name == "Flint Preview" && width == 2048 && layers == 49 && vocab == 150144 && + metadata(model.get(), "general.file_type") == "15", "model metadata is not the Flint Preview UD-Q4_K_XL release"); + for (size_t i = 0; i < ids.size(); ++i) { + require(ids[i] >= 0 && ids[i] < vocab, "token outside model vocabulary at position " + std::to_string(i)); + } + const int calls = 1 + static_cast(ids.size()) - prompt; + require(fs::create_directory(out_arg), "output directory must be new: " + out_arg); + const fs::path out = fs::canonical(out_arg); + auto manifest = output(out / "MANIFEST.tsv.partial"); + auto steps = output(out / "steps.tsv"); + auto tokens = output(out / "tokens.ids"); + auto logits_out = output(out / "logits.f32", true); + auto tensors = output(out / "tensors.tsv"); + std::ofstream layers_out, norm_out; + if (taps) { + layers_out = output(out / "layers.f32", true); + norm_out = output(out / "norm.f32", true); + } + manifest << "protocol\tflint-trajectory-v2\nmodel\t" << model_path << "\narch\t" << arch + << "\nmodel_name\t" << model_name << "\nmodel_bytes\t" << model_stat.st_size + << "\nmodel_identity_validation\tmetadata-and-file-size;SHA256-in-external-receipt\n" + << "mode\tcpu\ninstrumentation\t" << (taps ? "node-taps" : "no-taps") + << "\nids_source\t" << ids_path << "\nwidth\t" << width << "\nlayers\t" << layers + << "\nvocab\t" << vocab << "\nprompt\t" << prompt << "\ntotal_ids\t" << ids.size() + << "\ncalls\t" << calls << "\nlogits\tlogits.f32\nlayers_file\t" << (taps ? "layers.f32" : "absent:not-requested") + << "\nprompt_submission\tprefix-prefill-then-next\nprefix_prefill_ids\t21\nfirst_observed_call\tone-id-next" + << "\nnorm_file\t" << (taps ? "norm.f32" : "absent:not-requested") + << "\nlogits_layout\t[call][vocab];little-endian-f32\nlayers_layout\t[call][layer][width];little-endian-f32" + << "\nnorm_layout\t[call][width];little-endian-f32\ntensor_metadata\ttensors.tsv" + << "\nngpu\t0\nload_mode\tmmap\nuse_extra_bufts\tfalse\nthreads\t4\nthreads_batch\t4" + << "\noffload_kqv\tfalse\nop_offload\tfalse\nswa_full\tfalse\nkv_unified\tfalse" + << "\ntype_k\tf16\ntype_v\tf16\nflash_attn\tauto\nno_perf\ttrue\n"; + manifest.flush(); + tensors << "call\tname\ttype\tne0\tne1\tne2\tne3\tnb0\tnb1\tnb2\tnb3\tbytes\tselected_row\top\tcontiguous\n"; + tap_state state(width, layers, &tensors); + auto cp = llama_context_default_params(); + cp.n_ctx = capacity; + cp.n_seq_max = 1; + cp.n_batch = cp.n_ubatch = 32; + cp.n_threads = cp.n_threads_batch = 4; + cp.offload_kqv = false; + cp.op_offload = false; + cp.no_perf = true; + cp.swa_full = false; + cp.kv_unified = false; + cp.type_k = cp.type_v = GGML_TYPE_F16; + cp.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_AUTO; + if (taps) { + cp.cb_eval = on_node; + cp.cb_eval_user_data = &state; + } + std::unique_ptr ctx(llama_init_from_model(model.get(), cp), llama_free); + require(bool(ctx), "cannot create CPU context"); + require(llama_n_ctx_seq(ctx.get()) == static_cast(capacity) && llama_n_seq_max(ctx.get()) == 1 && + llama_n_batch(ctx.get()) == 32 && llama_n_ubatch(ctx.get()) == 32, "actual context/batch capacities differ from requested protocol"); + manifest << "slots\t" << llama_n_seq_max(ctx.get()) << "\nctx_per_slot\t" << llama_n_ctx_seq(ctx.get()) + << "\nctx_total\t" << llama_n_ctx(ctx.get()) << "\nn_batch\t" << llama_n_batch(ctx.get()) + << "\nn_ubatch\t" << llama_n_ubatch(ctx.get()) << '\n'; + manifest.flush(); + for (llama_token id : ids) tokens << id << '\n'; + tokens.close(); + steps << "call\tprocessed_pos\tpredict_pos\tinput_id\targmax\n"; + batch_guard batch; + // Match HTTP Slot::close_call: the prefix has no observation; the last prompt ID is next. + for (int call = -1; call < calls; ++call) { + const int at = call < 0 ? 0 : prompt - 1 + call; + const int count = call < 0 ? prompt - 1 : 1; + auto & b = batch.batch; + b.n_tokens = count; + for (int i = 0; i < count; ++i) { + b.token[i] = ids[at + i]; + b.pos[i] = at + i; + b.n_seq_id[i] = 1; + b.seq_id[i][0] = 0; + b.logits[i] = i == count - 1; + } + if (taps && call >= 0) state.begin(call, count); + const int status = llama_decode(ctx.get(), b); + if (taps && call >= 0) state.finish(); + require(status == 0, "llama_decode failed at call " + std::to_string(call) + ": " + std::to_string(status)); + if (call < 0) continue; + const float * row = llama_get_logits_ith(ctx.get(), -1); + require(row != nullptr, "missing logits at call " + std::to_string(call)); + int argmax = 0; + for (int i = 0; i < vocab; ++i) { + if (!std::isfinite(row[i])) { + throw std::runtime_error("nonfinite logits at call " + std::to_string(call) + " lane " + std::to_string(i)); + } + if (row[i] > row[argmax]) argmax = i; + } + write_row(logits_out, row, vocab); + if (taps) { + write_row(layers_out, state.rows.data(), static_cast(layers) * width); + write_row(norm_out, state.rows.data() + static_cast(layers) * width, width); + } + const int processed = at + count - 1; + steps << call << '\t' << processed << '\t' << processed + 1 << '\t' << ids[processed] << '\t' << argmax << '\n'; + } + logits_out.close(); + steps.close(); + tensors.close(); + if (taps) { + layers_out.close(); + norm_out.close(); + } + manifest << "complete\t1\n"; + manifest.close(); + fs::rename(out / "MANIFEST.tsv.partial", out / "MANIFEST.tsv"); + std::fprintf(stderr, "flint_trace: %d calls, width=%d layers=%d vocab=%d taps=%s into %s\n", + calls, width, layers, vocab, taps ? "on" : "off", out.c_str()); + return 0; + } catch (const std::exception & e) { + std::fprintf(stderr, "flint_trace: %s\n", e.what()); + return 1; + } +} diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index b038d2937..70dde1732 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -543,6 +543,19 @@ generate_ds41 swiglu_rule_rows 0 0 # PIN(2026-09-25): ds41_attn_seg_stage is the staged prefix segment pass (prefill only, never on the decode path); its ring ⧺ staging source keeps a few registers more than ds41_attn_seg, which already sits at the 128-register cap of a 512-thread block, and ptxas spills 8 bytes. ds41_attn_seg_sel_stage fits in 112 with none. # PIN(2026-10-21): raon_hc_route (Raon's mHC route, one block a token and layer) keeps its 4x3 and 4x4 route matrices in per-thread arrays the while loops index at run time: hz 48 B + e 64 B = 112 B of JIT local; the rest of its work is shared memory. ptxas spills nothing. +# PIN(2026-10-25): Flint entries measured at 313b9f0f88c90fcfae2365a4af53cd077762eb7c: +# sm_86, ptxas 13.3.33, RTX 3090 cc8.6, driver 595.91.07 (CUDA API 13.2). +# Full generate_ds41 JIT: 436 entries, cache off; each new entry has measured spill=0/jit_local=0. +generate_ds41 gqa_flash_merge_h128_swa_ring 0 0 +generate_ds41 gqa_flash_seg_h128_swa_ring 0 0 +generate_ds41 head_norm_neox_append_ring 0 0 +generate_ds41 normalize 0 0 +generate_ds41 qk_rms_append_ring 0 0 +generate_ds41 restore 0 0 +generate_ds41 routed_shared_residual 0 0 +generate_ds41 sigmoid_floor_256_6_router 0 0 +generate_ds41 sigmoid_floor_256_6_router_pick 0 0 + # gate_qwen3vl_tower (--features vision), the shared vision entries, including Gemma4 residual and post-norm precision. # PIN(2026-10-15): the seven Gemma4 entries have zero sm_86 ptxas spill; jit_local=0 is a derived cc8.6 ceiling, # not a cc8.6 driver measurement. Both sm_86 and sm_121 PTX have zero JIT local bytes on GB10 cc12.1. diff --git a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv index 850f9ecdf..008e17595 100644 --- a/tools/ref/ptx-shapes/sm_121-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121-cc12.1.tsv @@ -552,3 +552,13 @@ generate_ds41 gqa_prefill_flash_g7 0 0 generate_ds41 gqa_prefill_flash_g7_q8 0 0 generate_ds41 rope_norm_append 0 0 generate_ds41 rope_norm_append_q8 0 0 +# PIN(2026-10-27): Flint's entries (gqa_flash_merge_h128_swa_ring, gqa_flash_seg_h128_swa_ring, head_norm_neox_append_ring, normalize, qk_rms_append_ring, restore, routed_shared_residual, sigmoid_floor_256_6_router, sigmoid_floor_256_6_router_pick) at the cc 8.6 scan's values (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 gqa_flash_merge_h128_swa_ring 0 0 +generate_ds41 gqa_flash_seg_h128_swa_ring 0 0 +generate_ds41 head_norm_neox_append_ring 0 0 +generate_ds41 normalize 0 0 +generate_ds41 qk_rms_append_ring 0 0 +generate_ds41 restore 0 0 +generate_ds41 routed_shared_residual 0 0 +generate_ds41 sigmoid_floor_256_6_router 0 0 +generate_ds41 sigmoid_floor_256_6_router_pick 0 0 diff --git a/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv b/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv index c951ae267..d09b463df 100644 --- a/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_121a-cc12.1.tsv @@ -550,3 +550,13 @@ generate_ds41 gqa_prefill_flash_g7 0 0 generate_ds41 gqa_prefill_flash_g7_q8 0 0 generate_ds41 rope_norm_append 0 0 generate_ds41 rope_norm_append_q8 0 0 +# PIN(2026-10-27): Flint's entries (gqa_flash_merge_h128_swa_ring, gqa_flash_seg_h128_swa_ring, head_norm_neox_append_ring, normalize, qk_rms_append_ring, restore, routed_shared_residual, sigmoid_floor_256_6_router, sigmoid_floor_256_6_router_pick) at the cc 8.6 scan's values (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 gqa_flash_merge_h128_swa_ring 0 0 +generate_ds41 gqa_flash_seg_h128_swa_ring 0 0 +generate_ds41 head_norm_neox_append_ring 0 0 +generate_ds41 normalize 0 0 +generate_ds41 qk_rms_append_ring 0 0 +generate_ds41 restore 0 0 +generate_ds41 routed_shared_residual 0 0 +generate_ds41 sigmoid_floor_256_6_router 0 0 +generate_ds41 sigmoid_floor_256_6_router_pick 0 0 diff --git a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv index de7cb39c0..b105176f9 100644 --- a/tools/ref/ptx-shapes/sm_86-cc12.1.tsv +++ b/tools/ref/ptx-shapes/sm_86-cc12.1.tsv @@ -556,3 +556,13 @@ generate_ds41 gqa_prefill_flash_g7 0 0 generate_ds41 gqa_prefill_flash_g7_q8 0 80 generate_ds41 rope_norm_append 0 0 generate_ds41 rope_norm_append_q8 0 0 +# PIN(2026-10-27): Flint's entries (gqa_flash_merge_h128_swa_ring, gqa_flash_seg_h128_swa_ring, head_norm_neox_append_ring, normalize, qk_rms_append_ring, restore, routed_shared_residual, sigmoid_floor_256_6_router, sigmoid_floor_256_6_router_pick) at the cc 8.6 scan's values (tools/ref/ptx-shapes.tsv), not measured on cc 12.1, until a GB10 scan runs. +generate_ds41 gqa_flash_merge_h128_swa_ring 0 0 +generate_ds41 gqa_flash_seg_h128_swa_ring 0 0 +generate_ds41 head_norm_neox_append_ring 0 0 +generate_ds41 normalize 0 0 +generate_ds41 qk_rms_append_ring 0 0 +generate_ds41 restore 0 0 +generate_ds41 routed_shared_residual 0 0 +generate_ds41 sigmoid_floor_256_6_router 0 0 +generate_ds41 sigmoid_floor_256_6_router_pick 0 0 diff --git a/tools/release/build.sh b/tools/release/build.sh index d48cd3277..0d999bf40 100755 --- a/tools/release/build.sh +++ b/tools/release/build.sh @@ -191,7 +191,7 @@ else bin_args=() for b in "${BINS[@]}"; do bin_args+=(--bin "$b"); done # The commit goes to cargo alone: the serve build script reads it, and a running binary never does. - BLOOMERY_GIT_COMMIT="$COMMIT" cargo oxide build --arch "$ARCH" -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,gemma-embedding2,hf,checkpoint --release "${bin_args[@]}" + BLOOMERY_GIT_COMMIT="$COMMIT" cargo oxide build --arch "$ARCH" -- -p bloomery-gpu-gates --features glm5next,clef,vision,mimo2,solar2,minimaxm3,flint,gemma-embedding2,hf,checkpoint --release "${bin_args[@]}" fi fail=0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index 932801ead..739c1687c 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -626,7 +626,10 @@ gguf 7 2 0 # PIN(2026-10-26): gpu 2691 -> 2693 blocks, the six-slot F32 tier entry adds two blocks: # ffn_handoff_6_tier calls handoff_at and handoff_tier_at under its checked launch contract; # both blocks state their per-thread bounds and disjoint-write SAFETY invariant. -gpu 2693 346 10 +# PIN(2026-10-27): gpu 2693 -> 2748 blocks, 346 -> 350 fns: the existing Flint head128 ring +# attention, floor router and expert-input scaling retain their checked SAFETY contracts over +# main's new kernels; the six-slot entry is already included above. Counted by check-unsafe.sh. +gpu 2748 350 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. @@ -653,7 +656,12 @@ gpu 2693 346 10 # selector partials of dense.rs and experts.rs, `q5k_lane_partial`'s calls and their row stores) and the glue's Q5_K # token rows and IQ4_NL Engram rows (glue.rs: the value reads, the stream and input stores, the shared module), each # with its SAFETY line; counted by tools/check-unsafe.sh. -gpu-deepseek41 395 44 0 +# PIN(2026-10-26): gpu-deepseek41 395 -> 397 blocks: the configured residual module load +# and its checked row launch retain their SAFETY invariants over main's ISTA entries. +gpu-deepseek41 397 44 0 +# PIN(2026-10-24): gpu-flint starts at 2 blocks, 0 fns and 0 impls: step and prompt +# call the checked common attention binding with staged distinct positions, each with SAFETY. +gpu-flint 2 0 0 # PIN(2026-10-01): gpu-gates 60 -> 61 blocks, new code, not old code relaxed: gate_glm5next_serve starts # bloomery-serve with PR_SET_PDEATHSIG set between fork and exec (one pre_exec block, its SAFETY line), as the # Qwen3.8 serve gate does, so a bound that ends the gate does not leave the server holding the card. @@ -719,7 +727,9 @@ gpu-deepseek41 395 44 0 # and the host flag's drop once the stream that waited on it is drained, each with its SAFETY line. # PIN(2026-10-24): gpu-gates 103 -> 104 blocks, new code, not old code relaxed: gate_qwen35moe_attn's launch of the # head-256 append at no query head over the rows rope_gate::run checks distinct, with its SAFETY line. -gpu-gates 104 5 1 +# PIN(2026-10-24): gpu-gates 104 -> 106 blocks: ring-attention consumers launch +# checked distinct positions and bounded chunks under their stated SAFETY invariants. +gpu-gates 106 5 1 # PIN(2026-10-15): gpu-gemma-embedding2 0 -> 1 blocks, new code, not old code relaxed: the head-512 # launcher call requires distinct Consts::pos positions, with its SAFETY line. # PIN(2026-10-15): gpu-gemma-embedding2 1 -> 2 blocks, new code, not old code relaxed: Encoder::pass's call of